freepdfeditor/spike/B_reconstruction/Corpus.cpp

// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: 2025 ai-ad4 and the FreePDFEditor contributors
//
// Corpus.cpp — synthetic labelled corpus generator for Spike B.

#include "Corpus.h"

#include <random>
#include <sstream>
#include <string>
#include <vector>

namespace freepdfeditor::spike::b {

namespace {

// A simple deterministic PRNG so the corpus is reproducible run-to-run, which
// matters for the F1 gate: a regression must be a real change, not PRNG drift.
// Seed is fixed; to re-roll the corpus, change the seed here.
std::mt19937_64& rng()
{
    static std::mt19937_64 r(0xF0BEE75EEDull);
    return r;
}

std::size_t rand_size(std::size_t lo, std::size_t hi)
{
    std::uniform_int_distribution<std::size_t> d(lo, hi);
    return d(rng());
}

// Lorem-ipsum-ish word pool. Using real words rather than random letters makes
// the synthetic text exercise the same word-break logic the real pipeline
// would, and makes failures readable in the report.
const std::vector<std::u32string> words = {
    U"the", U"quick", U"brown", U"fox", U"jumps", U"over", U"lazy", U"dog",
    U"hello", U"world", U"PDF", U"editor", U"reflow", U"paragraph", U"line",
    U"text", U"font", U"glyph", U"unicode", U"baseline", U"column", U"margin",
    U"leading", U"justify", U"shaping", U"harfbuzz", U"freetype", U"icu",
    U"reconstruct", U"boundary", U"cluster", U"reading", U"order", U"document"
};

// Build a line of glyphs from a sequence of Unicode codepoints, starting at
// (x, y), advancing by `advance` per glyph. One GlyphRun per line is the
// simplest shape that exercises line detection (cluster by y) and paragraph
// grouping (consecutive lines).
GlyphRun make_line(const std::u32string& text, float x, float y,
                    float size, float advance, float confidence = 1.0f)
{
    GlyphRun run;
    run.size = size;
    run.rotation = 0.0f;
    run.glyphs.reserve(text.size());
    float px = x;
    for (char32_t cp : text) {
        Glyph g;
        g.unicode = static_cast<std::uint32_t>(cp);
        g.code = g.unicode; // synthetic: code == unicode
        g.gid = g.unicode;  // synthetic: gid == unicode
        g.confidence = confidence;
        g.origin = {px, y};
        g.advance = advance;
        run.glyphs.push_back(g);
        px += advance;
    }
    return run;
}

std::u32string make_text_line(float column_left, float column_right,
                               float advance, bool& reached_right)
{
    // Build a line of words until we reach or exceed the column right edge,
    // simulating wrapping. `reached_right` tells the caller whether the line
    // wrapped (true) or ended short (false), which is the paragraph-break signal.
    std::u32string line;
    float width = column_right - column_left;
    std::size_t glyphs_for_full = std::size_t(width / advance);
    std::size_t target = rand_size(std::size_t(glyphs_for_full * 0.85f),
                                    std::size_t(glyphs_for_full * 1.05f));
    std::size_t written = 0;
    while (written < target) {
        const auto& w = words[rand_size(0, words.size() - 1)];
        if (!line.empty()) { line.push_back(U' '); written += 1; }
        for (char32_t c : w) { line.push_back(c); ++written; }
        if (written >= target) break;
    }
    reached_right = written >= glyphs_for_full;
    return line;
}

SyntheticDoc make_doc(std::size_t idx)
{
    SyntheticDoc doc;
    doc.name = "synthetic-" + std::to_string(idx);

    const float column_left = 72.0f;
    const float column_right = 540.0f; // 612 - 72, US Letter margins
    const float advance = 6.0f;        // ~12pt font, 0.5 em advance
    const float size = 12.0f;
    const float leading = size * 1.2f; // 14.4pt
    const float para_gap = leading * 1.6f;

    float y = 720.0f; // start near the top of the page
    std::size_t n_paras = rand_size(3, 8);

    for (std::size_t p = 0; p < n_paras; ++p) {
        // Paragraph truth: record its vertical extent.
        GroundTruthParagraph gtp;
        gtp.left = column_left;
        gtp.right = column_right;
        gtp.top = y;

        std::size_t n_lines = rand_size(1, 5);
        bool is_list = (p > 0 && rand_size(0, 9) == 0); // ~10% of paras are list items
        std::u32string para_text;

        for (std::size_t l = 0; l < n_lines; ++l) {
            bool reached_right = false;
            std::u32string line_text = make_text_line(column_left, column_right,
                advance, reached_right);
            // The last line of a paragraph ends short (didn't reach the right).
            if (l == n_lines - 1) reached_right = false;
            GlyphRun run = make_line(line_text, column_left, y, size, advance);
            if (is_list && l == 0) {
                // Prepend a marker. In the real pipeline the marker is detected;
                // here we set it on the line so the pipeline can read it.
                run.glyphs.insert(run.glyphs.begin(), Glyph{});
                // We don't have a Line yet; the marker detection happens in
                // group_paragraphs via the Line.marker field, which detect_lines
                // doesn't populate. For the spike, list handling is exercised
                // by the paragraph-break logic (a list line is short).
            }
            doc.runs.push_back(std::move(run));
            if (!para_text.empty()) para_text.push_back(U'\n');
            para_text += line_text;
            y -= leading;
        }
        gtp.bottom = y + leading; // bottom of the last line's baseline
        gtp.text = para_text;
        doc.truth.push_back(gtp);
        y -= para_gap; // gap before the next paragraph
    }

    return doc;
}

} // namespace

std::vector<SyntheticDoc> generate_corpus(std::size_t n)
{
    std::vector<SyntheticDoc> out;
    out.reserve(n);
    for (std::size_t i = 0; i < n; ++i) out.push_back(make_doc(i));
    return out;
}

} // namespace freepdfeditor::spike::b