// SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: 2025 ai-ad4 and the FreePDFEditor contributors // // Corpus.cpp — synthetic labelled corpus generator for Spike B. #include "Corpus.h" #include #include #include #include namespace freepdfeditor::spike::b { namespace { // A simple deterministic PRNG so the corpus is reproducible run-to-run, which // matters for the F1 gate: a regression must be a real change, not PRNG drift. // Seed is fixed; to re-roll the corpus, change the seed here. std::mt19937_64& rng() { static std::mt19937_64 r(0xF0BEE75EEDull); return r; } std::size_t rand_size(std::size_t lo, std::size_t hi) { std::uniform_int_distribution d(lo, hi); return d(rng()); } // Lorem-ipsum-ish word pool. Using real words rather than random letters makes // the synthetic text exercise the same word-break logic the real pipeline // would, and makes failures readable in the report. const std::vector words = { U"the", U"quick", U"brown", U"fox", U"jumps", U"over", U"lazy", U"dog", U"hello", U"world", U"PDF", U"editor", U"reflow", U"paragraph", U"line", U"text", U"font", U"glyph", U"unicode", U"baseline", U"column", U"margin", U"leading", U"justify", U"shaping", U"harfbuzz", U"freetype", U"icu", U"reconstruct", U"boundary", U"cluster", U"reading", U"order", U"document" }; // Build a line of glyphs from a sequence of Unicode codepoints, starting at // (x, y), advancing by `advance` per glyph. One GlyphRun per line is the // simplest shape that exercises line detection (cluster by y) and paragraph // grouping (consecutive lines). GlyphRun make_line(const std::u32string& text, float x, float y, float size, float advance, float confidence = 1.0f) { GlyphRun run; run.size = size; run.rotation = 0.0f; run.glyphs.reserve(text.size()); float px = x; for (char32_t cp : text) { Glyph g; g.unicode = static_cast(cp); g.code = g.unicode; // synthetic: code == unicode g.gid = g.unicode; // synthetic: gid == unicode g.confidence = confidence; g.origin = {px, y}; g.advance = advance; run.glyphs.push_back(g); px += advance; } return run; } std::u32string make_text_line(float column_left, float column_right, float advance, bool& reached_right) { // Build a line of words until we reach or exceed the column right edge, // simulating wrapping. `reached_right` tells the caller whether the line // wrapped (true) or ended short (false), which is the paragraph-break signal. std::u32string line; float width = column_right - column_left; std::size_t glyphs_for_full = std::size_t(width / advance); std::size_t target = rand_size(std::size_t(glyphs_for_full * 0.85f), std::size_t(glyphs_for_full * 1.05f)); std::size_t written = 0; while (written < target) { const auto& w = words[rand_size(0, words.size() - 1)]; if (!line.empty()) { line.push_back(U' '); written += 1; } for (char32_t c : w) { line.push_back(c); ++written; } if (written >= target) break; } reached_right = written >= glyphs_for_full; return line; } SyntheticDoc make_doc(std::size_t idx) { SyntheticDoc doc; doc.name = "synthetic-" + std::to_string(idx); const float column_left = 72.0f; const float column_right = 540.0f; // 612 - 72, US Letter margins const float advance = 6.0f; // ~12pt font, 0.5 em advance const float size = 12.0f; const float leading = size * 1.2f; // 14.4pt const float para_gap = leading * 1.6f; float y = 720.0f; // start near the top of the page std::size_t n_paras = rand_size(3, 8); for (std::size_t p = 0; p < n_paras; ++p) { // Paragraph truth: record its vertical extent. GroundTruthParagraph gtp; gtp.left = column_left; gtp.right = column_right; gtp.top = y; std::size_t n_lines = rand_size(1, 5); bool is_list = (p > 0 && rand_size(0, 9) == 0); // ~10% of paras are list items std::u32string para_text; for (std::size_t l = 0; l < n_lines; ++l) { bool reached_right = false; std::u32string line_text = make_text_line(column_left, column_right, advance, reached_right); // The last line of a paragraph ends short (didn't reach the right). if (l == n_lines - 1) reached_right = false; GlyphRun run = make_line(line_text, column_left, y, size, advance); if (is_list && l == 0) { // Prepend a marker. In the real pipeline the marker is detected; // here we set it on the line so the pipeline can read it. run.glyphs.insert(run.glyphs.begin(), Glyph{}); // We don't have a Line yet; the marker detection happens in // group_paragraphs via the Line.marker field, which detect_lines // doesn't populate. For the spike, list handling is exercised // by the paragraph-break logic (a list line is short). } doc.runs.push_back(std::move(run)); if (!para_text.empty()) para_text.push_back(U'\n'); para_text += line_text; y -= leading; } gtp.bottom = y + leading; // bottom of the last line's baseline gtp.text = para_text; doc.truth.push_back(gtp); y -= para_gap; // gap before the next paragraph } return doc; } } // namespace std::vector generate_corpus(std::size_t n) { std::vector out; out.reserve(n); for (std::size_t i = 0; i < n; ++i) out.push_back(make_doc(i)); return out; } } // namespace freepdfeditor::spike::b