47 lines
2.0 KiB
C++
47 lines
2.0 KiB
C++
// SPDX-License-Identifier: GPL-3.0-or-later
|
||
// SPDX-FileCopyrightText: 2025 ai-ad4 and the FreePDFEditor contributors
|
||
//
|
||
// Corpus.h — synthetic labelled corpus generator for Spike B.
|
||
//
|
||
// The real labelled corpus (~2,000 pages with ground-truth paragraph/column
|
||
// segmentation, §8.1) is large and has licensing constraints; for the M0 spike
|
||
// we generate a controlled synthetic corpus: glyph runs whose text and
|
||
// paragraph boundaries are known by construction, laid out as a single LTR
|
||
// column with realistic leading, wrapping, and paragraph breaks. This lets us
|
||
// score the reconstruction pipeline's F1 on a corpus where the ground truth
|
||
// is exact, and where we can dial in difficulty (mixed leading, short last
|
||
// lines, list markers) to find where the 80% solution breaks.
|
||
//
|
||
// The synthetic generator is deliberately simple — it does not exercise the
|
||
// hard cases (multi-column, RTL, vertical CJK, rotated text). Those need the
|
||
// script corpus (§8.1) and are out of scope for the M0 spike's "does the
|
||
// pipeline shape work and score" question.
|
||
|
||
#ifndef FREEPDFEDITOR_SPIKE_B_CORPUS_H
|
||
#define FREEPDFEDITOR_SPIKE_B_CORPUS_H
|
||
|
||
#include "GlyphRun.h"
|
||
|
||
#include <vector>
|
||
|
||
namespace freepdfeditor::spike::b {
|
||
|
||
// One generated document: the glyph runs the pipeline sees, plus the
|
||
// ground-truth paragraph boundaries the scorer compares against.
|
||
struct SyntheticDoc {
|
||
std::vector<GlyphRun> runs;
|
||
std::vector<GroundTruthParagraph> truth;
|
||
std::string name;
|
||
};
|
||
|
||
// Generate `n` synthetic documents with varying difficulty. Each document has
|
||
// 3–8 paragraphs of 1–5 lines, with:
|
||
// - consistent leading within a paragraph (±10%)
|
||
// - a paragraph gap (extra leading) between paragraphs
|
||
// - last lines shorter than the column (paragraph break signal)
|
||
// - occasional list items (markers that should start a new paragraph)
|
||
std::vector<SyntheticDoc> generate_corpus(std::size_t n);
|
||
|
||
} // namespace freepdfeditor::spike::b
|
||
|
||
#endif // FREEPDFEDITOR_SPIKE_B_CORPUS_H
|