freepdfeditor/spike/B1_glyph_unicode/GlyphUnicode.cpp

// SPDX-License-Identifier: GPL-3.0-or-later
// SPDX-FileCopyrightText: 2025 ai-ad4 and the FreePDFEditor contributors
//
// GlyphUnicode.cpp — Spike B step 1 implementation. Walks a PDF's fonts via
// QPDF, finds one with a ToUnicode CMap and an embedded font stream, parses
// the ToUnicode, and falls back to the embedded font's cmap via FreeType.
//
// The ToUnicode CMap format (PDF spec §7.9.2) is a small declarative language:
//   /CIDInit /ProcSet findresource begin
//   12 dict begin begincmap
//   /CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
//   /CMapName /Adobe-Identity-UCS def
//   /CMapType 2 def
//   1 begincodespacerange <00> <FF>
//   1 beginbfchar <41> <0041>
//   endbfchar
//   endcmap CMapName currentdict /CMap defineresource pop end end
// We parse just the beginbfchar/begincodespacerange sections to build a
// code→Unicode map. The production parser (M4) handles the full grammar
// (bfrange, CID-keyed fonts, surrogate pairs); this spike covers the common
// bfchar case which is what most simple-font ToUnicode CMaps use.

#include "GlyphUnicode.h"

#include <qpdf/QPDF.hh>
#include <qpdf/QPDFObjectHandle.hh>
#include <qpdf/QPDFPageDocumentHelper.hh>

#include <ft2build.h>
#include FT_FREETYPE_H
#include FT_FONT_FORMATS_H

#include <cstdint>
#include <cstdio>
#include <cstring>
#include <fstream>
#include <sstream>
#include <string>
#include <unordered_map>
#include <vector>

namespace freepdfeditor::spike::b1 {

namespace {

// Parse a hex string like "41" or "0041" into a uint32. Returns false on
// malformed input.
bool parse_hex(const std::string& s, std::uint32_t& out)
{
    out = 0;
    if (s.empty()) return false;
    for (char c : s) {
        out <<= 4;
        if (c >= '0' && c <= '9') out |= std::uint32_t(c - '0');
        else if (c >= 'a' && c <= 'f') out |= std::uint32_t(c - 'a' + 10);
        else if (c >= 'A' && c <= 'F') out |= std::uint32_t(c - 'A' + 10);
        else return false;
    }
    return true;
}

// Extract the raw bytes of a PDF stream object (the decoded stream data).
std::string stream_bytes(QPDFObjectHandle stream)
{
    if (!stream.isStream()) return {};
    auto buf = stream.getStreamData();
    if (!buf) return {};
    return std::string(reinterpret_cast<const char*>(buf->getBuffer()),
                       buf->getSize());
}

// Parse the bfchar/bfrange sections of a ToUnicode CMap into a code→Unicode
// map. Returns the number of mappings found.
std::size_t parse_tounicode(const std::string& cmap_data,
                             std::unordered_map<std::uint32_t, std::uint32_t>& out)
{
    std::istringstream in(cmap_data);
    std::string tok;
    std::size_t found = 0;
    while (in >> tok) {
        if (tok == "beginbfchar") {
            // Read count then count pairs of <code> <unicode>.
            // We already consumed the count token before beginbfchar in real
            // PDFs, but the count comes BEFORE beginbfchar. Re-read: the
            // grammar is "N beginbfchar ... endbfchar" where N is the count.
            // We hit beginbfchar without the count, so the count was the
            // previous token. Simplest: read pairs until endbfchar.
            while (in >> tok && tok != "endbfchar") {
                // tok is <code>; next is <unicode>.
                if (tok.empty() || tok[0] != '<') { continue; }
                std::string code_hex = tok.substr(1, tok.find('>') - 1);
                std::string uni_hex;
                if (!(in >> uni_hex)) break;
                if (uni_hex.empty() || uni_hex[0] != '<') continue;
                uni_hex = uni_hex.substr(1, uni_hex.find('>') - 1);
                std::uint32_t code = 0, uni = 0;
                if (parse_hex(code_hex, code) && parse_hex(uni_hex, uni)) {
                    out[code] = uni;
                    ++found;
                }
            }
        }
        // bfrange parsing is omitted (M4 work); bfchar covers the common case.
    }
    return found;
}

// Read the embedded font stream (FontFile2 for TrueType) into bytes.
std::string embedded_font_bytes(QPDFObjectHandle font_obj)
{
    QPDFObjectHandle desc = font_obj.getKey("/FontDescriptor");
    if (!desc.isDictionary()) return {};
    for (const char* key : {"/FontFile2", "/FontFile3", "/FontFile"}) {
        QPDFObjectHandle ff = desc.getKey(key);
        if (ff.isStream()) return stream_bytes(ff);
    }
    return {};
}

} // namespace

MappingResult map_glyphs(const std::string& pdf_path)
{
    MappingResult r{};

    QPDF q;
    try {
        const std::string path = pdf_path;
        q.processFile(path.c_str());
    } catch (const std::exception& e) {
        r.error = std::string("QPDF open failed: ") + e.what();
        return r;
    }

    QPDFPageDocumentHelper helper(q);
    auto pages = helper.getAllPages();

    // Walk all pages' font resources to find a font with both a ToUnicode CMap
    // and an embedded font stream — the case that exercises both ladder steps.
    QPDFObjectHandle target_font;
    QPDFObjectHandle target_tounicode;
    std::string embedded_font;
    for (auto& page : pages) {
        QPDFObjectHandle resources = page.getAttribute("/Resources", true);
        if (!resources.isDictionary()) continue;
        QPDFObjectHandle fonts = resources.getKey("/Font");
        if (!fonts.isDictionary()) continue;
        for (auto& kv : fonts.getDictAsMap()) {
            QPDFObjectHandle font = kv.second;
            if (!font.isDictionary()) continue;
            QPDFObjectHandle tn = font.getKey("/ToUnicode");
            std::string font_bytes = embedded_font_bytes(font);
            if (tn.isStream() && !font_bytes.empty()) {
                target_font = font;
                target_tounicode = tn;
                embedded_font = std::move(font_bytes);
                break;
            }
        }
        if (target_font.isDictionary()) break;
    }

    if (!target_font.isDictionary()) {
        r.error = "no font with both ToUnicode and embedded stream found";
        return r;
    }

    // Step 1: parse the ToUnicode CMap.
    std::string tounicode_data = stream_bytes(target_tounicode);
    std::unordered_map<std::uint32_t, std::uint32_t> code_to_uni;
    std::size_t tn_count = parse_tounicode(tounicode_data, code_to_uni);
    r.mapped_via_tounicode = tn_count;

    // Step 4: for glyphs the ToUnicode didn't cover, fall back to the embedded
    // font's cmap via FreeType. Build a gid→unicode map from FreeType's cmap.
    std::unordered_map<std::uint32_t, std::uint32_t> cmap_gid_to_uni;
    if (!embedded_font.empty()) {
        FT_Library lib = nullptr;
        if (FT_Init_FreeType(&lib) == 0) {
            FT_Face face = nullptr;
            if (FT_New_Memory_Face(lib,
                    reinterpret_cast<const FT_Byte*>(embedded_font.data()),
                    static_cast<FT_Long>(embedded_font.size()), 0, &face) == 0) {
                // Walk the font's cmap: for each codepoint FreeType can map to
                // a glyph, record gid→codepoint (reverse of the usual lookup,
                // which is what the §4.1 step 4 "reverse lookup" describes).
                // FT_Get_First_Char returns the charcode and writes the gid.
                FT_UInt gid = 0;
                FT_ULong charcode = FT_Get_First_Char(face, &gid);
                while (gid != 0 && charcode != 0) {
                    cmap_gid_to_uni[gid] = static_cast<std::uint32_t>(charcode);
                    charcode = FT_Get_Next_Char(face, charcode, &gid);
                }
                FT_Done_Face(face);
            }
            FT_Done_FreeType(lib);
        }
    }

    // Assemble the per-glyph mapping. The glyph count comes from FreeType;
    // the ToUnicode map is keyed by character code, which for simple fonts is
    // often the gid (Identity encoding) — we record both and prefer ToUnicode.
    std::size_t total_glyphs = 0;
    if (!embedded_font.empty()) {
        FT_Library lib = nullptr;
        if (FT_Init_FreeType(&lib) == 0) {
            FT_Face face = nullptr;
            if (FT_New_Memory_Face(lib,
                    reinterpret_cast<const FT_Byte*>(embedded_font.data()),
                    static_cast<FT_Long>(embedded_font.size()), 0, &face) == 0) {
                total_glyphs = static_cast<std::size_t>(face->num_glyphs);
                FT_Done_Face(face);
            }
            FT_Done_FreeType(lib);
        }
    }

    std::size_t via_cmap = 0, unmapped = 0;
    double conf_sum = 0.0;
    for (std::uint32_t gid = 0; gid < total_glyphs; ++gid) {
        GlyphMap m;
        m.gid = gid;
        // Step 1: ToUnicode (keyed by code; for simple fonts code == gid here).
        auto it = code_to_uni.find(gid);
        if (it != code_to_uni.end()) {
            m.unicode = it->second;
            m.confidence = 1.0f;
            m.source = GlyphMap::Source::ToUnicode;
        } else {
            // Step 4: reverse lookup through the cmap.
            auto cit = cmap_gid_to_uni.find(gid);
            if (cit != cmap_gid_to_uni.end()) {
                m.unicode = cit->second;
                m.confidence = 0.7f;
                m.source = GlyphMap::Source::Cmap;
                ++via_cmap;
            } else {
                m.unicode = 0;
                m.confidence = 0.0f;
                m.source = GlyphMap::Source::Unknown;
                ++unmapped;
            }
        }
        if (m.unicode) conf_sum += m.confidence;
        r.mappings.push_back(m);
    }
    r.mapped_via_cmap = via_cmap;
    r.unmapped = unmapped;
    r.coverage = total_glyphs ? double(total_glyphs - unmapped) / double(total_glyphs) : 0.0;
    r.avg_confidence = (total_glyphs - unmapped) ? conf_sum / double(total_glyphs - unmapped) : 0.0;
    r.ok = total_glyphs > 0;
    return r;
}

} // namespace freepdfeditor::spike::b1