private static final ThreadLocal<Map<Integer, Double>> replacementCharRatios = ThreadLocal.withInitial(HashMap::new);

import java.util.HashMap;
import java.util.Map;

public static void setReplacementCharRatio(int pageNumber, double ratio) {
    replacementCharRatios.get().put(pageNumber, ratio);
}

public static double getReplacementCharRatio(int pageNumber) {
    return replacementCharRatios.get().getOrDefault(pageNumber, 0.0);
}

replacementCharRatios.get().clear();

git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/containers/StaticLayoutContainers.java
git commit -m "feat: add per-page replacement char ratio storage to StaticLayoutContainers"

@Test
public void testMeasureReplacementCharRatio_allReplacement() {
    List<IObject> contents = new ArrayList<>();
    contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0),
        "\uFFFD\uFFFD\uFFFD", 10, 10.0));

    double ratio = TextProcessor.measureReplacementCharRatio(contents);
    Assertions.assertEquals(1.0, ratio, 0.001);
}

@Test
public void testMeasureReplacementCharRatio_noReplacement() {
    List<IObject> contents = new ArrayList<>();
    contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0),
        "Hello World", 10, 10.0));

    double ratio = TextProcessor.measureReplacementCharRatio(contents);
    Assertions.assertEquals(0.0, ratio, 0.001);
}

@Test
public void testMeasureReplacementCharRatio_mixed() {
    List<IObject> contents = new ArrayList<>();
    // 3 replacement chars out of 10 total = 0.3
    contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0),
        "\uFFFD\uFFFD\uFFFDAbcdefg", 10, 10.0));

    double ratio = TextProcessor.measureReplacementCharRatio(contents);
    Assertions.assertEquals(0.3, ratio, 0.001);
}

@Test
public void testMeasureReplacementCharRatio_emptyContents() {
    List<IObject> contents = new ArrayList<>();

    double ratio = TextProcessor.measureReplacementCharRatio(contents);
    Assertions.assertEquals(0.0, ratio, 0.001);
}

@Test
public void testMeasureReplacementCharRatio_nonTextChunksIgnored() {
    List<IObject> contents = new ArrayList<>();
    contents.add(new ImageChunk(new BoundingBox(1, 10.0, 10.0, 100.0, 20.0)));
    contents.add(new TextChunk(new BoundingBox(1, 10.0, 30.0, 100.0, 40.0),
        "\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD", 10, 10.0));

    double ratio = TextProcessor.measureReplacementCharRatio(contents);
    // Only TextChunks counted: 5/5 = 1.0
    Assertions.assertEquals(1.0, ratio, 0.001);
}

public static double measureReplacementCharRatio(List<IObject> contents) {
    char replacementChar = ChunkParser.REPLACEMENT_CHARACTER_STRING.charAt(0);
    int totalChars = 0;
    int replacementChars = 0;
    for (IObject object : contents) {
        if (object instanceof TextChunk) {
            String value = ((TextChunk) object).getValue();
            totalChars += value.length();
            for (int i = 0; i < value.length(); i++) {
                if (value.charAt(i) == replacementChar) {
                    replacementChars++;
                }
            }
        }
    }
    if (totalChars == 0) {
        return 0.0;
    }
    return (double) replacementChars / totalChars;
}

git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TextProcessor.java
git add java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/processors/TextProcessorTest.java
git commit -m "feat: add measureReplacementCharRatio to TextProcessor

Counts U+FFFD replacement characters across TextChunks and returns
the ratio. Returns 0.0 for empty contents or pages with no text."

import org.opendataloader.pdf.containers.StaticLayoutContainers;

double replacementCharRatio = TextProcessor.measureReplacementCharRatio(pageContents);
StaticLayoutContainers.setReplacementCharRatio(pageNumber, replacementCharRatio);
if (replacementCharRatio >= 0.3) {
    LOGGER.log(Level.WARNING,
        "Page {0}: {1,number,#.#%} of characters are replacement characters (U+FFFD). "
        + "This PDF likely contains CID-keyed fonts without ToUnicode mappings. "
        + "Text extraction may be incomplete. Consider using --hybrid-mode for OCR fallback.",
        new Object[]{pageNumber + 1, replacementCharRatio});
}

git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ContentFilterProcessor.java
git commit -m "feat: detect CID font extraction failure and emit warning log

Measures U+FFFD ratio before replacement. Warns when >= 30% of
characters are replacement characters, suggesting hybrid mode."

@Test
public void testClassifyPage_highReplacementRatio_routesToBackend() {
    StaticLayoutContainers.clearContainers();
    StaticLayoutContainers.setReplacementCharRatio(0, 0.5);

    List<IObject> contents = new ArrayList<>();
    contents.add(createTextChunk(10, 100, 200, 120, "text"));

    TriageResult result = TriageProcessor.classifyPage(contents, 0, new HybridConfig());

    Assertions.assertEquals(TriageDecision.BACKEND, result.getDecision());
    Assertions.assertEquals(1.0, result.getConfidence(), 0.001);
}

@Test
public void testClassifyPage_lowReplacementRatio_noEffect() {
    StaticLayoutContainers.clearContainers();
    StaticLayoutContainers.setReplacementCharRatio(0, 0.1);

    List<IObject> contents = new ArrayList<>();
    contents.add(createTextChunk(10, 100, 200, 120, "normal text"));

    TriageResult result = TriageProcessor.classifyPage(contents, 0, new HybridConfig());

    Assertions.assertEquals(TriageDecision.JAVA, result.getDecision());
}

@Test
public void testClassifyPage_exactThreshold_routesToBackend() {
    StaticLayoutContainers.clearContainers();
    StaticLayoutContainers.setReplacementCharRatio(0, 0.3);

    List<IObject> contents = new ArrayList<>();
    contents.add(createTextChunk(10, 100, 200, 120, "text"));

    TriageResult result = TriageProcessor.classifyPage(contents, 0, new HybridConfig());

    Assertions.assertEquals(TriageDecision.BACKEND, result.getDecision());
    Assertions.assertEquals(1.0, result.getConfidence(), 0.001);
}

// Signal 0: CID font extraction failure (highest priority)
// Only fires in hybrid mode (classifyPage is only called from HybridDocumentProcessor)
double replacementRatio = StaticLayoutContainers.getReplacementCharRatio(pageNumber);
if (replacementRatio >= 0.3) {
    return TriageResult.backend(pageNumber, 1.0, signals);
}

import org.opendataloader.pdf.containers.StaticLayoutContainers;

 * <p>Signal priority (highest to lowest):
 * <ol>
 *   <li>CID font extraction failure (replacement char ratio >= 30%)</li>
 *   <li>TableBorder presence</li>
 *   ...

git add java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/TriageProcessor.java
git add java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/hybrid/TriageProcessorTest.java
git commit -m "feat: add CID font detection signal to TriageProcessor

Signal 0 (highest priority): routes pages with >= 30% replacement
characters to backend for OCR fallback in hybrid mode."

mkdir -p java/opendataloader-pdf-core/src/test/resources

git add java/opendataloader-pdf-core/src/test/resources/cid-font-no-tounicode.pdf
git add java/opendataloader-pdf-core/src/test/resources/generate-cid-test-pdf.py
git commit -m "test: add synthetic CID font PDF test fixture

PDF with CID-keyed font without ToUnicode mapping for testing
replacement character detection. Generation script included."

/*
 * Copyright 2025-2026 Hancom Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.opendataloader.pdf.processors;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;

/**
 * Integration test for CID font extraction failure detection.
 *
 * Tests the full pipeline: PDF parsing → ContentFilterProcessor →
 * measurement → StaticLayoutContainers storage → warning log.
 */
public class CidFontDetectionTest {

    private static final Path CID_PDF_PATH = Paths.get(
        "src/test/resources/cid-font-no-tounicode.pdf");

    private static boolean pdfAvailable = false;

    @BeforeAll
    static void checkFixture() {
        pdfAvailable = Files.exists(CID_PDF_PATH) && Files.isRegularFile(CID_PDF_PATH);
        if (!pdfAvailable) {
            System.out.println("CID font test PDF not found: " + CID_PDF_PATH.toAbsolutePath());
            System.out.println("Skipping integration tests. Generate fixture first.");
        }
    }

    @Test
    public void testCidPdf_highReplacementRatio_detected() throws IOException {
        Assumptions.assumeTrue(pdfAvailable, "CID font test PDF not available");

        String pdfPath = CID_PDF_PATH.toAbsolutePath().toString();
        Config config = new Config();

        DocumentProcessor.preprocessing(pdfPath, config);
        StaticLayoutContainers.clearContainers();

        int numPages = StaticContainers.getDocument().getNumberOfPages();
        Assertions.assertTrue(numPages > 0, "PDF should have at least 1 page");

        // Process page 0 through ContentFilterProcessor
        List<IObject> filteredContents = ContentFilterProcessor.getFilteredContents(
            pdfPath,
            StaticContainers.getDocument().getArtifacts(0),
            0,
            config
        );

        // Verify ratio was stored
        double ratio = StaticLayoutContainers.getReplacementCharRatio(0);
        Assertions.assertTrue(ratio >= 0.3,
            "CID font PDF should have >= 30% replacement characters, got "
            + String.format("%.1f%%", ratio * 100));
    }

    @Test
    public void testCidPdf_warningLogEmitted() throws IOException {
        Assumptions.assumeTrue(pdfAvailable, "CID font test PDF not available");

        // Capture warning logs
        Logger logger = Logger.getLogger(ContentFilterProcessor.class.getCanonicalName());
        List<String> warnings = new ArrayList<>();
        Handler handler = new Handler() {
            @Override public void publish(LogRecord r) {
                if (r.getLevel() == Level.WARNING) {
                    warnings.add(r.getMessage());
                }
            }
            @Override public void flush() {}
            @Override public void close() {}
        };
        logger.addHandler(handler);

        try {
            String pdfPath = CID_PDF_PATH.toAbsolutePath().toString();
            Config config = new Config();

            DocumentProcessor.preprocessing(pdfPath, config);
            StaticLayoutContainers.clearContainers();

            ContentFilterProcessor.getFilteredContents(
                pdfPath,
                StaticContainers.getDocument().getArtifacts(0),
                0,
                config
            );

            boolean hasReplacementWarning = warnings.stream()
                .anyMatch(w -> w.contains("replacement characters"));
            Assertions.assertTrue(hasReplacementWarning,
                "Expected WARNING log about replacement characters");
        } finally {
            logger.removeHandler(handler);
        }
    }

    /**
     * Unit-level boundary tests (no PDF fixture needed).
     */
    @Test
    public void testBoundary_belowThreshold_29percent() {
        // 29 replacement chars out of 100 = 0.29 (below threshold)
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < 29; i++) sb.append('\uFFFD');
        for (int i = 0; i < 71; i++) sb.append('A');

        List<IObject> contents = new ArrayList<>();
        contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 500.0, 20.0),
            sb.toString(), 10, 10.0));

        double ratio = TextProcessor.measureReplacementCharRatio(contents);
        Assertions.assertTrue(ratio < 0.3,
            "29% should be below threshold, got " + ratio);
    }

    @Test
    public void testBoundary_atThreshold_30percent() {
        // 30 replacement chars out of 100 = 0.30 (at threshold)
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < 30; i++) sb.append('\uFFFD');
        for (int i = 0; i < 70; i++) sb.append('A');

        List<IObject> contents = new ArrayList<>();
        contents.add(new TextChunk(new BoundingBox(1, 10.0, 10.0, 500.0, 20.0),
            sb.toString(), 10, 10.0));

        double ratio = TextProcessor.measureReplacementCharRatio(contents);
        Assertions.assertTrue(ratio >= 0.3,
            "30% should be at threshold, got " + ratio);
    }
}

git add java/opendataloader-pdf-core/src/test/java/org/opendataloader/pdf/processors/CidFontDetectionTest.java
git commit -m "test: add integration + boundary tests for CID font detection

Integration tests load cid-font-no-tounicode.pdf through full pipeline.
Boundary tests verify 29%/30% threshold behavior."

gh issue edit 286 --repo opendataloader-project/opendataloader-pdf --remove-label enhancement