diff --git a/src/detector.rs b/src/detector.rs index 81aac146..fee0d0ff 100644 --- a/src/detector.rs +++ b/src/detector.rs @@ -1785,7 +1785,8 @@ pub(crate) fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u (has_images, total_area, has_template_image) } -/// Computes both `(needs_ocr_for_template_image, has_vector_text)` for a +/// Computes `(needs_ocr_for_template_image, has_undecodable_fonts, +/// has_vector_text)` for a /// page from a single shared `analyze_page_content` pass — that call /// decompresses and scans every content stream (page + XObjects) plus /// image coverage, so `extract_pages_markdown_mem` must not invoke it @@ -1827,10 +1828,21 @@ pub(crate) fn analyze_page_images(doc: &Document, page_id: ObjectId) -> (bool, u /// these pages to OCR, independent of any template-image check, since /// outlined glyphs can't be extracted as text at all. /// +/// `has_undecodable_fonts` is true when the fonts actually used for text +/// operators cannot be mapped back to Unicode — Identity-H/V without a +/// ToUnicode CMap, or a page whose used fonts are all custom-encoded Type3 +/// without ToUnicode. This is the Phase-3 signal `detect_from_document` +/// flags as `suspected_garbled_text` regardless of whole-document type. +/// A custom Type3 Encoding can map every byte to a plausible ASCII letter +/// (e.g. a Caesar-shifted alphabet), so the extracted text passes the +/// cipher/CID/encoding heuristics `extract_pages_markdown_mem` applies to +/// its own output — the font-resource signal is the only detector that +/// cannot be spoofed by what the bytes happen to look like. +/// /// Exposed at crate visibility so `extract_pages_markdown_mem` can apply /// the same gates classification needs elsewhere instead of treating the /// raw signals alone as sufficient — see #227/#231. -pub(crate) fn page_ocr_signals(doc: &Document, page_id: ObjectId) -> (bool, bool) { +pub(crate) fn page_ocr_signals(doc: &Document, page_id: ObjectId) -> (bool, bool, bool) { let analysis = analyze_page_content(doc, page_id); let needs_ocr_for_template_image = if !analysis.has_template_image { @@ -1845,7 +1857,11 @@ pub(crate) fn page_ocr_signals(doc: &Document, page_id: ObjectId) -> (bool, bool looks_like_scan || insufficient_text }; - (needs_ocr_for_template_image, analysis.has_vector_text) + ( + needs_ocr_for_template_image, + analysis.has_identity_h_no_tounicode || analysis.has_only_type3_fonts, + analysis.has_vector_text, + ) } /// Recursively collect image dimensions from XObject resources, @@ -3100,7 +3116,7 @@ mod tests { analysis.unique_alphanum_chars >= 10, "sanity: masthead text is diverse, alphanum_low cannot fire" ); - let (needs_ocr, _) = page_ocr_signals(&doc, page_id); + let (needs_ocr, _, _) = page_ocr_signals(&doc, page_id); assert!( needs_ocr, "template image + text below the pages_with_text floor is a scan" @@ -3126,7 +3142,7 @@ mod tests { analysis.text_operator_count >= 10, "sanity: body text clears the floor" ); - let (needs_ocr, _) = page_ocr_signals(&doc, page_id); + let (needs_ocr, _, _) = page_ocr_signals(&doc, page_id); assert!( !needs_ocr, "a text page with a background image must stay native" diff --git a/src/lib.rs b/src/lib.rs index 2b0ecce2..bf2001e7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -607,12 +607,18 @@ fn extract_pages_markdown_mem_impl( // embedded-font body text elsewhere would otherwise still extract // non-empty, non-garbled markdown and miss OCR routing entirely. // detect_from_document's Mixed-type per-page routing always sends - // these pages to OCR; mirror that here too. Both signals share one - // analyze_page_content pass — see page_ocr_signals's doc comment. - let (has_template_image, has_vector_text) = lopdf_pages + // these pages to OCR; mirror that here too. Finally, a custom Type3 + // Encoding can map every byte to a plausible ASCII letter while the + // glyph programs render entirely different text, so the markdown + // passes every text-quality heuristic — mirror detection's + // font-resource gate (Identity-H/Type3 without ToUnicode, its + // Phase-3 `suspected_garbled_text` signal) here too. See #428. + // All signals share one analyze_page_content pass — see + // page_ocr_signals's doc comment. + let (has_template_image, has_undecodable_fonts, has_vector_text) = lopdf_pages .get(&page_1idx) .map(|&page_id| detector::page_ocr_signals(&doc, page_id)) - .unwrap_or((false, false)); + .unwrap_or((false, false, false)); // Build markdown with document-wide font stats let options = MarkdownOptions { @@ -651,6 +657,13 @@ fn extract_pages_markdown_mem_impl( OCR_REASON_SUSPECTED_GARBLED_TEXT, ); } + if has_undecodable_fonts { + add_ocr_reason( + &mut ocr_reasons_by_page, + page_1idx, + OCR_REASON_SUSPECTED_GARBLED_TEXT, + ); + } if has_template_image { add_ocr_reason(&mut ocr_reasons_by_page, page_1idx, OCR_REASON_SCANNED); } diff --git a/tests/fixtures/type3_custom_encoding_no_tounicode.pdf b/tests/fixtures/type3_custom_encoding_no_tounicode.pdf new file mode 100644 index 00000000..557b8ef0 Binary files /dev/null and b/tests/fixtures/type3_custom_encoding_no_tounicode.pdf differ diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 1491330b..b2221804 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -4399,3 +4399,51 @@ fn test_extract_pages_markdown_agrees_with_classify_on_scan_with_native_header() page.markdown ); } + +/// Regression for #428: `extract_pages_markdown`'s per-page `needs_ocr` +/// must also agree with `classify_pdf`/`detect_pdf_type` when the +/// undecodable-font signal — not an image — is what makes the detector +/// distrust the page. The fixture draws all page text through a Form XObject +/// using a custom-encoded Type3 font with no ToUnicode map: the visible +/// glyph programs render ordinary business text, while a native extractor +/// that trusts the Encoding names reads a Caesar-shifted string. That output +/// is clean ASCII with plausible word shapes, so the text-quality heuristics +/// (cipher statistics, CID garbage, encoding issues) all pass on this short +/// sample, while detection's font-resource check correctly flags the page as +/// `suspected_garbled_text`. The extraction API must apply the same font +/// signal instead of returning the wrong native text as trusted Markdown. +#[test] +fn test_extract_pages_markdown_agrees_with_classify_on_type3_custom_encoding() { + let buf = std::fs::read("tests/fixtures/type3_custom_encoding_no_tounicode.pdf").unwrap(); + + let cls = pdf_inspector::detector::detect_pdf_type_mem(&buf).expect("fixture should classify"); + assert!( + cls.pages_needing_ocr.contains(&1), + "classify_pdf should flag page 1 as needing OCR (Type3 without ToUnicode), got: {:?}", + cls.pages_needing_ocr + ); + + let ext = extract_pages_markdown_mem(&buf, None).expect("fixture should extract"); + let page = &ext.pages[0]; + assert!( + page.needs_ocr, + "extract_pages_markdown must agree with classify_pdf that this page needs OCR" + ); + assert!( + ext.pages_needing_ocr.contains(&1), + "the document-level OCR page list must include the flagged page, got: {:?}", + ext.pages_needing_ocr + ); + assert_eq!( + page.ocr_reason.as_deref(), + Some(pdf_inspector::OCR_REASON_SUSPECTED_GARBLED_TEXT), + "the detector-confirmed garble signal must be reported, got: {:?}", + page.ocr_reason + ); + assert!( + page.markdown.is_empty(), + "a page flagged needs_ocr must not return the Type3 Encoding-name text \ + as if extraction were trustworthy, got: {:?}", + page.markdown + ); +}