diff --git a/Cargo.lock b/Cargo.lock index cc49f60..0aacdca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,16 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "atoi_simd" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" +dependencies = [ + "debug_unsafe", + "rustversion", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -226,6 +236,23 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "calamine" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +dependencies = [ + "atoi_simd", + "byteorder", + "codepage", + "encoding_rs", + "fast-float2", + "log", + "quick-xml 0.41.0", + "serde", + "zip 8.6.0", +] + [[package]] name = "cbc" version = "0.1.2" @@ -280,6 +307,15 @@ dependencies = [ "inout", ] +[[package]] +name = "codepage" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdff162541cd8b79de82e2edcc7eff3a8c2a6dc3d75152636028f96d93de3b26" +dependencies = [ + "encoding_rs", +] + [[package]] name = "color" version = "0.3.3" @@ -371,6 +407,12 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "debug_unsafe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" + [[package]] name = "deranged" version = "0.5.8" @@ -515,6 +557,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fast-float2" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e8948ce679d00a02a94739ea185595dca7118ed04feb991127e443bd3d761f" + [[package]] name = "fast_image_resize" version = "5.5.0" @@ -1909,6 +1957,7 @@ dependencies = [ name = "tinydocs" version = "0.1.22" dependencies = [ + "calamine", "docx-rs", "hayro", "pdf-extract", diff --git a/Cargo.toml b/Cargo.toml index 9d92355..fdc45cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,8 @@ pdf-extract = { version = "0.12", optional = true } zip = { version = "8", default-features = false, features = ["deflate"], optional = true } # Streaming XML events avoid building attacker-controlled DOM trees. quick-xml = { version = "0.41", optional = true } +# Workbook row conversion for document-memory Markdown; implementation only. +calamine = { version = "0.36", optional = true } # Pure Rust PDF rasterization, isolated from the default extraction build. hayro = { version = "0.5", optional = true } @@ -93,6 +95,8 @@ pptx = ["dep:ppt-rs"] pdf = ["dep:pdf-extract"] # Bounded DOCX/PPTX/XLSX and per-page PDF text intake. intake = ["pdf", "dep:zip", "dep:quick-xml"] +# Full normalized Markdown conversion compatible with document-memory ingestion. +markdown = ["intake", "dep:calamine"] # Selected PDF page rendering to PNG. pdf-render = ["pdf", "dep:hayro"] diff --git a/README.md b/README.md index 4ae6bd2..e00eafd 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ GenerateDocx(DocumentSpec) -> OutputRef GeneratePptx(deck, Option) -> OutputRef ExtractText(StreamRef) -> OutputRef ExtractDocument(ExtractDocumentSpec, StreamRef) -> ExtractedDocument +ConvertMarkdown(DocumentFormat, StreamRef) -> OutputRef RenderPdf(RenderPdfSpec, StreamRef) -> RenderedPdf ReadOutput(output_id, offset, len) -> base64 ReleaseOutput(output_id) -> () @@ -259,3 +260,12 @@ an older release can still serve the original five methods but will not have require a published module release for intake. Do not derive release checksums from a local build. `OutputRef` is now defined in `tinydocs-bus` and remains re-exported by `tinydocs-module` with the same wire fields. + +## Complete memory conversion + +Optional `markdown` converts PDF, DOCX, PPTX, and XLSX to complete normalized +Markdown, preserving TinyMemory OfficeConverter output semantics. The compiled +module exposes `ConvertMarkdown` in contract version 3; hosts must pin a +compatible published artifact before using it. See the [conversion spec](docs/specs/markdown-conversion.md) +and [parser limits](src/markdown/README.md). Read the output through `ReadOutput` +and explicitly release it with `ReleaseOutput`. diff --git a/crates/tinydocs-bus/README.md b/crates/tinydocs-bus/README.md index 81651cb..ec6ee5a 100644 --- a/crates/tinydocs-bus/README.md +++ b/crates/tinydocs-bus/README.md @@ -14,3 +14,6 @@ type, not compatible-looking duplicates. The module serves the `METHODS` at `BUS_NAME` and `OBJECT_PATH`. Keep changes here backward compatible or advance `CONTRACT_VERSION` according to the documented compatibility rule. + +Contract version 3 adds `ConvertMarkdown(DocumentFormat, StreamRef) -> OutputRef` +without changing existing member arities. See the [conversion spec](../../docs/specs/markdown-conversion.md). diff --git a/crates/tinydocs-bus/src/intake/mod_tests.rs b/crates/tinydocs-bus/src/intake/mod_tests.rs index a0bcde2..41d8c2c 100644 --- a/crates/tinydocs-bus/src/intake/mod_tests.rs +++ b/crates/tinydocs-bus/src/intake/mod_tests.rs @@ -69,3 +69,26 @@ fn rejects_unknown_wire_fields_and_formats() { .is_err() ); } + +#[test] +fn complete_conversion_preserves_format_vocabulary_and_requires_version_three() { + assert_eq!(crate::CONTRACT_VERSION, 3); + assert!(crate::is_compatible(3)); + assert!(!crate::is_compatible(2)); + assert!(crate::METHODS.contains(&crate::names::methods::CONVERT_MARKDOWN)); + for (format, wire) in [ + (DocumentFormat::Pdf, "pdf"), + (DocumentFormat::Docx, "docx"), + (DocumentFormat::Pptx, "pptx"), + (DocumentFormat::Xlsx, "xlsx"), + ] { + assert_eq!( + serde_json::to_value(format).unwrap(), + serde_json::json!(wire) + ); + assert_eq!( + serde_json::from_value::(serde_json::json!(wire)).unwrap(), + format + ); + } +} diff --git a/crates/tinydocs-bus/src/names.rs b/crates/tinydocs-bus/src/names.rs index 6596f62..0c058b5 100644 --- a/crates/tinydocs-bus/src/names.rs +++ b/crates/tinydocs-bus/src/names.rs @@ -16,6 +16,8 @@ pub mod methods { pub const EXTRACT_TEXT: &str = "ExtractText"; /// `ExtractDocument` — bounded document text and provenance from a stream. pub const EXTRACT_DOCUMENT: &str = "ExtractDocument"; + /// `ConvertMarkdown` — convert a streamed Office/PDF document to full Markdown. + pub const CONVERT_MARKDOWN: &str = "ConvertMarkdown"; /// `RenderPdf` — selected PDF pages as held PNG outputs. pub const RENDER_PDF: &str = "RenderPdf"; /// `ReadOutput` — read a bounded base64-encoded output chunk. @@ -25,11 +27,12 @@ pub mod methods { } /// All method names in the declaration order used by the module interface. -pub const METHODS: [&str; 7] = [ +pub const METHODS: [&str; 8] = [ methods::GENERATE_DOCX, methods::GENERATE_PPTX, methods::EXTRACT_TEXT, methods::EXTRACT_DOCUMENT, + methods::CONVERT_MARKDOWN, methods::RENDER_PDF, methods::READ_OUTPUT, methods::RELEASE_OUTPUT, diff --git a/crates/tinydocs-bus/src/version.rs b/crates/tinydocs-bus/src/version.rs index bb2ae7e..82433b3 100644 --- a/crates/tinydocs-bus/src/version.rs +++ b/crates/tinydocs-bus/src/version.rs @@ -2,9 +2,10 @@ /// Current `TinyDocs` wire-contract version. /// -/// Intake methods are additive to version 2. Older version-2 modules may not -/// serve them; hosts must check method availability or require a newer release. -pub const CONTRACT_VERSION: u32 = 2; +/// Version 3 adds full Markdown conversion. Older modules retain their existing +/// member signatures but do not serve `ConvertMarkdown`; hosts requiring it +/// must pin a compatible released artifact. +pub const CONTRACT_VERSION: u32 = 3; /// Returns whether a host requiring `required` can bind to this contract. /// diff --git a/crates/tinydocs-module/Cargo.toml b/crates/tinydocs-module/Cargo.toml index 85a5a9b..dd129db 100644 --- a/crates/tinydocs-module/Cargo.toml +++ b/crates/tinydocs-module/Cargo.toml @@ -17,7 +17,7 @@ crate-type = ["rlib", "cdylib"] [dependencies] # The pure document library remains independently publishable and bus-agnostic. -tinydocs = { path = "../..", default-features = false, features = ["docx", "pptx", "pdf", "intake", "pdf-render"] } +tinydocs = { path = "../..", default-features = false, features = ["docx", "pptx", "pdf", "intake", "pdf-render", "markdown"] } # The transport-free vocabulary shared with every TinyBus host. tinydocs-bus = { path = "../tinydocs-bus" } # TinyBus provides the typed service interface and dynamic module host ABI. diff --git a/crates/tinydocs-module/src/service/mod.rs b/crates/tinydocs-module/src/service/mod.rs index 76a2135..4d7c7bf 100644 --- a/crates/tinydocs-module/src/service/mod.rs +++ b/crates/tinydocs-module/src/service/mod.rs @@ -1,12 +1,13 @@ //! `TinyBus` service boundary for the document surface. //! -//! One object, `/ai/tinyhumans/tinydocs/Documents`, exporting seven methods: +//! One object, `/ai/tinyhumans/tinydocs/Documents`, exporting eight methods: //! //! ```text //! GenerateDocx(DocumentSpec) -> OutputRef //! GeneratePptx(WirePresentationSpec, Option) -> OutputRef //! ExtractText(StreamRef) -> OutputRef //! ExtractDocument(spec, StreamRef) -> ExtractedDocument +//! ConvertMarkdown(format, StreamRef) -> OutputRef //! RenderPdf(spec, StreamRef) -> RenderedPdf //! ReadOutput(output_id, offset, len) -> base64 //! ReleaseOutput(output_id) -> () @@ -133,6 +134,17 @@ impl Documents { blocking(move || tinydocs::intake::extract(&bytes, &spec)).await } + /// Convert a streamed document to the full normalized Markdown used by memory. + async fn convert_markdown( + &self, + format: tinydocs_bus::DocumentFormat, + document: StreamRef, + ) -> BusResult { + let bytes = self.read_stream(&document).await?; + let markdown = blocking(move || tinydocs::markdown::convert(&bytes, format)).await?; + self.hold(markdown.into_bytes()) + } + /// Render explicitly selected PDF pages and hold each PNG output. async fn render_pdf(&self, spec: RenderPdfSpec, document: StreamRef) -> BusResult { let bytes = self.read_stream(&document).await?; @@ -388,6 +400,7 @@ pub(crate) mod exports { "GeneratePptx", "ExtractText", "ExtractDocument", + "ConvertMarkdown", "RenderPdf", "ReadOutput", "ReleaseOutput", diff --git a/crates/tinydocs-module/src/service/mod_tests.rs b/crates/tinydocs-module/src/service/mod_tests.rs index 7c92091..6e23412 100644 --- a/crates/tinydocs-module/src/service/mod_tests.rs +++ b/crates/tinydocs-module/src/service/mod_tests.rs @@ -23,6 +23,7 @@ const DECLARED_METHODS: &[&str] = &[ "GeneratePptx", "ExtractText", "ExtractDocument", + "ConvertMarkdown", "RenderPdf", "ReadOutput", "ReleaseOutput", diff --git a/crates/tinydocs-module/tests/module_e2e.rs b/crates/tinydocs-module/tests/module_e2e.rs index 7b75bc4..27bdef5 100644 --- a/crates/tinydocs-module/tests/module_e2e.rs +++ b/crates/tinydocs-module/tests/module_e2e.rs @@ -49,6 +49,7 @@ async fn the_built_module_serves_every_format_over_a_real_broker() { generates_a_pptx_from_a_streamed_image_pair(&client, &target, &proxy).await; extracts_text_from_a_streamed_pdf(&client, &target, &proxy).await; extracts_and_renders_document_intake(&client, &target, &proxy).await; + converts_complete_markdown(&client, &target, &proxy).await; refuses_a_stream_that_contradicts_the_spec(&client, &target).await; wait_until_idle(&modules).await; @@ -372,6 +373,49 @@ async fn refuses_a_stream_that_contradicts_the_spec(client: &Connection, target: } } +/// Memory conversion returns complete Markdown through the output lifecycle. +async fn converts_complete_markdown(client: &Connection, target: &Target, proxy: &tinybus::Proxy) { + let text = "Full memory conversion through the compiled module"; + let docx = docx_with_text(text); + let output: OutputRef = client + .call_with_stream( + target.destination.clone(), + target.path.clone(), + target.interface.clone(), + tinybus::MemberName::new(methods::CONVERT_MARKDOWN).unwrap(), + |stream| serde_json::json!([tinydocs_bus::DocumentFormat::Docx, stream]), + &docx, + ) + .await + .expect("ConvertMarkdown should succeed"); + assert_eq!( + String::from_utf8(download(proxy, &output).await).unwrap(), + text + ); + proxy + .call::<()>(methods::RELEASE_OUTPUT, (output.output_id.clone(),)) + .await + .expect("converted Markdown should be released"); + proxy + .call::(methods::READ_OUTPUT, (output.output_id, 0_u64, READ_CHUNK)) + .await + .expect_err("released Markdown should no longer be readable"); + let empty: tinybus::Result = client + .call_with_stream( + target.destination.clone(), + target.path.clone(), + target.interface.clone(), + tinybus::MemberName::new(methods::CONVERT_MARKDOWN).unwrap(), + |stream| serde_json::json!([tinydocs_bus::DocumentFormat::Docx, stream]), + b"invalid archive", + ) + .await; + assert!( + matches!(empty, Err(tinybus::Error::MethodFailed { name, .. }) + if name == "ai.tinyhumans.tinydocs.Error.ExtractionFailed") + ); +} + /// Read a held document back in chunks and verify its digest. async fn download(proxy: &tinybus::Proxy, handle: &OutputRef) -> Vec { let decoder = base64::engine::general_purpose::STANDARD; diff --git a/docs/specs/markdown-conversion.md b/docs/specs/markdown-conversion.md new file mode 100644 index 0000000..4e06586 --- /dev/null +++ b/docs/specs/markdown-conversion.md @@ -0,0 +1,29 @@ +# Complete Markdown conversion + +Contract version 3 adds `ConvertMarkdown(format, StreamRef) -> OutputRef`. +Existing member argument counts and representations remain unchanged. Hosts +must pin a released version-3 module and verify its artifact digest before +calling this member. Version-2 modules do not provide this operation. + +`format` uses the existing `DocumentFormat` wire values: `pdf`, `docx`, `pptx`, +and `xlsx`. Authorized bytes arrive over the existing TinyBus input stream. +The module executes the Markdown converter and retains UTF-8 output in its +bounded output store. The host reads it with `ReadOutput` and releases it with +`ReleaseOutput`; normal expiry also releases abandoned output. + +Unlike `ExtractDocument`, which intentionally produces bounded previews, +`ConvertMarkdown` returns complete normalized text for document persistence. +It preserves the OfficeConverter text shape used by TinyMemory: paragraphs, +XML entity text, numeric PowerPoint slide order, worksheet row labels, and +PDF form-feed page boundaries, including blank pages between text pages. +Hosts keep their existing stored documents and ingestion metadata. + +Empty and oversized input return `InvalidInput`. Malformed, overexpanding, +and entirely empty or scanned documents return `ExtractionFailed`. The +32-MiB input ceiling, 64-MiB declared Office expansion ceiling, and one-million +cell spreadsheet extent ceiling apply before materializing parser data. +No local fallback is required or provided by the bus contract. + +Regression fixtures exercise all four formats and error cases. The compiled +artifact test calls conversion over an actual broker, checks returned bytes, +and verifies output release and structured parse errors. diff --git a/src/lib.rs b/src/lib.rs index b55e699..ae58f60 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -105,3 +105,7 @@ pub mod intake; /// Selected PDF page rendering for host-owned vision/OCR. #[cfg(feature = "pdf-render")] pub mod pdf_render; + +/// Full normalized PDF/Office Markdown conversion for document ingestion. +#[cfg(feature = "markdown")] +pub mod markdown; diff --git a/src/markdown/README.md b/src/markdown/README.md new file mode 100644 index 0000000..a306e5a --- /dev/null +++ b/src/markdown/README.md @@ -0,0 +1,19 @@ +# Markdown conversion + +`tinydocs::markdown::convert(bytes, DocumentFormat)` extracts PDF, DOCX, PPTX, +and XLSX to the complete normalized Markdown used by memory ingestion. It +retains TinyMemory OfficeConverter semantics, including form-feed PDF page +boundaries, numeric slide order, and `sheet | cell | cell` spreadsheet rows. +The implementation is adapted from TinyMemory under GPL-3.0-only. + +Input is limited to 32 MiB. Office archives are admitted only when their +aggregate declared expansion is at most 64 MiB; spreadsheet ranges are checked +with a sparse scan before allocating a dense grid of at most one million +cells. An entirely empty or scanned document returns `ExtractionFailed`. + +The TinyBus `ConvertMarkdown(format, StreamRef)` member executes this converter +inside the compiled module and returns an `OutputRef`. Download with +`ReadOutput(output_id, offset, length)` and release with +`ReleaseOutput(output_id)`. The existing output store limits and expiry apply. +Callers retain authorization, provenance, ingestion metadata, and cancellation. +The pure bus contract contains no parser dependency. diff --git a/src/markdown/mod.rs b/src/markdown/mod.rs new file mode 100644 index 0000000..e3e8f20 --- /dev/null +++ b/src/markdown/mod.rs @@ -0,0 +1,66 @@ +//! PDF and Office conversion to the Markdown used by document-memory ingestion. +//! +//! These readers retain TinyMemory's OfficeConverter conversion semantics: +//! normalized paragraphs, numeric slide order, worksheet row labels, and PDF +//! form-feed page breaks. Parsing stays inside the document module; callers +//! supply authorized bytes and retain their ingestion metadata and policy. +//! The implementation is adapted from TinyMemory's GPL-3.0 Office converter. + +mod normalize; +mod ooxml; +mod pdf; +mod xlsx; + +use crate::intake::DocumentFormat; +use crate::{Error, Result}; + +/// Largest compressed input accepted for Markdown conversion. +pub const MAX_DOCUMENT_BYTES: usize = 32 * 1024 * 1024; +/// Largest declared expansion of an Office archive. +pub const MAX_DECOMPRESSED_BYTES: u64 = 64 * 1024 * 1024; +/// Largest dense used range accepted before spreadsheet grid allocation. +pub const MAX_SPREADSHEET_DENSE_CELLS: usize = 1_000_000; +/// Separates PDF pages, including pages with no text layer. +pub const PAGE_BREAK: char = '\u{c}'; + +/// Convert authorized PDF, DOCX, PPTX or XLSX bytes to normalized Markdown. +/// +/// The result is complete rather than silently truncated. Hosts can store it +/// with their existing document provenance. PDF page boundaries are preserved; +/// an entirely scanned/empty document is refused instead of stored as empty. +/// +/// # Errors +/// Returns [`Error::InvalidInput`] for empty or oversized input, and +/// [`Error::ExtractionFailed`] for malformed, overexpanding, or empty documents. +pub fn convert(bytes: &[u8], format: DocumentFormat) -> Result { + if bytes.is_empty() || bytes.len() > MAX_DOCUMENT_BYTES { + return Err(Error::invalid_input( + "document", + "expected 1 to 33554432 bytes", + )); + } + let markdown = match format { + DocumentFormat::Pdf => pdf::extract(bytes)? + .iter() + .map(|page| normalize::normalize(page)) + .collect::>() + .join(&PAGE_BREAK.to_string()), + DocumentFormat::Docx => normalize::normalize(&ooxml::docx(bytes)?), + DocumentFormat::Pptx => normalize::normalize(&ooxml::pptx(bytes)?), + DocumentFormat::Xlsx => normalize::normalize(&xlsx::extract(bytes)?), + }; + if markdown.trim().is_empty() { + return Err(unreadable(&format!( + "converting {format:?} produced no text" + ))); + } + Ok(markdown) +} + +fn unreadable(reason: &str) -> Error { + Error::extraction_failed(reason) +} + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod tests; diff --git a/src/markdown/mod_tests.rs b/src/markdown/mod_tests.rs new file mode 100644 index 0000000..c04ee92 --- /dev/null +++ b/src/markdown/mod_tests.rs @@ -0,0 +1,360 @@ +//! Markdown compatibility regressions using local PDF/Office fixtures. +#![allow(clippy::unwrap_used, clippy::panic)] +use super::*; +use std::fmt::Write as _; + +/// A deflated zip archive of `(path, contents)` parts. +fn package(parts: &[(&str, &str)]) -> Vec { + use std::io::Write; + + let mut buffer = std::io::Cursor::new(Vec::new()); + let mut writer = zip::ZipWriter::new(&mut buffer); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated); + for (path, contents) in parts { + writer.start_file(*path, options).unwrap(); + writer.write_all(contents.as_bytes()).unwrap(); + } + writer.finish().unwrap(); + buffer.into_inner() +} + +/// A Word document whose body is `paragraphs`, each a list of text runs. +fn docx(paragraphs: &[&[&str]]) -> Vec { + let mut body = String::new(); + for runs in paragraphs { + body.push_str(""); + for run in *runs { + write!(body, "{run}").unwrap(); + } + body.push_str(""); + } + let document = format!( + r#"{body}"# + ); + package(&[("word/document.xml", &document)]) +} + +/// A deck holding one slide per `(slide number, text)` pair, written to the +/// archive in the order given. +fn pptx(slides: &[(u32, &str)]) -> Vec { + let mut parts = vec![( + "ppt/presentation.xml".to_string(), + r#""#.to_string(), + )]; + for (number, text) in slides { + parts.push(( + format!("ppt/slides/slide{number}.xml"), + format!( + r#"{text}"# + ), + )); + } + let borrowed: Vec<(&str, &str)> = parts + .iter() + .map(|(path, contents)| (path.as_str(), contents.as_str())) + .collect(); + package(&borrowed) +} + +/// A minimal XLSX archive from a worksheet's `sheetData` fragment. +/// +/// The parts are the smallest set calamine's Xlsx reader accepts: the +/// content-type map, the root and workbook relationships, and the workbook +/// itself. No shared strings or styles, which the reader tolerates. +fn xlsx_with_sheet(sheet_data: &str) -> Vec { + let content_types = r#" + + + + + +"#; + let root_rels = r#" + + +"#; + let workbook = r#" + + +"#; + let workbook_rels = r#" + + +"#; + let worksheet = format!( + r#" + +{sheet_data} +"# + ); + package(&[ + ("[Content_Types].xml", content_types), + ("_rels/.rels", root_rels), + ("xl/workbook.xml", workbook), + ("xl/_rels/workbook.xml.rels", workbook_rels), + ("xl/worksheets/sheet1.xml", &worksheet), + ]) +} + +/// A one-page PDF whose content stream is `content`, with a correct +/// cross-reference table so the parser takes the normal path rather than a +/// recovery one. +fn pdf(content: &str) -> Vec { + pdf_pages(&[content]) +} + +/// A PDF with one page per content stream, in order. +fn pdf_pages(contents: &[&str]) -> Vec { + // 1 catalog, 2 pages, 3 font, then each page and its content stream. + let kids: Vec = (0..contents.len()) + .map(|index| format!("{} 0 R", 4 + 2 * index)) + .collect(); + let mut objects = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + format!( + "<< /Type /Pages /Kids [{}] /Count {} >>", + kids.join(" "), + contents.len() + ), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + ]; + for (index, content) in contents.iter().enumerate() { + objects.push(format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents {} 0 R \ + /Resources << /Font << /F1 3 0 R >> >> >>", + 5 + 2 * index + )); + objects.push(format!( + "<< /Length {} >>\nstream\n{content}\nendstream", + content.len() + )); + } + let mut out = String::from("%PDF-1.4\n"); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(out.len()); + write!(out, "{} 0 obj\n{object}\nendobj\n", index + 1).unwrap(); + } + let xref = out.len(); + write!(out, "xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).unwrap(); + for offset in offsets { + writeln!(out, "{offset:010} 00000 n ").unwrap(); + } + write!( + out, + "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", + objects.len() + 1 + ) + .unwrap(); + out.into_bytes() +} + +fn convert(name: &str, bytes: &[u8]) -> Result { + let format = match name.rsplit('.').next().unwrap() { + "pdf" => DocumentFormat::Pdf, + "docx" => DocumentFormat::Docx, + "pptx" => DocumentFormat::Pptx, + "xlsx" => DocumentFormat::Xlsx, + _ => panic!("test fixture must name an office format"), + }; + super::convert(bytes, format) +} + +fn refusal(name: &str, bytes: &[u8]) -> String { + convert(name, bytes).unwrap_err().to_string() +} +#[test] +fn a_docx_yields_its_paragraphs_in_order() { + let bytes = docx(&[&["First heading"], &["Second ", "paragraph."]]); + let converted = convert("spec.docx", &bytes).unwrap(); + let text = &converted; + assert!(text.starts_with("First heading"), "{text}"); + assert!( + text.contains("Second paragraph."), + "runs inside one paragraph join without a break: {text}" + ); + assert!( + text.contains("First heading\n\nSecond"), + "paragraphs keep their break: {text:?}" + ); +} + +#[test] +fn escaped_characters_in_a_run_survive_extraction() { + // XML escapes `&` and `<` in text, and the parser reports each escape as + // its own event: dropping those events would turn "Q&A" into "QA". + let bytes = docx(&[&["Q&A: 3 < 4 & "done""]]); + let converted = convert("faq.docx", &bytes).unwrap(); + assert_eq!(converted, "Q&A: 3 < 4 & \"done\""); +} + +#[test] +fn a_deck_reads_its_slides_in_numeric_order() { + // `slide10.xml` sorts before `slide2.xml` as a string, and a deck that + // recalls out of order is worse than one that does not recall at all. + let bytes = pptx(&[(10, "Tenth"), (2, "Second"), (1, "First")]); + let converted = convert("deck.pptx", &bytes).unwrap(); + assert_eq!(converted, "First\n\nSecond\n\nTenth"); +} + +#[test] +fn a_small_spreadsheet_yields_one_line_per_row() { + // The dense-range guard must not refuse ordinary files, and the concrete + // (non-auto) Xlsx open must not either. + let bytes = xlsx_with_sheet( + r#"1alpha + 2beta"#, + ); + let converted = convert("ledger.xlsx", &bytes).unwrap(); + assert_eq!(converted, "Sheet1 | 1 | alpha\nSheet1 | 2 | beta"); +} + +#[test] +fn a_spreadsheet_with_far_cells_is_refused_not_allocated() { + // One cell at `A1` and one at `XFD1048576` passes the decompression cap + // (the archive is a few hundred bytes) yet would make calamine + // materialize a ~17-billion-cell dense grid. Refused before that. + let bytes = xlsx_with_sheet( + r#"1 + 2"#, + ); + let reason = refusal("spread.xlsx", &bytes); + assert!(reason.contains("used range"), "{reason}"); +} + +#[test] +fn an_overexpanding_document_is_refused_not_allocated() { + // One entry of zero bytes just over the cap: zeroes compress to almost + // nothing, so the archive is tiny while its declared expansion exceeds + // the limit — exactly the shape of a crafted bomb. + let zeroes = "\0".repeat(usize::try_from(MAX_DECOMPRESSED_BYTES).unwrap() + 1); + let bytes = package(&[("word/document.xml", &zeroes)]); + assert!( + bytes.len() < MAX_DOCUMENT_BYTES, + "the fixture must pass intake's own cap" + ); + let reason = refusal("bomb.docx", &bytes); + assert!(reason.contains("expands"), "{reason}"); +} + +#[test] +fn an_overexpanding_spreadsheet_is_refused_before_calamine_opens_it() { + let zeroes = "\0".repeat(usize::try_from(MAX_DECOMPRESSED_BYTES).unwrap() + 1); + let bytes = package(&[("xl/workbook.xml", &zeroes)]); + let reason = refusal("bomb.xlsx", &bytes); + assert!(reason.contains("expands"), "{reason}"); +} + +#[test] +fn a_pdf_yields_its_text_layer() { + let bytes = pdf("BT /F1 24 Tf 72 720 Td (Quarterly revenue rose) Tj ET"); + let converted = convert("report.pdf", &bytes).unwrap(); + assert!( + converted.contains("Quarterly revenue rose"), + "{converted:?}" + ); +} + +#[test] +fn a_pdf_keeps_its_page_boundaries_as_page_breaks() { + let bytes = pdf_pages(&[ + "BT /F1 24 Tf 72 720 Td (Refunds take five days) Tj ET", + "BT /F1 24 Tf 72 720 Td (Shipping is free over fifty) Tj ET", + "", + "BT /F1 24 Tf 72 720 Td (Returns need a receipt) Tj ET", + ]); + let converted = convert("policy.pdf", &bytes).unwrap(); + let pages: Vec<&str> = converted.split(PAGE_BREAK).collect(); + assert_eq!(pages.len(), 4, "{converted:?}"); + assert!(pages[0].contains("Refunds take five days"), "{pages:?}"); + assert!( + pages[1].contains("Shipping is free over fifty"), + "{pages:?}" + ); + assert!(pages[2].trim().is_empty(), "an empty page keeps its place"); + assert!(pages[3].contains("Returns need a receipt"), "{pages:?}"); +} + +#[test] +fn a_pdf_without_a_text_layer_says_so_rather_than_storing_nothing() { + // A scanned PDF carries pictures of words. That is not a parse failure, + // but storing an empty body would lose the upload while looking like one + // succeeded. + let reason = refusal("scan.pdf", &pdf("")); + assert!(reason.contains("no text"), "{reason}"); + // Several empty pages are joined by page breaks, which are not text. + let reason = refusal("scan.pdf", &pdf_pages(&["", "", ""])); + assert!(reason.contains("no text"), "{reason}"); +} + +#[test] +fn a_malformed_pdf_is_an_error_not_a_crash() { + let reason = refusal("broken.pdf", b"%PDF-1.7\nthis is not a pdf"); + assert!(reason.contains("PDF"), "{reason}"); +} + +#[test] +fn a_document_with_no_text_is_an_error_not_an_empty_document() { + let reason = refusal("blank.docx", &docx(&[&[" "]])); + assert!(reason.contains("produced no text"), "{reason}"); +} + +#[test] +fn input_limits_are_checked_before_the_parser_runs() { + for bytes in [Vec::new(), vec![0; MAX_DOCUMENT_BYTES + 1]] { + assert!(matches!( + super::convert(&bytes, DocumentFormat::Pdf), + Err(Error::InvalidInput { .. }) + )); + } +} + +#[test] +fn malformed_office_archives_are_extraction_errors() { + for format in [ + DocumentFormat::Docx, + DocumentFormat::Pptx, + DocumentFormat::Xlsx, + ] { + assert!(matches!( + super::convert(b"plain words", format), + Err(Error::ExtractionFailed { .. }) + )); + } +} + +#[test] +fn normalization_collapses_whitespace_without_merging_paragraphs() { + assert_eq!( + normalize::normalize(" First line \n\n\n second\tline \n"), + "First line\n\n second line" + ); +} + +#[test] +fn a_pdf_with_an_invalid_text_operand_is_refused_even_if_the_parser_panics() { + let bytes = pdf("BT /F1 24 Tf 72 720 Td 42 Tj ET"); + let reason = refusal("malformed-operand.pdf", &bytes); + assert!(reason.contains("parser gave up"), "{reason}"); +} + +#[test] +fn spreadsheet_blank_rows_and_cells_preserve_the_remaining_row_labels() { + let bytes = xlsx_with_sheet( + r#"1 + 3"#, + ); + assert_eq!( + convert("gaps.xlsx", &bytes).unwrap(), + "Sheet1 | 1 |\nSheet1 | | 3" + ); + let empty = xlsx_with_sheet(""); + assert!(refusal("empty.xlsx", &empty).contains("no text")); +} + +#[test] +fn a_readable_archive_without_a_workbook_is_an_extraction_error() { + let bytes = package(&[("unrelated.xml", "")]); + assert!(refusal("not-a-workbook.xlsx", &bytes).contains("could not be read")); +} diff --git a/src/markdown/normalize.rs b/src/markdown/normalize.rs new file mode 100644 index 0000000..825e80f --- /dev/null +++ b/src/markdown/normalize.rs @@ -0,0 +1,37 @@ +//! Whitespace normalization for extracted text. + +/// Collapses runs of blank lines to one, interior whitespace runs to a single +/// space, and trims every line's end and the whole text. +/// +/// PDF extraction pads columns with dozens of spaces and OOXML paragraph +/// breaks stack up around empty paragraphs; neither carries meaning, and both +/// would cost a chunker and an embedder for nothing. +pub(super) fn normalize(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut blank_run = 0; + for line in text.lines() { + let line = line.trim_end(); + if line.trim().is_empty() { + blank_run += 1; + if blank_run == 1 { + out.push('\n'); + } + continue; + } + blank_run = 0; + let mut last_space = false; + for ch in line.chars() { + if ch.is_whitespace() { + if !last_space { + out.push(' '); + } + last_space = true; + } else { + out.push(ch); + last_space = false; + } + } + out.push('\n'); + } + out.trim().to_string() +} diff --git a/src/markdown/ooxml.rs b/src/markdown/ooxml.rs new file mode 100644 index 0000000..3082db3 --- /dev/null +++ b/src/markdown/ooxml.rs @@ -0,0 +1,137 @@ +//! Word documents and `PowerPoint` decks: zip archives of XML, walked directly. + +use std::io::{Cursor, Read, Seek}; + +use quick_xml::events::Event; +use zip::ZipArchive; + +use super::{MAX_DECOMPRESSED_BYTES, unreadable}; +use crate::Result; + +/// The body text of a Word document. +/// +/// `w:p` is a paragraph and `w:t` a run of text inside it: joining runs +/// without a paragraph break would run every heading into the sentence after +/// it, and chunkers split on paragraphs. +pub(super) fn docx(bytes: &[u8]) -> Result { + let mut archive = open(bytes, "document")?; + Ok(read_parts( + &mut archive, + &["word/document.xml"], + "w:p", + "w:t", + )) +} + +/// The text of every slide in a deck, in slide order. +pub(super) fn pptx(bytes: &[u8]) -> Result { + const PREFIX: &str = "ppt/slides/slide"; + + let mut archive = open(bytes, "deck")?; + // Numeric, not lexicographic: `slide10.xml` sorts before `slide2.xml` as + // a string. + let mut slides: Vec<(u32, String)> = archive + .file_names() + .filter_map(|name| { + let number = name.strip_prefix(PREFIX)?.strip_suffix(".xml")?; + Some((number.parse().ok()?, name.to_string())) + }) + .collect(); + slides.sort_unstable(); + let names: Vec<&str> = slides.iter().map(|(_, name)| name.as_str()).collect(); + Ok(read_parts(&mut archive, &names, "a:p", "a:t")) +} + +/// Opens an Office archive, refusing one whose declared expansion exceeds +/// [`MAX_DECOMPRESSED_BYTES`] before any entry data is read. +/// +/// `noun` names the document in the refusal ("the spreadsheet expands…"). +pub(super) fn open<'a>(bytes: &'a [u8], noun: &str) -> Result>> { + let mut archive = ZipArchive::new(Cursor::new(bytes)) + .map_err(|error| unreadable(&format!("the {noun} is not a readable archive: {error}")))?; + if declared_uncompressed(&mut archive).is_none_or(|total| total > MAX_DECOMPRESSED_BYTES) { + return Err(unreadable(&format!( + "the {noun} expands beyond the size this build can read safely" + ))); + } + Ok(archive) +} + +/// Sums the uncompressed sizes every entry declares, touching only the +/// central directory — so this stays cheap however far the archive expands. +/// `None` means an entry could not be inspected, which callers refuse. +fn declared_uncompressed(archive: &mut ZipArchive) -> Option { + (0..archive.len()).try_fold(0u64, |total, index| { + archive + .by_index(index) + .ok() + .map(|file| total.saturating_add(file.size())) + }) +} + +/// Reads the named parts in order and pulls their `text_tag` runs, breaking a +/// paragraph wherever `paragraph_tag` closes. A part that is missing or cannot +/// be read is skipped: the rest of the document is still worth having. +fn read_parts( + archive: &mut ZipArchive, + parts: &[&str], + paragraph_tag: &str, + text_tag: &str, +) -> String { + let mut out = String::new(); + for part in parts { + let Ok(file) = archive.by_name(part) else { + continue; + }; + // Capped as well as pre-checked: an archive that declares small sizes + // but streams more cannot force an unbounded allocation either. + let mut xml = String::new(); + if file + .take(MAX_DECOMPRESSED_BYTES) + .read_to_string(&mut xml) + .is_err() + { + continue; + } + out.push_str(&xml_text(&xml, paragraph_tag, text_tag)); + } + out +} + +/// Concatenates every `text_tag` run, with a blank line at each +/// `paragraph_tag` close. +fn xml_text(xml: &str, paragraph_tag: &str, text_tag: &str) -> String { + let mut reader = quick_xml::Reader::from_str(xml); + let mut out = String::new(); + let mut in_text = false; + loop { + match reader.read_event() { + Ok(Event::Start(tag)) if tag.name().as_ref() == text_tag.as_bytes() => in_text = true, + Ok(Event::End(tag)) if tag.name().as_ref() == text_tag.as_bytes() => in_text = false, + Ok(Event::End(tag)) if tag.name().as_ref() == paragraph_tag.as_bytes() => { + out.push_str("\n\n"); + } + Ok(Event::Text(text)) if in_text => { + out.push_str(&text.decode().unwrap_or_default()); + } + // The parser reports each `&` / `&` as its own event; + // dropping them would turn "Q&A" into "QA". + Ok(Event::GeneralRef(reference)) if in_text => { + if let Ok(Some(ch)) = reference.resolve_char_ref() { + out.push(ch); + } else if let Some(entity) = reference + .decode() + .ok() + .and_then(|name| quick_xml::escape::resolve_predefined_entity(&name)) + { + out.push_str(entity); + } + } + // A malformed part yields what was read up to the fault: a + // truncated document still holds the text before the break. + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + } + out +} diff --git a/src/markdown/pdf.rs b/src/markdown/pdf.rs new file mode 100644 index 0000000..1c3d8bb --- /dev/null +++ b/src/markdown/pdf.rs @@ -0,0 +1,22 @@ +//! A PDF's text layer, page by page. + +use super::unreadable; +use crate::Result; + +/// Extracts a PDF's text layer, one string per page, any of which may be +/// empty. +/// +/// A scanned PDF has none: the file was read, it simply carries pictures of +/// words. That comes back as empty pages, and the converter reports it as a +/// document with no text rather than as a parse failure. +pub(super) fn extract(bytes: &[u8]) -> Result> { + // `pdf-extract` panics on some malformed documents rather than erroring. + // Caught so one bad file is one refused document, not a crashed task. + match std::panic::catch_unwind(|| pdf_extract::extract_text_from_mem_by_pages(bytes)) { + Ok(Ok(pages)) => Ok(pages), + Ok(Err(error)) => Err(unreadable(&format!("the PDF could not be read: {error}"))), + Err(_) => Err(unreadable( + "the PDF is malformed enough that the parser gave up on it", + )), + } +} diff --git a/src/markdown/xlsx.rs b/src/markdown/xlsx.rs new file mode 100644 index 0000000..7f494c7 --- /dev/null +++ b/src/markdown/xlsx.rs @@ -0,0 +1,77 @@ +//! Spreadsheets, flattened to one `sheet | cell | cell` line per row. + +use std::io::Cursor; + +use calamine::{Data, Reader, Xlsx}; + +use super::{MAX_SPREADSHEET_DENSE_CELLS, ooxml, unreadable}; +use crate::Result; + +/// Extracts every non-empty row of every sheet, in sheet order. +pub(super) fn extract(bytes: &[u8]) -> Result { + // The zip-bomb guard runs before calamine materializes anything. + drop(ooxml::open(bytes, "spreadsheet")?); + + // Opened as a concrete `Xlsx` rather than auto-detected: the other formats + // calamine's auto-open falls back to (`.xls`, `.xlsb`, `.ods`) build their + // dense ranges during open — before the extent guard below could run — so + // accepting a mislabelled file would reopen the same allocation attack. + let mut workbook: Xlsx<_> = calamine::open_workbook_from_rs(Cursor::new(bytes)) + .map_err(|error| unreadable(&format!("the spreadsheet could not be read: {error}")))?; + let mut out = String::new(); + for name in workbook.sheet_names() { + match dense_cells(&mut workbook, &name) { + None => continue, + Some(cells) if cells > MAX_SPREADSHEET_DENSE_CELLS => { + return Err(unreadable( + "the spreadsheet's used range exceeds the size this build can read safely", + )); + } + Some(_) => {} + } + let Ok(range) = workbook.worksheet_range(&name) else { + continue; + }; + for row in range.rows() { + let cells: Vec = row + .iter() + .map(|cell| match cell { + Data::Empty => String::new(), + other => other.to_string(), + }) + .collect(); + if cells.iter().all(|cell| cell.trim().is_empty()) { + continue; + } + out.push_str(&name); + for cell in cells { + out.push_str(" | "); + out.push_str(cell.trim()); + } + out.push('\n'); + } + } + Ok(out) +} + +/// The number of cells in the bounding box of a sheet's actual cells — what +/// `worksheet_range` would allocate — found by a sparse scan that allocates +/// no grid. `None` for a sheet that cannot be read or has no cells. +fn dense_cells(workbook: &mut Xlsx>, sheet: &str) -> Option { + let mut reader = workbook.worksheet_cells_reader(sheet).ok()?; + let (mut row_min, mut row_max) = (u32::MAX, 0); + let (mut col_min, mut col_max) = (u32::MAX, 0); + while let Ok(Some(cell)) = reader.next_cell() { + let (row, col) = cell.get_position(); + row_min = row_min.min(row); + row_max = row_max.max(row); + col_min = col_min.min(col); + col_max = col_max.max(col); + } + if row_min == u32::MAX { + return None; + } + let rows = u64::from(row_max - row_min) + 1; + let cols = u64::from(col_max - col_min) + 1; + Some(usize::try_from(rows.saturating_mul(cols)).unwrap_or(usize::MAX)) +}