Execute complete memory document conversion through TinyBus

Cargo.toml

@@ -63,6 +63,8 @@ pdf-extract = { version = "0.12", optional = true }
 zip = { version = "8", default-features = false, features = ["deflate"], optional = true }
 # Streaming XML events avoid building attacker-controlled DOM trees.
 quick-xml = { version = "0.41", optional = true }
+# Workbook row conversion for document-memory Markdown; implementation only.
+calamine = { version = "0.36", optional = true }
 # Pure Rust PDF rasterization, isolated from the default extraction build.
 hayro = { version = "0.5", optional = true }
 
@@ -93,6 +95,8 @@ pptx = ["dep:ppt-rs"]
 pdf = ["dep:pdf-extract"]
 # Bounded DOCX/PPTX/XLSX and per-page PDF text intake.
 intake = ["pdf", "dep:zip", "dep:quick-xml"]
+# Full normalized Markdown conversion compatible with document-memory ingestion.
+markdown = ["intake", "dep:calamine"]
 # Selected PDF page rendering to PNG.
 pdf-render = ["pdf", "dep:hayro"]
 

README.md

@@ -137,6 +137,7 @@ GenerateDocx(DocumentSpec)                         -> OutputRef
 GeneratePptx(deck, Option<StreamRef>)              -> OutputRef
 ExtractText(StreamRef)                             -> OutputRef
 ExtractDocument(ExtractDocumentSpec, StreamRef)     -> ExtractedDocument
+ConvertMarkdown(DocumentFormat, StreamRef)          -> OutputRef
 RenderPdf(RenderPdfSpec, StreamRef)                 -> RenderedPdf
 ReadOutput(output_id, offset, len)                 -> base64
 ReleaseOutput(output_id)                           -> ()
@@ -259,3 +260,12 @@ an older release can still serve the original five methods but will not have
 require a published module release for intake. Do not derive release checksums
 from a local build. `OutputRef` is now defined in `tinydocs-bus` and remains
 re-exported by `tinydocs-module` with the same wire fields.
+
+## Complete memory conversion
+
+Optional `markdown` converts PDF, DOCX, PPTX, and XLSX to complete normalized
+Markdown, preserving TinyMemory OfficeConverter output semantics. The compiled
+module exposes `ConvertMarkdown` in contract version 3; hosts must pin a
+compatible published artifact before using it. See the [conversion spec](docs/specs/markdown-conversion.md)
+and [parser limits](src/markdown/README.md). Read the output through `ReadOutput`
+and explicitly release it with `ReleaseOutput`.

crates/tinydocs-bus/README.md

@@ -14,3 +14,6 @@ type, not compatible-looking duplicates.
 The module serves the `METHODS` at `BUS_NAME` and `OBJECT_PATH`. Keep changes
 here backward compatible or advance `CONTRACT_VERSION` according to the
 documented compatibility rule.
+
+Contract version 3 adds `ConvertMarkdown(DocumentFormat, StreamRef) -> OutputRef`
+without changing existing member arities. See the [conversion spec](../../docs/specs/markdown-conversion.md).

crates/tinydocs-bus/src/intake/mod_tests.rs

@@ -69,3 +69,26 @@ fn rejects_unknown_wire_fields_and_formats() {
         .is_err()
     );
 }
+
+#[test]
+fn complete_conversion_preserves_format_vocabulary_and_requires_version_three() {
+    assert_eq!(crate::CONTRACT_VERSION, 3);
+    assert!(crate::is_compatible(3));
+    assert!(!crate::is_compatible(2));
+    assert!(crate::METHODS.contains(&crate::names::methods::CONVERT_MARKDOWN));
+    for (format, wire) in [
+        (DocumentFormat::Pdf, "pdf"),
+        (DocumentFormat::Docx, "docx"),
+        (DocumentFormat::Pptx, "pptx"),
+        (DocumentFormat::Xlsx, "xlsx"),
+    ] {
+        assert_eq!(
+            serde_json::to_value(format).unwrap(),
+            serde_json::json!(wire)
+        );
+        assert_eq!(
+            serde_json::from_value::<DocumentFormat>(serde_json::json!(wire)).unwrap(),
+            format
+        );
+    }
+}

crates/tinydocs-bus/src/names.rs

@@ -16,6 +16,8 @@ pub mod methods {
     pub const EXTRACT_TEXT: &str = "ExtractText";
     /// `ExtractDocument` — bounded document text and provenance from a stream.
     pub const EXTRACT_DOCUMENT: &str = "ExtractDocument";
+    /// `ConvertMarkdown` — convert a streamed Office/PDF document to full Markdown.
+    pub const CONVERT_MARKDOWN: &str = "ConvertMarkdown";
     /// `RenderPdf` — selected PDF pages as held PNG outputs.
     pub const RENDER_PDF: &str = "RenderPdf";
     /// `ReadOutput` — read a bounded base64-encoded output chunk.
@@ -25,11 +27,12 @@ pub mod methods {
 }
 
 /// All method names in the declaration order used by the module interface.
-pub const METHODS: [&str; 7] = [
+pub const METHODS: [&str; 8] = [
     methods::GENERATE_DOCX,
     methods::GENERATE_PPTX,
     methods::EXTRACT_TEXT,
     methods::EXTRACT_DOCUMENT,
+    methods::CONVERT_MARKDOWN,
     methods::RENDER_PDF,
     methods::READ_OUTPUT,
     methods::RELEASE_OUTPUT,

crates/tinydocs-bus/src/version.rs

@@ -2,9 +2,10 @@
 
 /// Current `TinyDocs` wire-contract version.
 ///
-/// Intake methods are additive to version 2. Older version-2 modules may not
-/// serve them; hosts must check method availability or require a newer release.
-pub const CONTRACT_VERSION: u32 = 2;
+/// Version 3 adds full Markdown conversion. Older modules retain their existing
+/// member signatures but do not serve `ConvertMarkdown`; hosts requiring it
+/// must pin a compatible released artifact.
+pub const CONTRACT_VERSION: u32 = 3;
 
 /// Returns whether a host requiring `required` can bind to this contract.
 ///

crates/tinydocs-module/Cargo.toml

@@ -17,7 +17,7 @@ crate-type = ["rlib", "cdylib"]
 
 [dependencies]
 # The pure document library remains independently publishable and bus-agnostic.
-tinydocs = { path = "../..", default-features = false, features = ["docx", "pptx", "pdf", "intake", "pdf-render"] }
+tinydocs = { path = "../..", default-features = false, features = ["docx", "pptx", "pdf", "intake", "pdf-render", "markdown"] }
 # The transport-free vocabulary shared with every TinyBus host.
 tinydocs-bus = { path = "../tinydocs-bus" }
 # TinyBus provides the typed service interface and dynamic module host ABI.

crates/tinydocs-module/src/service/mod.rs

@@ -1,12 +1,13 @@
 //! `TinyBus` service boundary for the document surface.
 //!
-//! One object, `/ai/tinyhumans/tinydocs/Documents`, exporting seven methods:
+//! One object, `/ai/tinyhumans/tinydocs/Documents`, exporting eight methods:
 //!
 //! ```text
 //! GenerateDocx(DocumentSpec)                        -> OutputRef
 //! GeneratePptx(WirePresentationSpec, Option<Stream>) -> OutputRef
 //! ExtractText(StreamRef)                            -> OutputRef
 //! ExtractDocument(spec, StreamRef)                   -> ExtractedDocument
+//! ConvertMarkdown(format, StreamRef)                -> OutputRef
 //! RenderPdf(spec, StreamRef)                         -> RenderedPdf
 //! ReadOutput(output_id, offset, len)                -> base64
 //! ReleaseOutput(output_id)                          -> ()
@@ -133,6 +134,17 @@ impl Documents {
         blocking(move || tinydocs::intake::extract(&bytes, &spec)).await
     }
 
+    /// Convert a streamed document to the full normalized Markdown used by memory.
+    async fn convert_markdown(
+        &self,
+        format: tinydocs_bus::DocumentFormat,
+        document: StreamRef,
+    ) -> BusResult<OutputRef> {
+        let bytes = self.read_stream(&document).await?;
+        let markdown = blocking(move || tinydocs::markdown::convert(&bytes, format)).await?;
+        self.hold(markdown.into_bytes())
+    }
+
     /// Render explicitly selected PDF pages and hold each PNG output.
     async fn render_pdf(&self, spec: RenderPdfSpec, document: StreamRef) -> BusResult<RenderedPdf> {
         let bytes = self.read_stream(&document).await?;
@@ -388,6 +400,7 @@ pub(crate) mod exports {
             "GeneratePptx",
             "ExtractText",
             "ExtractDocument",
+            "ConvertMarkdown",
             "RenderPdf",
             "ReadOutput",
             "ReleaseOutput",

crates/tinydocs-module/src/service/mod_tests.rs

@@ -23,6 +23,7 @@ const DECLARED_METHODS: &[&str] = &[
     "GeneratePptx",
     "ExtractText",
     "ExtractDocument",
+    "ConvertMarkdown",
     "RenderPdf",
     "ReadOutput",
     "ReleaseOutput",

crates/tinydocs-module/tests/module_e2e.rs

@@ -49,6 +49,7 @@ async fn the_built_module_serves_every_format_over_a_real_broker() {
     generates_a_pptx_from_a_streamed_image_pair(&client, &target, &proxy).await;
     extracts_text_from_a_streamed_pdf(&client, &target, &proxy).await;
     extracts_and_renders_document_intake(&client, &target, &proxy).await;
+    converts_complete_markdown(&client, &target, &proxy).await;
     refuses_a_stream_that_contradicts_the_spec(&client, &target).await;
 
     wait_until_idle(&modules).await;
@@ -372,6 +373,49 @@ async fn refuses_a_stream_that_contradicts_the_spec(client: &Connection, target:
     }
 }
 
+/// Memory conversion returns complete Markdown through the output lifecycle.
+async fn converts_complete_markdown(client: &Connection, target: &Target, proxy: &tinybus::Proxy) {
+    let text = "Full memory conversion through the compiled module";
+    let docx = docx_with_text(text);
+    let output: OutputRef = client
+        .call_with_stream(
+            target.destination.clone(),
+            target.path.clone(),
+            target.interface.clone(),
+            tinybus::MemberName::new(methods::CONVERT_MARKDOWN).unwrap(),
+            |stream| serde_json::json!([tinydocs_bus::DocumentFormat::Docx, stream]),
+            &docx,
+        )
+        .await
+        .expect("ConvertMarkdown should succeed");
+    assert_eq!(
+        String::from_utf8(download(proxy, &output).await).unwrap(),
+        text
+    );
+    proxy
+        .call::<()>(methods::RELEASE_OUTPUT, (output.output_id.clone(),))
+        .await
+        .expect("converted Markdown should be released");
+    proxy
+        .call::<String>(methods::READ_OUTPUT, (output.output_id, 0_u64, READ_CHUNK))
+        .await
+        .expect_err("released Markdown should no longer be readable");
+    let empty: tinybus::Result<OutputRef> = client
+        .call_with_stream(
+            target.destination.clone(),
+            target.path.clone(),
+            target.interface.clone(),
+            tinybus::MemberName::new(methods::CONVERT_MARKDOWN).unwrap(),
+            |stream| serde_json::json!([tinydocs_bus::DocumentFormat::Docx, stream]),
+            b"invalid archive",
+        )
+        .await;
+    assert!(
+        matches!(empty, Err(tinybus::Error::MethodFailed { name, .. })
+        if name == "ai.tinyhumans.tinydocs.Error.ExtractionFailed")
+    );
+}
+
 /// Read a held document back in chunks and verify its digest.
 async fn download(proxy: &tinybus::Proxy, handle: &OutputRef) -> Vec<u8> {
     let decoder = base64::engine::general_purpose::STANDARD;

docs/specs/markdown-conversion.md

@@ -0,0 +1,29 @@
+# Complete Markdown conversion
+
+Contract version 3 adds `ConvertMarkdown(format, StreamRef) -> OutputRef`.
+Existing member argument counts and representations remain unchanged. Hosts
+must pin a released version-3 module and verify its artifact digest before
+calling this member. Version-2 modules do not provide this operation.
+
+`format` uses the existing `DocumentFormat` wire values: `pdf`, `docx`, `pptx`,
+and `xlsx`. Authorized bytes arrive over the existing TinyBus input stream.
+The module executes the Markdown converter and retains UTF-8 output in its
+bounded output store. The host reads it with `ReadOutput` and releases it with
+`ReleaseOutput`; normal expiry also releases abandoned output.
+
+Unlike `ExtractDocument`, which intentionally produces bounded previews,
+`ConvertMarkdown` returns complete normalized text for document persistence.
+It preserves the OfficeConverter text shape used by TinyMemory: paragraphs,
+XML entity text, numeric PowerPoint slide order, worksheet row labels, and
+PDF form-feed page boundaries, including blank pages between text pages.
+Hosts keep their existing stored documents and ingestion metadata.
+
+Empty and oversized input return `InvalidInput`. Malformed, overexpanding,
+and entirely empty or scanned documents return `ExtractionFailed`. The
+32-MiB input ceiling, 64-MiB declared Office expansion ceiling, and one-million
+cell spreadsheet extent ceiling apply before materializing parser data.
+No local fallback is required or provided by the bus contract.
+
+Regression fixtures exercise all four formats and error cases. The compiled
+artifact test calls conversion over an actual broker, checks returned bytes,
+and verifies output release and structured parse errors.

src/lib.rs

@@ -105,3 +105,7 @@ pub mod intake;
 /// Selected PDF page rendering for host-owned vision/OCR.
 #[cfg(feature = "pdf-render")]
 pub mod pdf_render;
+
+/// Full normalized PDF/Office Markdown conversion for document ingestion.
+#[cfg(feature = "markdown")]
+pub mod markdown;

src/markdown/README.md

@@ -0,0 +1,19 @@
+# Markdown conversion
+
+`tinydocs::markdown::convert(bytes, DocumentFormat)` extracts PDF, DOCX, PPTX,
+and XLSX to the complete normalized Markdown used by memory ingestion. It
+retains TinyMemory OfficeConverter semantics, including form-feed PDF page
+boundaries, numeric slide order, and `sheet | cell | cell` spreadsheet rows.
+The implementation is adapted from TinyMemory under GPL-3.0-only.
+
+Input is limited to 32 MiB. Office archives are admitted only when their
+aggregate declared expansion is at most 64 MiB; spreadsheet ranges are checked
+with a sparse scan before allocating a dense grid of at most one million
+cells. An entirely empty or scanned document returns `ExtractionFailed`.
+
+The TinyBus `ConvertMarkdown(format, StreamRef)` member executes this converter
+inside the compiled module and returns an `OutputRef`. Download with
+`ReadOutput(output_id, offset, length)` and release with
+`ReleaseOutput(output_id)`. The existing output store limits and expiry apply.
+Callers retain authorization, provenance, ingestion metadata, and cancellation.
+The pure bus contract contains no parser dependency.