This is the main article text " + "with enough content to pass the readability minimum " + "character count for extraction purposes now.
" + "Second paragraph with more detailed content about " + "the topic being discussed on this test page.
diff --git a/tests/test_library.py b/tests/test_library.py new file mode 100644 index 000000000..2ede9cfab --- /dev/null +++ b/tests/test_library.py @@ -0,0 +1,1274 @@ +"""Tests for the Library app — store, pipeline, routes, and collections handoff.""" + +from __future__ import annotations + +import json +import tempfile +from pathlib import Path +from unittest.mock import patch, MagicMock + +import pytest +import pytest_asyncio +from httpx import ASGITransport, AsyncClient +from tinyagentos.library_pipeline import ( + FileProcessor, + HeavyDownloadProcessor, + ImageProcessor, + PdfProcessor, + TextProcessor, + WebProcessor, + YouTubeProcessor, + detect_kind, + run_pipeline, +) +from tinyagentos.library_store import LibraryStore +from tinyagentos.library_collections import handoff_to_collections + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest_asyncio.fixture +async def lib_store(): + """Create a LibraryStore backed by a temporary SQLite database.""" + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as f: + db_path = Path(f.name) + + store = LibraryStore(db_path) + await store.init() + + yield store + + await store.close() + try: + db_path.unlink() + except OSError: + pass + + +@pytest.fixture +def storage_dir(): + """Create a temporary directory for file artifacts.""" + with tempfile.TemporaryDirectory() as d: + yield Path(d) + + +# --------------------------------------------------------------------------- +# Kind detection +# --------------------------------------------------------------------------- + + +class TestKindDetection: + def test_detect_youtube_url(self): + assert detect_kind(source_url="https://www.youtube.com/watch?v=abc123") == "url:youtube" + assert detect_kind(source_url="https://youtube.com/watch?v=abc123") == "url:youtube" + assert detect_kind(source_url="https://youtu.be/abc123") == "url:youtube" + assert detect_kind(source_url="https://m.youtube.com/watch?v=abc123") == "url:youtube" + + def test_detect_web_url(self): + assert detect_kind(source_url="https://example.com") == "url:web" + assert detect_kind(source_url="http://blog.example.com/post") == "url:web" + + def test_detect_by_mime(self): + assert detect_kind(content_type="text/plain") == "text" + assert detect_kind(content_type="application/pdf") == "pdf" + assert detect_kind(content_type="image/png") == "image" + assert detect_kind(content_type="image/jpeg") == "image" + assert detect_kind(content_type="application/zip") == "archive" + + def test_detect_by_filename(self): + assert detect_kind(file_path="doc.txt") == "text" + assert detect_kind(file_path="report.pdf") == "pdf" + assert detect_kind(file_path="photo.jpg") == "image" + assert detect_kind(file_path="icon.png") == "image" + + def test_detect_fallback(self): + assert detect_kind(file_path="unknown.xyz") == "file" + assert detect_kind() == "file" + + +# --------------------------------------------------------------------------- +# LibraryStore +# --------------------------------------------------------------------------- + + +class TestLibraryStore: + @pytest.mark.asyncio + async def test_create_and_get_item(self, lib_store): + item_id = await lib_store.create_item( + kind="text", + title="test.txt", + source_url="", + storage_path="/tmp/test.txt", + size_bytes=42, + ) + assert item_id + + item = await lib_store.get_item(item_id) + assert item is not None + assert item["kind"] == "text" + assert item["title"] == "test.txt" + assert item["status"] == "pending" + assert item["bytes"] == 42 + + @pytest.mark.asyncio + async def test_get_nonexistent_item(self, lib_store): + item = await lib_store.get_item("nonexistent") + assert item is None + + @pytest.mark.asyncio + async def test_list_items(self, lib_store): + id1 = await lib_store.create_item(kind="text", title="a.txt") + id2 = await lib_store.create_item(kind="pdf", title="b.pdf") + id3 = await lib_store.create_item(kind="image", title="c.png") + + items = await lib_store.list_items() + assert len(items) == 3 + + text_items = await lib_store.list_items(kind="text") + assert len(text_items) == 1 + assert text_items[0]["title"] == "a.txt" + + assert len(await lib_store.list_items(status="pending")) == 3 + assert len(await lib_store.list_items(status="ready")) == 0 + + @pytest.mark.asyncio + async def test_update_item(self, lib_store): + item_id = await lib_store.create_item(kind="text", title="old") + await lib_store.update_item(item_id, title="new", status="ready") + + item = await lib_store.get_item(item_id) + assert item["title"] == "new" + assert item["status"] == "ready" + + @pytest.mark.asyncio + async def test_update_item_meta(self, lib_store): + item_id = await lib_store.create_item(kind="text", title="test") + await lib_store.update_item(item_id, meta_json={"preview": "hello"}) + + item = await lib_store.get_item(item_id) + meta = json.loads(item["meta_json"]) + assert meta["preview"] == "hello" + + @pytest.mark.asyncio + async def test_update_invalid_status(self, lib_store): + item_id = await lib_store.create_item(kind="text", title="test") + with pytest.raises(ValueError): + await lib_store.update_item_status(item_id, "invalid_status") + + @pytest.mark.asyncio + async def test_delete_item(self, lib_store): + item_id = await lib_store.create_item(kind="text", title="test") + item = await lib_store.get_item(item_id) + assert item is not None + + await lib_store.delete_item(item_id) + item = await lib_store.get_item(item_id) + assert item is None + + @pytest.mark.asyncio + async def test_artifacts(self, lib_store): + item_id = await lib_store.create_item(kind="text", title="test") + art_id = await lib_store.add_artifact(item_id, kind="text", path="/tmp/test.txt") + + artifacts = await lib_store.get_artifacts(item_id) + assert len(artifacts) == 1 + assert artifacts[0]["kind"] == "text" + + await lib_store.delete_artifact(art_id) + assert len(await lib_store.get_artifacts(item_id)) == 0 + + @pytest.mark.asyncio + async def test_cascade_delete_artifacts(self, lib_store): + item_id = await lib_store.create_item(kind="text", title="test") + await lib_store.add_artifact(item_id, kind="text", path="/tmp/a.txt") + await lib_store.add_artifact(item_id, kind="thumbnail", path="/tmp/thumb.jpg") + + await lib_store.delete_item(item_id) + assert len(await lib_store.get_artifacts(item_id)) == 0 + + @pytest.mark.asyncio + async def test_jobs(self, lib_store): + item_id = await lib_store.create_item(kind="text", title="test") + job_id = await lib_store.create_job(item_id, "ingest") + + job = await lib_store.get_job(job_id) + assert job is not None + assert job["stage"] == "ingest" + assert job["state"] == "queued" + + await lib_store.update_job(job_id, state="done") + job = await lib_store.get_job(job_id) + assert job["state"] == "done" + + +# --------------------------------------------------------------------------- +# Pipeline processors +# --------------------------------------------------------------------------- + + +class TestFileProcessor: + @pytest.mark.asyncio + async def test_process_existing_file(self, lib_store, storage_dir): + file_path = storage_dir / "test.txt" + file_path.write_text("hello world") + + item_id = await lib_store.create_item( + kind="file", title="test.txt", storage_path=str(file_path) + ) + item = await lib_store.get_item(item_id) + + proc = FileProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert len(artifacts) == 1 + assert artifacts[0]["kind"] == "metadata" + + @pytest.mark.asyncio + async def test_process_missing_file(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="file", title="missing.txt", storage_path="/nonexistent/file.txt" + ) + item = await lib_store.get_item(item_id) + + proc = FileProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert len(artifacts) == 0 + + +class TestTextProcessor: + @pytest.mark.asyncio + async def test_extract_text(self, lib_store, storage_dir): + file_path = storage_dir / "notes.txt" + file_path.write_text("line one\nline two\nline three") + + item_id = await lib_store.create_item( + kind="text", title="notes.txt", storage_path=str(file_path) + ) + item = await lib_store.get_item(item_id) + + proc = TextProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert len(artifacts) == 1 + assert artifacts[0]["kind"] == "text" + assert artifacts[0]["meta"]["line_count"] == 3 + assert artifacts[0]["meta"]["char_count"] == 28 + + text_path = Path(artifacts[0]["path"]) + assert text_path.exists() + assert "line one" in text_path.read_text() + + @pytest.mark.asyncio + async def test_text_auto_title(self, lib_store, storage_dir): + file_path = storage_dir / "notes.txt" + file_path.write_text("My Title\nmore content here") + + item_id = await lib_store.create_item( + kind="text", title="", storage_path=str(file_path) + ) + item = await lib_store.get_item(item_id) + + proc = TextProcessor(lib_store, storage_dir) + await proc.process(item) + + updated = await lib_store.get_item(item_id) + assert updated["title"] == "My Title" + + @pytest.mark.asyncio + async def test_text_preview(self, lib_store, storage_dir): + file_path = storage_dir / "long.txt" + file_path.write_text("A" * 500) + + item_id = await lib_store.create_item( + kind="text", title="long.txt", storage_path=str(file_path) + ) + item = await lib_store.get_item(item_id) + + proc = TextProcessor(lib_store, storage_dir) + await proc.process(item) + + updated = await lib_store.get_item(item_id) + meta = json.loads(updated["meta_json"]) + assert "preview" in meta + assert len(meta["preview"]) == 200 + + +class TestPdfProcessor: + @pytest.mark.asyncio + async def test_process_pdf(self, lib_store, storage_dir): + file_path = storage_dir / "test.pdf" + _create_minimal_pdf(file_path) + + item_id = await lib_store.create_item( + kind="pdf", title="test.pdf", storage_path=str(file_path) + ) + item = await lib_store.get_item(item_id) + + proc = PdfProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + + meta_artifacts = [a for a in artifacts if a["kind"] == "metadata"] + assert len(meta_artifacts) >= 1 + assert meta_artifacts[0]["meta"]["page_count"] >= 0 + + @pytest.mark.asyncio + async def test_process_missing_pdf(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="pdf", title="missing.pdf", storage_path="/nonexistent/file.pdf" + ) + item = await lib_store.get_item(item_id) + + proc = PdfProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert len(artifacts) == 0 + + +class TestImageProcessor: + @pytest.mark.asyncio + async def test_process_image(self, lib_store, storage_dir): + file_path = storage_dir / "test.png" + _create_test_image(file_path) + + item_id = await lib_store.create_item( + kind="image", title="test.png", storage_path=str(file_path) + ) + item = await lib_store.get_item(item_id) + + proc = ImageProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + + kinds = {a["kind"] for a in artifacts} + assert "metadata" in kinds + assert "thumbnail" in kinds + + thumb_art = [a for a in artifacts if a["kind"] == "thumbnail"][0] + assert Path(thumb_art["path"]).exists() + + @pytest.mark.asyncio + async def test_process_missing_image(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="image", title="missing.png", storage_path="/nonexistent/file.png" + ) + item = await lib_store.get_item(item_id) + + proc = ImageProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert len(artifacts) == 0 + + +class TestYouTubeProcessor: + @pytest.mark.asyncio + async def test_process_youtube_url(self, lib_store, storage_dir): + """YouTube processor extracts metadata, transcript, thumbnail, chapters.""" + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://www.youtube.com/watch?v=test123", + title="", + ) + item = await lib_store.get_item(item_id) + + proc = YouTubeProcessor(lib_store, storage_dir) + + mock_result = { + "title": "Test Video", + "author": "TestChannel", + "content": "This is the transcript text with enough content to verify.", + "thumbnail": None, + "metadata": { + "video_id": "test123", + "channel": "TestChannel", + "views": 1000, + "likes": 50, + "duration": 120.5, + "upload_date": "20250101", + "chapters": [ + {"title": "Intro", "start_time": 0.0, "end_time": 30.0}, + {"title": "Main", "start_time": 30.0, "end_time": 90.0}, + ], + }, + } + + with ( + patch( + "tinyagentos.knowledge_fetchers.youtube.fetch", + _async_return(mock_result), + ), + patch( + "tinyagentos.knowledge_fetchers.youtube.format_timestamp", + side_effect=lambda s: f"{int(s // 60):02d}:{int(s % 60):02d}", + ), + ): + artifacts = await proc.process(item) + + kinds = {a["kind"] for a in artifacts} + assert "metadata" in kinds + assert "transcript" in kinds + assert "chapters" in kinds + + updated = await lib_store.get_item(item_id) + meta = json.loads(updated["meta_json"]) + assert meta["video_id"] == "test123" + assert meta["channel"] == "TestChannel" + assert meta["duration"] == 120.5 + assert "preview" in meta + + updated_title = updated.get("title", "") + # Processor should set the title from video metadata since + # the item was created with an empty title. + assert updated_title, f"Expected non-empty title, got {updated_title!r}" + + @pytest.mark.asyncio + async def test_process_youtube_url_no_source(self, lib_store, storage_dir): + """YouTube processor returns empty when source_url is missing.""" + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="", + title="", + ) + item = await lib_store.get_item(item_id) + + proc = YouTubeProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert artifacts == [] + + @pytest.mark.asyncio + async def test_youtube_handles_missing_thumbnail(self, lib_store, storage_dir): + """YouTube processor does not error when thumbnail path is absent.""" + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://youtube.com/watch?v=no-thumb", + ) + item = await lib_store.get_item(item_id) + + proc = YouTubeProcessor(lib_store, storage_dir) + + mock_result = { + "title": "No Thumbnail Video", + "author": "TestChannel", + "content": "Some transcript text.", + "thumbnail": "/nonexistent/path/thumb.png", + "metadata": {"video_id": "no-thumb", "channel": "TestChannel"}, + } + + with patch( + "tinyagentos.knowledge_fetchers.youtube.fetch", + _async_return(mock_result), + ): + artifacts = await proc.process(item) + + # Should not have a thumbnail artifact + thumb_artifacts = [a for a in artifacts if a["kind"] == "thumbnail"] + assert len(thumb_artifacts) == 0 + + @pytest.mark.asyncio + async def test_youtube_pipeline_integration(self, lib_store, storage_dir): + """run_pipeline routes url:youtube items to YouTubeProcessor.""" + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://youtube.com/watch?v=pipeline-test", + ) + + mock_result = { + "title": "Pipeline Video", + "author": "PipelineChannel", + "content": "Pipeline transcript.", + "thumbnail": None, + "metadata": { + "video_id": "pipeline-test", + "channel": "PipelineChannel", + }, + } + + with patch( + "tinyagentos.knowledge_fetchers.youtube.fetch", + _async_return(mock_result), + ): + await run_pipeline(lib_store, item_id, storage_dir) + + item = await lib_store.get_item(item_id) + assert item["status"] == "ready" + + artifacts = await lib_store.get_artifacts(item_id) + artifact_kinds = {a["kind"] for a in artifacts} + # FileProcessor runs first (no storage_path → empty), then YouTubeProcessor + assert "metadata" in artifact_kinds + assert "transcript" in artifact_kinds + + +class TestWebProcessor: + @pytest.mark.asyncio + async def test_process_web_url(self, lib_store, storage_dir): + """Web processor fetches HTML, extracts text, stores artifacts.""" + html = ( + "
This is the main article text " + "with enough content to pass the readability minimum " + "character count for extraction purposes now.
" + "Second paragraph with more detailed content about " + "the topic being discussed on this test page.
This article has enough text content " + "to ensure that the readability extractor returns a full " + "result instead of falling back to the simple tag stripper " + "which would otherwise happen for very short pages.
Pipeline test content that is sufficiently " + "long to pass the readability minimum character count." + "
" + ) + + item_id = await lib_store.create_item( + kind="url:web", + source_url="https://example.com/pipeline-web", + ) + + mock_resp = _mock_httpx_response(html, 200) + with ( + patch("httpx.AsyncClient") as mock_client_cls, + patch( + "tinyagentos.routes.desktop_browser.ssrf.validate_url_or_raise", + ), + ): + _mock_httpx_client(mock_client_cls, mock_resp) + await run_pipeline(lib_store, item_id, storage_dir) + + item = await lib_store.get_item(item_id) + assert item["status"] == "ready" + + artifacts = await lib_store.get_artifacts(item_id) + artifact_kinds = {a["kind"] for a in artifacts} + assert "metadata" in artifact_kinds + assert "text" in artifact_kinds + + +# --------------------------------------------------------------------------- +# run_pipeline +# --------------------------------------------------------------------------- + + +class TestRunPipeline: + @pytest.mark.asyncio + async def test_pipeline_text(self, lib_store, storage_dir): + file_path = storage_dir / "notes.txt" + file_path.write_text("sample content") + + item_id = await lib_store.create_item( + kind="text", title="notes.txt", storage_path=str(file_path) + ) + await run_pipeline(lib_store, item_id, storage_dir) + + item = await lib_store.get_item(item_id) + assert item["status"] == "ready" + + artifacts = await lib_store.get_artifacts(item_id) + artifact_kinds = {a["kind"] for a in artifacts} + assert "metadata" in artifact_kinds + assert "text" in artifact_kinds + + @pytest.mark.asyncio + async def test_pipeline_file(self, lib_store, storage_dir): + file_path = storage_dir / "unknown.xyz" + file_path.write_text("raw data") + + item_id = await lib_store.create_item( + kind="file", title="unknown.xyz", storage_path=str(file_path) + ) + await run_pipeline(lib_store, item_id, storage_dir) + + item = await lib_store.get_item(item_id) + assert item["status"] == "ready" + + @pytest.mark.asyncio + async def test_pipeline_error_status(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="file", title="missing.txt", storage_path="/missing/file.txt" + ) + await run_pipeline(lib_store, item_id, storage_dir) + item = await lib_store.get_item(item_id) + assert item["status"] == "error" + + +# --------------------------------------------------------------------------- +# Collections handoff +# --------------------------------------------------------------------------- + + +class TestCollectionsHandoff: + @pytest.mark.asyncio + async def test_handoff(self, lib_store, storage_dir): + file_path = storage_dir / "notes.txt" + file_path.write_text("content for collections") + + item_id = await lib_store.create_item( + kind="text", title="notes.txt", storage_path=str(file_path) + ) + item = await lib_store.get_item(item_id) + + proc = TextProcessor(lib_store, storage_dir) + await proc.process(item) + + collections_dir = storage_dir / "collections" + count = await handoff_to_collections(lib_store, item_id, collections_dir) + + assert count >= 1 + + manifest_path = collections_dir / item_id / "manifest.json" + assert manifest_path.exists() + + manifest = json.loads(manifest_path.read_text()) + assert manifest["item_id"] == item_id + assert manifest["title"] == "notes.txt" + + @pytest.mark.asyncio + async def test_handoff_no_text_artifacts(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="file", title="no_text", storage_path="/some/path" + ) + collections_dir = storage_dir / "collections" + count = await handoff_to_collections(lib_store, item_id, collections_dir) + assert count == 0 + + +# --------------------------------------------------------------------------- +# API routes +# --------------------------------------------------------------------------- + + +class TestLibraryRoutes: + @pytest.mark.asyncio + async def test_library_page(self, client): + resp = await client.get("/library") + assert resp.status_code == 200 + assert "Library" in resp.text + assert "drop-zone" in resp.text + + @pytest.mark.asyncio + async def test_ingest_no_input(self, client): + resp = await client.post("/api/library/ingest") + assert resp.status_code == 400 + data = resp.json() + assert "error" in data + + @pytest.mark.asyncio + async def test_ingest_url(self, client): + resp = await client.post("/api/library/ingest", data={"url": "https://example.com/page"}) + assert resp.status_code == 202 + data = resp.json() + assert "item_id" in data + assert data["status"] == "pending" + + @pytest.mark.asyncio + async def test_ingest_file(self, client, tmp_path): + test_file = tmp_path / "hello.txt" + test_file.write_text("hello world") + + with open(test_file, "rb") as f: + resp = await client.post( + "/api/library/ingest", + files={"file": ("hello.txt", f, "text/plain")}, + ) + assert resp.status_code == 202 + data = resp.json() + assert "item_id" in data + + @pytest.mark.asyncio + async def test_list_items(self, client): + resp = await client.get("/api/library/items") + assert resp.status_code == 200 + data = resp.json() + assert "items" in data + assert "count" in data + + @pytest.mark.asyncio + async def test_get_nonexistent_item(self, client): + resp = await client.get("/api/library/items/nonexistent") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_delete_nonexistent_item(self, client): + resp = await client.delete("/api/library/items/nonexistent") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_reprocess_nonexistent_item(self, client): + resp = await client.post("/api/library/items/nonexistent/reprocess") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_ingest_and_get(self, client): + resp = await client.post("/api/library/ingest", data={"url": "https://example.com"}) + assert resp.status_code == 202 + item_id = resp.json()["item_id"] + + resp = await client.get(f"/api/library/items/{item_id}") + assert resp.status_code == 200 + data = resp.json() + assert data["item"]["id"] == item_id + assert "artifacts" in data + + @pytest.mark.asyncio + async def test_filter_by_kind(self, client): + await client.post("/api/library/ingest", data={"url": "https://example.com"}) + await client.post("/api/library/ingest", data={"url": "https://youtube.com/watch?v=abc"}) + + resp = await client.get("/api/library/items", params={"kind": "url:youtube"}) + data = resp.json() + for item in data["items"]: + assert item["kind"] == "url:youtube" + + +# --------------------------------------------------------------------------- +# P3: LibraryStore — rules + storage accounting +# --------------------------------------------------------------------------- + + +class TestLibraryStoreRules: + @pytest.mark.asyncio + async def test_create_and_list_rules(self, lib_store): + rid = await lib_store.create_rule( + source_pattern="*.youtube.com/*", + quality="1080", + auto_download=True, + ) + assert rid + + rules = await lib_store.list_rules() + assert len(rules) == 1 + assert rules[0]["source_pattern"] == "*.youtube.com/*" + assert rules[0]["quality"] == "1080" + assert rules[0]["auto_download"] == 1 + + @pytest.mark.asyncio + async def test_get_rule(self, lib_store): + rid = await lib_store.create_rule(source_pattern="*.example.com/*") + rule = await lib_store.get_rule(rid) + assert rule is not None + assert rule["source_pattern"] == "*.example.com/*" + + assert await lib_store.get_rule("nonexistent") is None + + @pytest.mark.asyncio + async def test_delete_rule(self, lib_store): + rid = await lib_store.create_rule(source_pattern="*.youtube.com/*") + await lib_store.delete_rule(rid) + assert len(await lib_store.list_rules()) == 0 + + @pytest.mark.asyncio + async def test_match_rules(self, lib_store): + await lib_store.create_rule( + source_pattern="*youtube.com/*", quality="720", auto_download=True, + ) + await lib_store.create_rule( + source_pattern="*vimeo.com/*", quality="1080", auto_download=False, + ) + + matched = await lib_store.match_rules("https://www.youtube.com/watch?v=abc") + assert len(matched) == 1 + assert matched[0]["quality"] == "720" + + matched_vimeo = await lib_store.match_rules("https://vimeo.com/12345") + assert len(matched_vimeo) == 1 + + matched_none = await lib_store.match_rules("https://example.com") + assert len(matched_none) == 0 + + @pytest.mark.asyncio + async def test_match_rules_disabled_ignored(self, lib_store): + rid = await lib_store.create_rule( + source_pattern="*example.com/*", + enabled=False, + ) + matched = await lib_store.match_rules("https://example.com/page") + assert len(matched) == 0 + + @pytest.mark.asyncio + async def test_storage_summary(self, lib_store): + await lib_store.create_item(kind="text", title="a.txt", size_bytes=100) + await lib_store.create_item(kind="pdf", title="b.pdf", size_bytes=500) + await lib_store.create_item(kind="url:youtube", title="c", size_bytes=0) + + summary = await lib_store.get_storage_summary() + assert summary["total_count"] == 3 + assert summary["total_bytes"] == 600 + assert "text" in summary["by_kind"] + assert "pdf" in summary["by_kind"] + + @pytest.mark.asyncio + async def test_storage_summary_empty(self, lib_store): + summary = await lib_store.get_storage_summary() + assert summary["total_count"] == 0 + assert summary["total_bytes"] == 0 + + +# --------------------------------------------------------------------------- +# P3: HeavyDownloadProcessor +# --------------------------------------------------------------------------- + + +class TestHeavyDownloadProcessor: + @pytest.mark.asyncio + async def test_process_heavy_download(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://youtube.com/watch?v=heavy-test", + ) + await lib_store.update_item(item_id, quality="480") + item = await lib_store.get_item(item_id) + + proc = HeavyDownloadProcessor(lib_store, storage_dir) + + mock_path = str(storage_dir / "downloads" / "test123.mp4") + storage_dir.joinpath("downloads").mkdir(parents=True, exist_ok=True) + storage_dir.joinpath("downloads", "test123.mp4").write_text("fake video data") + + with patch( + "tinyagentos.knowledge_fetchers.youtube.download_video", + _async_return(mock_path), + ): + artifacts = await proc.process(item) + + assert len(artifacts) == 1 + assert artifacts[0]["kind"] == "download" + assert artifacts[0]["meta"]["quality"] == "480" + + updated = await lib_store.get_item(item_id) + assert updated["download_path"] == mock_path + assert updated["download_bytes"] > 0 + + @pytest.mark.asyncio + async def test_process_heavy_download_non_youtube(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="url:web", + source_url="https://example.com", + ) + item = await lib_store.get_item(item_id) + + proc = HeavyDownloadProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert artifacts == [] + + @pytest.mark.asyncio + async def test_process_heavy_download_no_source(self, lib_store, storage_dir): + item_id = await lib_store.create_item( + kind="url:youtube", source_url="", + ) + item = await lib_store.get_item(item_id) + + proc = HeavyDownloadProcessor(lib_store, storage_dir) + artifacts = await proc.process(item) + assert artifacts == [] + + @pytest.mark.asyncio + async def test_process_heavy_download_invalid_quality(self, lib_store, storage_dir): + """Invalid quality should fall back to 720.""" + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://youtube.com/watch?v=qtest", + ) + await lib_store.update_item(item_id, quality="9999") + item = await lib_store.get_item(item_id) + + proc = HeavyDownloadProcessor(lib_store, storage_dir) + + mock_path = str(storage_dir / "downloads" / "test.mp4") + storage_dir.joinpath("downloads").mkdir(parents=True, exist_ok=True) + storage_dir.joinpath("downloads", "test.mp4").write_text("data") + + with patch( + "tinyagentos.knowledge_fetchers.youtube.download_video", + _async_return(mock_path), + ): + artifacts = await proc.process(item) + + # Quality should have been normalized to 720 + assert artifacts[0]["meta"]["quality"] == "720" + + +# --------------------------------------------------------------------------- +# P3: run_heavy_pipeline +# --------------------------------------------------------------------------- + + +class TestRunHeavyPipeline: + @pytest.mark.asyncio + async def test_run_heavy_pipeline_rule_quality(self, lib_store, storage_dir): + """When a matching rule exists, its quality is used.""" + await lib_store.create_rule( + source_pattern="*youtube.com/*", + quality="480", + auto_download=False, + ) + + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://youtube.com/watch?v=rule-test", + ) + storage_dir.joinpath("downloads").mkdir(parents=True, exist_ok=True) + + mock_path = str(storage_dir / "downloads" / "test.mp4") + storage_dir.joinpath("downloads", "test.mp4").write_text("fake data") + + from tinyagentos.library_pipeline import run_heavy_pipeline + + with patch( + "tinyagentos.knowledge_fetchers.youtube.download_video", + _async_return(mock_path), + ): + result = await run_heavy_pipeline( + lib_store, item_id, storage_dir, quality="", + ) + + assert result is not None + assert result["quality"] == "480" # From rule + + @pytest.mark.asyncio + async def test_run_heavy_pipeline_explicit_quality(self, lib_store, storage_dir): + """Explicit quality parameter overrides rule quality.""" + await lib_store.create_rule( + source_pattern="*youtube.com/*", + quality="480", + ) + + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://youtube.com/watch?v=explicit-test", + ) + storage_dir.joinpath("downloads").mkdir(parents=True, exist_ok=True) + + mock_path = str(storage_dir / "downloads" / "test.mp4") + storage_dir.joinpath("downloads", "test.mp4").write_text("fake data") + + from tinyagentos.library_pipeline import run_heavy_pipeline + + with patch( + "tinyagentos.knowledge_fetchers.youtube.download_video", + _async_return(mock_path), + ): + result = await run_heavy_pipeline( + lib_store, item_id, storage_dir, quality="1080", + ) + + assert result["quality"] == "1080" # Explicit wins + + @pytest.mark.asyncio + async def test_run_heavy_pipeline_non_youtube(self, lib_store, storage_dir): + """run_heavy_pipeline returns None for non-YouTube items.""" + item_id = await lib_store.create_item( + kind="url:web", + source_url="https://example.com/page", + ) + + from tinyagentos.library_pipeline import run_heavy_pipeline + result = await run_heavy_pipeline( + lib_store, item_id, storage_dir, + ) + assert result is None + + @pytest.mark.asyncio + async def test_run_heavy_pipeline_creates_job(self, lib_store, storage_dir): + """Heavy pipeline creates a job entry.""" + item_id = await lib_store.create_item( + kind="url:youtube", + source_url="https://youtube.com/watch?v=job-test", + ) + storage_dir.joinpath("downloads").mkdir(parents=True, exist_ok=True) + + mock_path = str(storage_dir / "downloads" / "test.mp4") + storage_dir.joinpath("downloads", "test.mp4").write_text("fake data") + + from tinyagentos.library_pipeline import run_heavy_pipeline + + with patch( + "tinyagentos.knowledge_fetchers.youtube.download_video", + _async_return(mock_path), + ): + await run_heavy_pipeline(lib_store, item_id, storage_dir, quality="720") + + jobs = await lib_store.get_item_jobs(item_id) + heavy_jobs = [j for j in jobs if j["stage"] == "heavy_download"] + assert len(heavy_jobs) >= 1 + + +# --------------------------------------------------------------------------- +# P3: Library routes — download, rules, usage +# --------------------------------------------------------------------------- + + +class TestLibraryRoutesP3: + @pytest.mark.asyncio + async def test_trigger_download(self, client): + """POST /api/library/items/{id}/download triggers heavy download.""" + # Ingest a YouTube URL first + resp = await client.post( + "/api/library/ingest", data={"url": "https://youtube.com/watch?v=dl-test"} + ) + assert resp.status_code == 202 + item_id = resp.json()["item_id"] + + # Wait for pipeline to reach a terminal status with bounded polling. + import asyncio as _asyncio + for _ in range(20): + item_resp = await client.get(f"/api/library/items/{item_id}") + if item_resp.status_code == 200: + if item_resp.json().get("status") in ("ready", "error"): + break + await _asyncio.sleep(0.25) + + resp = await client.post( + f"/api/library/items/{item_id}/download", + data={"quality": "480"}, + ) + assert resp.status_code == 202 + data = resp.json() + assert data["status"] == "downloading" + assert data["quality"] == "480" + + @pytest.mark.asyncio + async def test_trigger_download_nonexistent(self, client): + resp = await client.post("/api/library/items/nonexistent/download") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_trigger_download_non_youtube(self, client): + """Download endpoint rejects non-YouTube items.""" + resp = await client.post( + "/api/library/ingest", data={"url": "https://example.com/page"} + ) + assert resp.status_code == 202 + item_id = resp.json()["item_id"] + + import asyncio as _asyncio + for _ in range(20): + item_resp = await client.get(f"/api/library/items/{item_id}") + if item_resp.status_code == 200: + if item_resp.json().get("status") in ("ready", "error"): + break + await _asyncio.sleep(0.25) + + resp = await client.post(f"/api/library/items/{item_id}/download") + assert resp.status_code == 400 + + @pytest.mark.asyncio + async def test_download_status(self, client): + resp = await client.post( + "/api/library/ingest", data={"url": "https://youtube.com/watch?v=status"} + ) + item_id = resp.json()["item_id"] + + import asyncio as _asyncio + for _ in range(20): + item_resp = await client.get(f"/api/library/items/{item_id}") + if item_resp.status_code == 200: + if item_resp.json().get("status") in ("ready", "error"): + break + await _asyncio.sleep(0.25) + + resp = await client.get(f"/api/library/items/{item_id}/download/status") + assert resp.status_code == 200 + data = resp.json() + assert data["item_id"] == item_id + assert "downloaded" in data + + @pytest.mark.asyncio + async def test_create_rule(self, client): + resp = await client.post( + "/api/library/rules", + data={"source_pattern": "*.youtube.com/*", "quality": "1080", "auto_download": "true"}, + ) + assert resp.status_code == 201 + data = resp.json() + assert data["rule_id"] + assert data["source_pattern"] == "*.youtube.com/*" + + @pytest.mark.asyncio + async def test_list_rules(self, client): + # Create a rule first + await client.post( + "/api/library/rules", data={"source_pattern": "*.example.com/*"} + ) + resp = await client.get("/api/library/rules") + assert resp.status_code == 200 + data = resp.json() + assert "rules" in data + assert data["count"] >= 1 + + @pytest.mark.asyncio + async def test_delete_rule(self, client): + resp = await client.post( + "/api/library/rules", data={"source_pattern": "*.test.com/*"} + ) + rule_id = resp.json()["rule_id"] + + resp = await client.delete(f"/api/library/rules/{rule_id}") + assert resp.status_code == 200 + assert resp.json()["status"] == "deleted" + + # Verify it's gone + resp = await client.get("/api/library/rules") + data = resp.json() + rule_ids = [r["id"] for r in data["rules"]] + assert rule_id not in rule_ids + + @pytest.mark.asyncio + async def test_delete_nonexistent_rule(self, client): + resp = await client.delete("/api/library/rules/nonexistent") + assert resp.status_code == 404 + + @pytest.mark.asyncio + async def test_storage_usage(self, client): + resp = await client.get("/api/library/usage") + assert resp.status_code == 200 + data = resp.json() + assert "total_count" in data + assert "total_bytes" in data + assert "by_kind" in data + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _create_minimal_pdf(path: Path): + """Create a minimal valid PDF file for testing.""" + pdf_content = ( + b"%PDF-1.4\n" + b"1 0 obj<>endobj\n" + b"2 0 obj<>endobj\n" + b"3 0 obj<>endobj\n" + b"xref\n0 4\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n0000000115 00000 n \n" + b"trailer<>\n" + b"startxref\n190\n%%EOF\n" + ) + path.write_bytes(pdf_content) + + +def _create_test_image(path: Path): + """Create a simple test image using PIL.""" + from PIL import Image + img = Image.new("RGB", (100, 50), color="blue") + img.save(path) + + +def _async_return(value): + """Return an async callable that returns ``value`` (for mocking async functions).""" + async def _inner(*args, **kwargs): + return value + return _inner + + +def _mock_httpx_client(mock_client_cls, mock_resp): + """Configure a patched httpx.AsyncClient to support both .get() and .stream(). + + .get() is used for redirect resolution (Phase 1); .stream() is used for + the final body download with in-stream size capping (Phase 2). + """ + from unittest.mock import MagicMock + + stream_ctx = MagicMock() + stream_ctx.__aenter__ = _async_return(mock_resp) + stream_ctx.__aexit__ = _async_return(None) + + client = MagicMock() + client.get = _async_return(mock_resp) + client.head = _async_return(mock_resp) + client.stream = MagicMock(return_value=stream_ctx) + + mock_client_cls.return_value.__aenter__.return_value = client + + +def _mock_httpx_response(html: str, status_code: int = 200): + """Return a mock httpx Response with the given HTML body.""" + mock = MagicMock() + mock.text = html + mock.status_code = status_code + mock.is_redirect = False + mock.encoding = "utf-8" + mock.headers = {"content-type": "text/html; charset=utf-8"} + + async def _aiter_bytes(_chunk_size: int = 8192): + yield html.encode("utf-8") + + mock.aiter_bytes = _aiter_bytes + return mock diff --git a/tinyagentos/library_collections.py b/tinyagentos/library_collections.py new file mode 100644 index 000000000..05ef6b742 --- /dev/null +++ b/tinyagentos/library_collections.py @@ -0,0 +1,120 @@ +"""Library collections handoff — writes processed text artifacts to taosmd. + +After the ingest pipeline produces text artifacts (extracted text, transcripts, +descriptions), this module hands them off to taosmd collections so agents can +query the content through collection grants. + +The design doc (docs/design/library-app.md) says: + "Collections handoff: write text artifacts into a per-target folder under an + allowed root, then taosmd collections index; link to project; grants stay + EXPLICIT" +""" + +from __future__ import annotations + +import logging +import json +from pathlib import Path + +from tinyagentos.library_store import LibraryStore + +logger = logging.getLogger(__name__) + +# Text artifact kinds that should be indexed into collections +_TEXT_ARTIFACT_KINDS = frozenset({"text", "transcript", "description", "ocr"}) + + +async def handoff_to_collections( + store: LibraryStore, + item_id: str, + collections_dir: Path, + project_id: str | None = None, +) -> int: + """Hand off all text artifacts for an item to the taosmd collection index. + + Reads text artifacts from the library store and ingests them into taosmd. + Returns the number of artifacts successfully handed off. + + The collections_dir is the allowed root (e.g. ``data/collections/``). + Each library item gets a subfolder named by its id, so a future + re-index or deletion can target it cleanly. + """ + artifacts = await store.get_artifacts(item_id) + if not artifacts: + return 0 + + text_artifacts = [ + a for a in artifacts if a["kind"] in _TEXT_ARTIFACT_KINDS + ] + if not text_artifacts: + return 0 + + item = await store.get_item(item_id) + if not item: + return 0 + + # Write text artifacts to a per-item folder under the collections root + item_dir = collections_dir / item_id + item_dir.mkdir(parents=True, exist_ok=True) + + handed_off = 0 + for art in text_artifacts: + art_path = art.get("path", "") + if not art_path: + continue + + src = Path(art_path) + if not src.exists(): + continue + + # Copy to collections folder + dst = item_dir / src.name + try: + dst.write_bytes(src.read_bytes()) + except OSError: + logger.warning("Failed to copy artifact %s → %s", src, dst, + exc_info=True) + continue + + # Ingest into taosmd + try: + import taosmd + + text_content = src.read_text(encoding="utf-8", errors="replace") + await taosmd.ingest( + text_content, + agent=f"library-{item_id[:12]}", + project=project_id, + ) + handed_off += 1 + logger.debug("Library item %s artifact %s ingested into taosmd", + item_id, art["kind"]) + except ImportError: + logger.debug("taosmd not available — collection indexing skipped") + # Still count as handed off since file is in place + handed_off += 1 + except Exception: + logger.warning( + "taosmd ingest failed for item %s artifact %s", + item_id, art["kind"], exc_info=True, + ) + + # Write a manifest so downstream knows what's here + manifest = { + "item_id": item_id, + "title": item.get("title", ""), + "kind": item.get("kind", ""), + "source_url": item.get("source_url", ""), + "created_at": item.get("created_at", 0), + "artifacts": [ + {"kind": a["kind"], "file": Path(a["path"]).name} + for a in text_artifacts if a.get("path") + ], + } + manifest_path = item_dir / "manifest.json" + try: + manifest_path.write_text(json.dumps(manifest, indent=2)) + except OSError: + pass + + return handed_off diff --git a/tinyagentos/library_pipeline.py b/tinyagentos/library_pipeline.py new file mode 100644 index 000000000..5e24e8696 --- /dev/null +++ b/tinyagentos/library_pipeline.py @@ -0,0 +1,858 @@ +"""Library ingest pipeline — processors for cheap-tier file/text/pdf/image ingestion. + +Processors are registered per detected kind and run asynchronously after ingest. +Each processor produces artifacts (e.g. metadata, extracted text, thumbnails) +that are stored on the item and optionally handed off to taosmd collections. +""" + +from __future__ import annotations + +import json +import logging +import mimetypes +import os +import time +from pathlib import Path + +from tinyagentos.library_store import LibraryStore + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Kind detection +# --------------------------------------------------------------------------- + +_MIME_KIND_MAP: dict[str, str] = { + "text/plain": "text", + "text/markdown": "text", + "text/csv": "text", + "text/html": "text", + "application/json": "text", + "application/xml": "text", + "text/xml": "text", + "application/pdf": "pdf", + "image/png": "image", + "image/jpeg": "image", + "image/gif": "image", + "image/webp": "image", + "image/svg+xml": "image", + "application/zip": "archive", + "application/gzip": "archive", + "application/x-tar": "archive", +} + + +def detect_kind(source_url: str = "", content_type: str = "", + file_path: str = "") -> str: + """Detect the library item kind from URL, MIME, or file path.""" + # URL-based detection + if source_url: + lower = source_url.lower() + if any(lower.startswith(p) for p in ("https://www.youtube.com/", + "https://youtube.com/", + "https://youtu.be/", + "https://m.youtube.com/")): + return "url:youtube" + if any(lower.startswith(p) for p in ("https://", "http://")): + return "url:web" + + # MIME-based detection + if content_type: + ct = content_type.split(";")[0].strip().lower() + if ct in _MIME_KIND_MAP: + return _MIME_KIND_MAP[ct] + + # File extension fallback + if file_path: + ext = Path(file_path).suffix.lower() + ext_map = { + ".txt": "text", ".md": "text", ".csv": "text", + ".json": "text", ".xml": "text", ".html": "text", + ".pdf": "pdf", + ".png": "image", ".jpg": "image", ".jpeg": "image", + ".gif": "image", ".webp": "image", ".svg": "image", + ".zip": "archive", ".gz": "archive", ".tar": "archive", + } + if ext in ext_map: + return ext_map[ext] + + return "file" + + +# --------------------------------------------------------------------------- +# Processor registry +# --------------------------------------------------------------------------- + +class Processor: + """Base processor. Subclasses handle one kind of library item.""" + + def __init__(self, store: LibraryStore, storage_dir: Path): + self.store = store + self.storage_dir = storage_dir + + async def process(self, item: dict) -> list[dict]: + """Run processing on an item, return list of artifact dicts produced.""" + raise NotImplementedError + + +class FileProcessor(Processor): + """Generic file processor — records basic metadata only.""" + + async def process(self, item: dict) -> list[dict]: + item_id = item["id"] + artifacts: list[dict] = [] + + storage_path = item.get("storage_path", "") + if storage_path: + p = Path(storage_path) + if p.exists(): + stat = p.stat() + file_meta = { + "size_bytes": stat.st_size, + "mtime": stat.st_mtime, + } + # Try mimetype detection + mime_type, _ = mimetypes.guess_type(p.name) + if mime_type: + file_meta["mime_type"] = mime_type + + await self.store.add_artifact( + item_id, kind="metadata", path=storage_path, meta=file_meta + ) + artifacts.append({"kind": "metadata", "path": storage_path, "meta": file_meta}) + + # Update item bytes + await self.store.update_item(item_id, bytes=stat.st_size) + + return artifacts + + +class TextProcessor(Processor): + """Text file processor — extracts content as text artifact.""" + + async def process(self, item: dict) -> list[dict]: + item_id = item["id"] + artifacts: list[dict] = [] + + storage_path = item.get("storage_path", "") + if not storage_path: + return artifacts + + p = Path(storage_path) + if not p.exists(): + logger.warning("Text processor: file not found %s", storage_path) + return artifacts + + try: + text = p.read_text(encoding="utf-8", errors="replace") + except Exception: + logger.warning("Text processor: could not read %s", storage_path, + exc_info=True) + return artifacts + + # Write extracted text as an artifact + text_dir = self.storage_dir / "text" + text_dir.mkdir(parents=True, exist_ok=True) + text_path = text_dir / f"{item_id}.txt" + text_path.write_text(text, encoding="utf-8") + + text_meta = { + "char_count": len(text), + "line_count": text.count("\n") + 1, + } + await self.store.add_artifact( + item_id, kind="text", path=str(text_path), meta=text_meta + ) + artifacts.append({"kind": "text", "path": str(text_path), "meta": text_meta}) + + # Store a preview (first 200 chars) + preview = text[:200] + meta = json.loads(item.get("meta_json", "{}")) + meta["preview"] = preview + await self.store.update_item(item_id, meta_json=meta) + + # Auto-title from content if no title + if not item.get("title"): + title = text.strip().split("\n", 1)[0][:100] + if title: + await self.store.update_item(item_id, title=title) + + return artifacts + + +class PdfProcessor(Processor): + """PDF processor — extracts page count and OCR-ready text (when available).""" + + async def process(self, item: dict) -> list[dict]: + item_id = item["id"] + artifacts: list[dict] = [] + + storage_path = item.get("storage_path", "") + if not storage_path: + return artifacts + + p = Path(storage_path) + if not p.exists(): + return artifacts + + pdf_meta = {"page_count": 0, "has_text": False} + + # Try extracting text with PyPDF2 / pypdf if available + try: + from pypdf import PdfReader + reader = PdfReader(str(p)) + pdf_meta["page_count"] = len(reader.pages) + + # Extract text from all pages + pages_text: list[str] = [] + for page in reader.pages: + page_text = page.extract_text() + if page_text: + pages_text.append(page_text) + + if pages_text: + text_content = "\n\n".join(pages_text) + pdf_meta["has_text"] = True + pdf_meta["char_count"] = len(text_content) + + text_dir = self.storage_dir / "text" + text_dir.mkdir(parents=True, exist_ok=True) + text_path = text_dir / f"{item_id}_pdf.txt" + text_path.write_text(text_content, encoding="utf-8") + + await self.store.add_artifact( + item_id, kind="text", path=str(text_path), + meta={"char_count": len(text_content), "pages": len(reader.pages)}, + ) + artifacts.append({ + "kind": "text", "path": str(text_path), + "meta": {"char_count": len(text_content), "pages": len(reader.pages)}, + }) + + # Update item with preview + preview = text_content[:200] + meta = json.loads(item.get("meta_json", "{}")) + meta["preview"] = preview + await self.store.update_item(item_id, meta_json=meta) + except ImportError: + logger.debug("pypdf not installed — PDF text extraction skipped") + except Exception: + logger.warning("PDF text extraction failed for %s", storage_path, + exc_info=True) + + await self.store.add_artifact( + item_id, kind="metadata", path=storage_path, meta=pdf_meta + ) + artifacts.append({"kind": "metadata", "path": storage_path, "meta": pdf_meta}) + + return artifacts + + +class YouTubeProcessor(Processor): + """YouTube URL processor — cheap tier: metadata, thumbnail, transcript, chapters. + + Uses yt-dlp via the knowledge_fetchers.youtube module to fetch video + metadata and captions without downloading the video file. Produces + artifacts that flow into taosmd collections for agent querying. + + The cheap tier (per the design doc, docs/design/library-app.md section 4) + covers steps 1-4: canonical link, title, channel, description, thumbnail, + duration, upload date, subtitles/transcript, chapters. + """ + + async def process(self, item: dict) -> list[dict]: + item_id = item["id"] + source_url = item.get("source_url", "") + artifacts: list[dict] = [] + + if not source_url: + return artifacts + + # Only catch ImportError (missing yt-dlp). Let fetch errors + # propagate so run_pipeline marks the item as "error" — a failed + # yt-dlp invocation must not silently look successful. + from tinyagentos.knowledge_fetchers.youtube import ( + fetch, + format_timestamp, + ) + + media_dir = self.storage_dir / "youtube" + result = await fetch(source_url, media_dir=media_dir) + + title = result.get("title", "") + if title and not item.get("title"): + await self.store.update_item(item_id, title=title) + + meta = result.get("metadata", {}) + + # Update item meta_json with structured video metadata + stored_meta = json.loads(item.get("meta_json", "{}")) + stored_meta.update({ + "video_id": meta.get("video_id", ""), + "channel": meta.get("channel", ""), + "duration": meta.get("duration"), + "views": meta.get("views"), + "upload_date": meta.get("upload_date", ""), + }) + await self.store.update_item(item_id, meta_json=stored_meta) + + # Artifact: metadata + await self.store.add_artifact( + item_id, kind="metadata", path=source_url, meta=meta, + ) + artifacts.append({"kind": "metadata", "path": source_url, "meta": meta}) + + # Artifact: thumbnail (if downloaded) + thumbnail = result.get("thumbnail") + if thumbnail and Path(thumbnail).exists(): + await self.store.add_artifact( + item_id, kind="thumbnail", path=thumbnail, + meta={"source": "youtube"}, + ) + artifacts.append({ + "kind": "thumbnail", "path": thumbnail, + "meta": {"source": "youtube"}, + }) + + # Artifact: transcript + content = result.get("content", "") + if content: + transcript_dir = self.storage_dir / "transcripts" + transcript_dir.mkdir(parents=True, exist_ok=True) + transcript_path = transcript_dir / f"{item_id}_transcript.txt" + transcript_path.write_text(content, encoding="utf-8") + + transcript_meta = { + "char_count": len(content), + "language": "en", + } + await self.store.add_artifact( + item_id, kind="transcript", path=str(transcript_path), + meta=transcript_meta, + ) + artifacts.append({ + "kind": "transcript", "path": str(transcript_path), + "meta": transcript_meta, + }) + + # Store preview for item card + preview = content[:200] + stored_meta["preview"] = preview + await self.store.update_item(item_id, meta_json=stored_meta) + + # Artifact: chapters (if available) + chapters = meta.get("chapters", []) + if chapters: + chapters_lines: list[str] = [] + for ch in chapters: + ts = format_timestamp(ch.get("start_time", 0)) + ch_title = ch.get("title", "") + chapters_lines.append(f"[{ts}] {ch_title}") + + chapters_text = "\n".join(chapters_lines) + chapters_dir = self.storage_dir / "chapters" + chapters_dir.mkdir(parents=True, exist_ok=True) + chapters_path = chapters_dir / f"{item_id}_chapters.txt" + chapters_path.write_text(chapters_text, encoding="utf-8") + + await self.store.add_artifact( + item_id, kind="chapters", path=str(chapters_path), + meta={"count": len(chapters)}, + ) + artifacts.append({ + "kind": "chapters", "path": str(chapters_path), + "meta": {"count": len(chapters)}, + }) + + return artifacts + + +class WebProcessor(Processor): + """Generic web-page URL processor — extracts readable text from HTML. + + Fetches the URL (SSRF-guarded against loopback/link-local/private hosts), + then extracts the main content using readability-lxml. Falls back to a + simple tag-stripping approach when readability-lxml is not installed. + + Produces a text artifact suitable for taosmd collection indexing so that + agents granted the collection can query the page content. + """ + + async def process(self, item: dict) -> list[dict]: + item_id = item["id"] + source_url = item.get("source_url", "") + artifacts: list[dict] = [] + + if not source_url: + return artifacts + + # Fetch the page (SSRF-guarded, redirect-safe, size-capped). + # Follows the same pattern as knowledge_ingest._download_article: + # disable auto-redirects, manually validate each hop against the + # SSRF blocklist, and cap total response bytes. + import httpx + from urllib.parse import urljoin + + from tinyagentos.routes.desktop_browser.ssrf import ( + SsrfBlockedError, + validate_url_or_raise, + ) + + _MAX_WEB_REDIRECTS = 5 + _MAX_WEB_BYTES = 10 * 1024 * 1024 # 10 MB + + # Phase 1: resolve redirects using HEAD to avoid buffering + # bodies. Falls back to GET on connection/network errors, but + # a server that rejects HEAD (e.g. 405) won't be a redirect + # response either — so the loop exits and Phase 2 streams. + current_url = source_url + for _hop in range(_MAX_WEB_REDIRECTS + 1): + validate_url_or_raise(current_url) + + async with httpx.AsyncClient( + timeout=httpx.Timeout(30), + follow_redirects=False, + ) as client: + try: + resp = await client.head(current_url) + except Exception: + resp = await client.get(current_url) + + if resp.is_redirect and resp.headers.get("location"): + current_url = urljoin(current_url, resp.headers["location"]) + continue + break + else: + raise SsrfBlockedError( + f"too many redirects fetching {source_url!r}" + ) + + # Phase 2: stream the final URL, enforcing _MAX_WEB_BYTES during + # download rather than after the full body is buffered in memory. + body_chunks: list[bytes] = [] + total = 0 + async with httpx.AsyncClient( + timeout=httpx.Timeout(30), + follow_redirects=False, + ) as client: + async with client.stream("GET", current_url) as stream_resp: + stream_resp.raise_for_status() + async for chunk in stream_resp.aiter_bytes(8192): + total += len(chunk) + if total > _MAX_WEB_BYTES: + raise ValueError( + f"Response body exceeds {_MAX_WEB_BYTES} bytes " + f"for {source_url!r}" + ) + body_chunks.append(chunk) + encoding = stream_resp.encoding or "utf-8" + html = b"".join(body_chunks).decode(encoding, errors="replace") + + if not html: + return artifacts + + # Extract readable text + content = _extract_readable_text(html, source_url) + + # Extract title fromNo items yet. Drop a file or paste a URL above.
" + "{pat} ({qual}p, {auto})'
+ f''
+ f'Drop files here or paste a URL
+ +No items yet. Drop a file or paste a URL above.
+