diff --git a/.gitignore b/.gitignore index b1de70c..65802d6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ hugging_face/assets/ results/ test_sample/ pretrained_models/ -data/ \ No newline at end of file +data/ +.worktrees/ +runtime/ +.venv/ diff --git a/README.md b/README.md index a948af2..03a4f87 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,21 @@ pip3 install -r hugging_face/requirements.txt ``` +### Windows `.venv` Setup For Internal Web App +If you are running the internal web app on a Windows workstation, a local `.venv` is the most reliable setup path: + +```powershell +py -3.10 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install torch==2.7.0 torchvision==0.22.0 torchaudio==2.7.0 --index-url https://download.pytorch.org/whl/cu128 +python -m pip install -e . +python -m pip install decord +python -m pip install -e D:\my_app\lens_hunter2\models\sam3\sam3_repo --no-deps +``` + +The gradio demo keeps its extra dependencies in [`hugging_face/requirements.txt`](./hugging_face/requirements.txt), so install those separately only when you need the demo UI. + ## ๐Ÿ”ฅ Inference ### Download Model @@ -138,6 +153,110 @@ By launching, an interactive interface will appear as follow. ![overall_teaser](assets/teaser_demo.gif) +## ๐Ÿงช Internal Web App +The repository now also includes a desktop-first internal web app for the queue-based review workflow. + +Current flow: + +1. Upload a short source clip from the `New Session` page. +2. Enter the `Annotation Workbench` and build one or more target layers from the template frame. +3. Submit the queued job and inspect `source / overlay / alpha / foreground` from the `Result Review` page before downloading artifacts. + +Current outputs: + +- `foreground.mp4` +- `alpha.mp4` +- `rgba_png.zip` +- `output_prores4444.mov` + +### Annotation Workbench + +The current workbench is a single-page desktop UI built around `SAM3 -> key-frame mask -> MatAnyone2 bidirectional video matting`. + +Workbench capabilities: + +- multi-target layer management +- arbitrary template-frame selection before annotation +- `Positive / Negative` point placement +- `Balanced / Hair / Edge / Motion` refine presets +- `Add / Remove / Feather` brush cleanup +- numeric `Preset strength / Motion softness / Temporal stability` controls +- `Source / Overlay / Mask` review modes +- export-mask selection before queue submission + +The internal web app now uses local `SAM3` as the default interactive segmentation backend for the annotation workbench, while `MatAnyone2` remains the video matting backend after submission. When the selected template frame is not frame `0`, the worker now splits inference into forward and backward passes from that frame and stitches the result before export. You can still override the segmentation backend or checkpoint at launch time: + +```powershell +$env:MATANYONE2_WEBAPP_SAM_BACKEND = "sam3" +$env:MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH = "D:\my_app\lens_hunter2\models\sam3\checkpoints\sam3.pt" +# fallback: switch back to SAM2 if needed +$env:MATANYONE2_WEBAPP_SAM_BACKEND = "sam2" +$env:MATANYONE2_WEBAPP_SAM2_VARIANT = "sam2.1_hiera_large" +``` + +### Start For Testing + +Recommended on Windows: + +```powershell +.\scripts\start_internal_webapp.ps1 +``` + +Open: + +```text +http://127.0.0.1:8010 +``` + +Check or stop the service: + +```powershell +.\scripts\check_internal_webapp.ps1 +.\scripts\stop_internal_webapp.ps1 +``` + +Manual split-process launch is still available: + +```shell +python scripts/run_internal_webapp.py +``` + +Run the worker in a separate process: + +```shell +python scripts/run_internal_worker.py +``` + +### Quick Test Path + +1. Upload a short clip. +2. Choose the best template frame for segmentation. +3. Use positive and negative clicks to isolate the person. +4. Switch the preset to `Hair`, `Edge`, `Motion`, or `Balanced` based on the edge you care about. +5. Tune `Preset strength`, `Motion softness`, and `Temporal stability`. +6. Use `Add / Remove / Feather` to manually correct the key-frame mask. +7. Save one or more targets, choose which saved masks to export, and submit the job. +6. Review `Overlay / Alpha / Foreground` on the result page before downloading artifacts. + +### Smoke Test + +Run an end-to-end smoke test that launches the web app, launches the worker, submits two back-to-back jobs, and verifies queueing plus artifact export: + +```shell +python scripts/smoke_internal_webapp.py --copies 2 +``` + +Windows one-click helpers are also available: + +```powershell +.\scripts\start_internal_webapp.ps1 +.\scripts\check_internal_webapp.ps1 +.\scripts\stop_internal_webapp.ps1 +.\scripts\smoke_internal_webapp.ps1 --copies 2 +``` + +The smoke flow submits jobs through the same HTTP workflow used by the UI, carries the active SAM backend settings into the temporary service environment, and verifies that the queue reaches a completed result with generated artifacts. + ## ๐Ÿ“Š Evaluation Please refer to the [evaluation documentation](docs/EVAL.md) for details. diff --git a/docs/superpowers/plans/2026-03-26-matanyone2-internal-web-app-plan.md b/docs/superpowers/plans/2026-03-26-matanyone2-internal-web-app-plan.md new file mode 100644 index 0000000..6e2f632 --- /dev/null +++ b/docs/superpowers/plans/2026-03-26-matanyone2-internal-web-app-plan.md @@ -0,0 +1,726 @@ +# MatAnyone2 Internal Web App Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a single-machine internal web app that supports upload, template-frame target selection, queued MatAnyone2 processing, and transparent result download for short clips. + +**Architecture:** Add a new `matanyone2.webapp` package inside the existing repository. Keep the current Hugging Face demo as reference-only code, extract reusable masking and inference behaviors into service modules, and run GPU inference/export in a separate worker process backed by SQLite job persistence and per-job runtime directories. + +**Tech Stack:** Python 3.10+, FastAPI, Jinja2 templates, vanilla JavaScript, sqlite3, ffmpeg, pytest, existing MatAnyone2 inference code, existing SAM interaction code from `hugging_face/tools` + +--- + +## File Structure + +### Modify + +- `D:\my_app\matanyone2\pyproject.toml` + Add runtime/test dependencies for the internal web app and expose optional scripts if desired. +- `D:\my_app\matanyone2\README.md` + Add a short internal-web-app section with run commands after implementation is complete. + +### Create + +- `D:\my_app\matanyone2\matanyone2\webapp\__init__.py` + Package marker. +- `D:\my_app\matanyone2\matanyone2\webapp\config.py` + Environment-driven settings object for runtime paths, size limits, queue policy, and export toggles. +- `D:\my_app\matanyone2\matanyone2\webapp\runtime_paths.py` + Helpers for creating and resolving runtime directories for jobs, drafts, uploads, and exports. +- `D:\my_app\matanyone2\matanyone2\webapp\db.py` + SQLite schema initialization and connection helpers. +- `D:\my_app\matanyone2\matanyone2\webapp\models.py` + Dataclasses and typed enums for job status, draft payloads, and export outcomes. +- `D:\my_app\matanyone2\matanyone2\webapp\repository.py` + CRUD operations for jobs and queue ordering. +- `D:\my_app\matanyone2\matanyone2\webapp\queue.py` + Single-worker queue coordinator and restart recovery logic. +- `D:\my_app\matanyone2\matanyone2\webapp\worker.py` + Worker loop that consumes queued jobs and drives inference/export. +- `D:\my_app\matanyone2\matanyone2\webapp\services\__init__.py` + Services package marker. +- `D:\my_app\matanyone2\matanyone2\webapp\services\video.py` + Upload staging, metadata extraction, frame extraction, and draft creation helpers. +- `D:\my_app\matanyone2\matanyone2\webapp\services\masking.py` + SAM-backed click refinement, multi-mask merge, and mask persistence logic. +- `D:\my_app\matanyone2\matanyone2\webapp\services\inference.py` + MatAnyone2 wrapper that accepts a job payload and writes foreground/alpha intermediates. +- `D:\my_app\matanyone2\matanyone2\webapp\services\export.py` + RGBA PNG sequence generation, zip packaging, and best-effort ProRes export. +- `D:\my_app\matanyone2\matanyone2\webapp\api\__init__.py` + API package marker. +- `D:\my_app\matanyone2\matanyone2\webapp\api\app.py` + FastAPI app factory and startup wiring. +- `D:\my_app\matanyone2\matanyone2\webapp\api\dependencies.py` + Shared dependency providers for settings, repository, queue, and services. +- `D:\my_app\matanyone2\matanyone2\webapp\api\routes\__init__.py` + Routes package marker. +- `D:\my_app\matanyone2\matanyone2\webapp\api\routes\pages.py` + HTML pages for upload, annotation, and job status. +- `D:\my_app\matanyone2\matanyone2\webapp\api\routes\uploads.py` + Video upload, draft creation, and template-frame metadata endpoints. +- `D:\my_app\matanyone2\matanyone2\webapp\api\routes\annotation.py` + Click-refinement, mask merge, and submission endpoints. +- `D:\my_app\matanyone2\matanyone2\webapp\api\routes\jobs.py` + Job status JSON endpoints and artifact download endpoints. +- `D:\my_app\matanyone2\matanyone2\webapp\templates\base.html` + Shared layout. +- `D:\my_app\matanyone2\matanyone2\webapp\templates\upload.html` + Upload step UI. +- `D:\my_app\matanyone2\matanyone2\webapp\templates\annotate.html` + Annotation step UI with template frame, controls, and queue submit button. +- `D:\my_app\matanyone2\matanyone2\webapp\templates\job.html` + Job status and download UI. +- `D:\my_app\matanyone2\matanyone2\webapp\static\styles.css` + Minimal UI styling. +- `D:\my_app\matanyone2\matanyone2\webapp\static\annotator.js` + Client-side click capture, mask preview refresh, and polling. +- `D:\my_app\matanyone2\scripts\run_internal_webapp.py` + App startup entry point. +- `D:\my_app\matanyone2\scripts\run_internal_worker.py` + Worker startup entry point. +- `D:\my_app\matanyone2\tests\webapp\conftest.py` + Shared fixtures for temp runtime directories, fake settings, and app factory helpers. +- `D:\my_app\matanyone2\tests\webapp\test_app_factory.py` + App-factory and startup smoke tests. +- `D:\my_app\matanyone2\tests\webapp\test_repository.py` + SQLite repository and queue ordering tests. +- `D:\my_app\matanyone2\tests\webapp\test_video_service.py` + Draft creation and metadata extraction tests. +- `D:\my_app\matanyone2\tests\webapp\test_masking_service.py` + Click refinement and mask merge tests with SAM monkeypatching. +- `D:\my_app\matanyone2\tests\webapp\test_inference_service.py` + Inference wrapper tests with processor monkeypatching. +- `D:\my_app\matanyone2\tests\webapp\test_export_service.py` + RGBA PNG, zip, and warning-path export tests. +- `D:\my_app\matanyone2\tests\webapp\test_worker.py` + Worker state transition and restart recovery tests. +- `D:\my_app\matanyone2\tests\webapp\test_api_flow.py` + End-to-end request flow tests with fake inference/export services. + +## Task 1: Bootstrap The Web App Package + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\__init__.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\config.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\api\__init__.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\api\app.py` +- Create: `D:\my_app\matanyone2\scripts\run_internal_webapp.py` +- Create: `D:\my_app\matanyone2\tests\webapp\conftest.py` +- Create: `D:\my_app\matanyone2\tests\webapp\test_app_factory.py` +- Modify: `D:\my_app\matanyone2\pyproject.toml` + +- [ ] **Step 1: Write the failing smoke test for settings and app factory** + +```python +from fastapi.testclient import TestClient + +from matanyone2.webapp.api.app import create_app +from matanyone2.webapp.config import WebAppSettings + + +def test_create_app_builds_health_route(tmp_path, monkeypatch): + monkeypatch.setenv("MATANYONE2_WEBAPP_RUNTIME_ROOT", str(tmp_path)) + settings = WebAppSettings() + app = create_app(settings=settings) + + with TestClient(app) as client: + response = client.get("/healthz") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} +``` + +- [ ] **Step 2: Run the smoke test to verify it fails** + +Run: `python -m pytest tests\webapp\test_app_factory.py -q` +Expected: FAIL with `ModuleNotFoundError` or missing `create_app`. + +- [ ] **Step 3: Implement settings, app factory, and startup script** + +```python +# matanyone2/webapp/config.py +from dataclasses import dataclass +from pathlib import Path +import os + + +@dataclass(slots=True) +class WebAppSettings: + runtime_root: Path = Path(os.getenv("MATANYONE2_WEBAPP_RUNTIME_ROOT", "runtime/webapp")) + database_path: Path = Path(os.getenv("MATANYONE2_WEBAPP_DATABASE_PATH", "runtime/webapp/jobs.db")) + max_video_seconds: int = int(os.getenv("MATANYONE2_WEBAPP_MAX_VIDEO_SECONDS", "10")) + max_upload_bytes: int = int(os.getenv("MATANYONE2_WEBAPP_MAX_UPLOAD_BYTES", str(2 * 1024 * 1024 * 1024))) + enable_prores_export: bool = os.getenv("MATANYONE2_WEBAPP_ENABLE_PRORES", "1") == "1" + + +# matanyone2/webapp/api/app.py +from fastapi import FastAPI + + +def create_app(settings=None) -> FastAPI: + app = FastAPI(title="MatAnyone2 Internal Web App") + app.state.settings = settings + + @app.get("/healthz") + def healthcheck(): + return {"status": "ok"} + + return app +``` + +- [ ] **Step 4: Update dependencies and rerun the smoke test** + +Add to `pyproject.toml` dependencies: + +```toml +"fastapi>=0.111,<1.0", +"uvicorn>=0.30,<1.0", +"jinja2>=3.1,<4.0", +"python-multipart>=0.0.9,<1.0", +"httpx>=0.27,<1.0", +"pytest>=8.0,<9.0", +``` + +Run: `python -m pytest tests\webapp\test_app_factory.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the bootstrap** + +```bash +git add pyproject.toml matanyone2/webapp scripts/run_internal_webapp.py tests/webapp +git commit -m "feat: bootstrap internal web app package" +``` + +## Task 2: Add Persistent Runtime Paths And SQLite Job Storage + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\runtime_paths.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\db.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\models.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\repository.py` +- Create: `D:\my_app\matanyone2\tests\webapp\test_repository.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\api\app.py` + +- [ ] **Step 1: Write failing repository tests for create, update, and queue order** + +```python +from matanyone2.webapp.models import JobStatus +from matanyone2.webapp.repository import JobRepository + + +def test_repository_creates_job_and_reports_queue_position(tmp_path): + repo = JobRepository.from_path(tmp_path / "jobs.db") + first = repo.create_job(source_video_path="a.mp4", template_frame_index=0, mask_path="a.png", params_json="{}") + second = repo.create_job(source_video_path="b.mp4", template_frame_index=0, mask_path="b.png", params_json="{}") + + assert repo.get_job(first.job_id).status is JobStatus.QUEUED + assert repo.get_queue_position(second.job_id) == 2 +``` + +- [ ] **Step 2: Run the repository tests to verify they fail** + +Run: `python -m pytest tests\webapp\test_repository.py -q` +Expected: FAIL because the repository and models do not exist yet. + +- [ ] **Step 3: Implement SQLite schema, typed statuses, and runtime-path helpers** + +```python +from dataclasses import dataclass +from enum import StrEnum + + +class JobStatus(StrEnum): + QUEUED = "queued" + PREPARING = "preparing" + RUNNING = "running" + EXPORTING = "exporting" + COMPLETED = "completed" + COMPLETED_WITH_WARNING = "completed_with_warning" + FAILED = "failed" + INTERRUPTED = "interrupted" + + +@dataclass(slots=True) +class JobRecord: + job_id: str + status: JobStatus + source_video_path: str + mask_path: str + template_frame_index: int + params_json: str + warning_text: str | None = None + error_text: str | None = None +``` + +- [ ] **Step 4: Wire database initialization into app startup and rerun tests** + +Run: `python -m pytest tests\webapp\test_repository.py tests\webapp\test_app_factory.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the persistence layer** + +```bash +git add matanyone2/webapp/runtime_paths.py matanyone2/webapp/db.py matanyone2/webapp/models.py matanyone2/webapp/repository.py matanyone2/webapp/api/app.py tests/webapp/test_repository.py +git commit -m "feat: add persistent job repository" +``` + +## Task 3: Add Queue Coordination And Worker Recovery + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\queue.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\worker.py` +- Create: `D:\my_app\matanyone2\scripts\run_internal_worker.py` +- Create: `D:\my_app\matanyone2\tests\webapp\test_worker.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\api\app.py` + +- [ ] **Step 1: Write failing queue and restart-recovery tests** + +```python +from matanyone2.webapp.models import JobStatus +from matanyone2.webapp.queue import QueueCoordinator +from matanyone2.webapp.repository import JobRepository + + +def test_recover_running_jobs_marks_them_interrupted(tmp_path): + repo = JobRepository.from_path(tmp_path / "jobs.db") + job = repo.create_job(source_video_path="a.mp4", template_frame_index=0, mask_path="a.png", params_json="{}") + repo.update_status(job.job_id, JobStatus.RUNNING) + + coordinator = QueueCoordinator(repo) + coordinator.recover_interrupted_jobs() + + assert repo.get_job(job.job_id).status is JobStatus.INTERRUPTED +``` + +- [ ] **Step 2: Run the worker tests to verify they fail** + +Run: `python -m pytest tests\webapp\test_worker.py -q` +Expected: FAIL because queue coordination does not exist yet. + +- [ ] **Step 3: Implement queue coordinator, worker loop, and script entry point** + +```python +class QueueCoordinator: + def __init__(self, repository): + self.repository = repository + + def recover_interrupted_jobs(self) -> None: + self.repository.mark_running_jobs_interrupted() + + def next_job_id(self) -> str | None: + next_job = self.repository.next_queued_job() + return None if next_job is None else next_job.job_id +``` + +- [ ] **Step 4: Rerun worker tests and verify app startup triggers recovery** + +Run: `python -m pytest tests\webapp\test_worker.py tests\webapp\test_app_factory.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the queue layer** + +```bash +git add matanyone2/webapp/queue.py matanyone2/webapp/worker.py scripts/run_internal_worker.py matanyone2/webapp/api/app.py tests/webapp/test_worker.py +git commit -m "feat: add queue coordination and worker recovery" +``` + +## Task 4: Add Video Draft Creation And Upload Validation + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\services\video.py` +- Create: `D:\my_app\matanyone2\tests\webapp\test_video_service.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\models.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\runtime_paths.py` + +- [ ] **Step 1: Write failing tests for draft creation and duration limits** + +```python +from pathlib import Path + +from matanyone2.webapp.services.video import VideoDraftService + + +def test_create_draft_extracts_template_frame_and_metadata(tmp_path, sample_video_path): + service = VideoDraftService(runtime_root=tmp_path, max_video_seconds=10, max_upload_bytes=10_000_000) + draft = service.create_draft(Path(sample_video_path)) + + assert draft.frame_count > 0 + assert draft.template_frame_path.exists() + assert draft.duration_seconds <= 10 +``` + +- [ ] **Step 2: Run the draft tests to verify they fail** + +Run: `python -m pytest tests\webapp\test_video_service.py -q` +Expected: FAIL because `VideoDraftService` does not exist yet. + +- [ ] **Step 3: Implement upload staging, metadata extraction, and template-frame image output** + +```python +@dataclass(slots=True) +class DraftRecord: + draft_id: str + video_path: Path + template_frame_path: Path + width: int + height: int + fps: float + frame_count: int + duration_seconds: float +``` + +- [ ] **Step 4: Rerun draft tests with a generated short sample video fixture** + +Run: `python -m pytest tests\webapp\test_video_service.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the draft service** + +```bash +git add matanyone2/webapp/services/video.py matanyone2/webapp/models.py matanyone2/webapp/runtime_paths.py tests/webapp/test_video_service.py tests/webapp/conftest.py +git commit -m "feat: add upload draft and video validation service" +``` + +## Task 5: Extract SAM Masking And Multi-Mask Merge Into A Service + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\services\masking.py` +- Create: `D:\my_app\matanyone2\tests\webapp\test_masking_service.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\models.py` + +- [ ] **Step 1: Write failing tests for click refinement and merged mask output** + +```python +import numpy as np + +from matanyone2.webapp.services.masking import merge_masks + + +def test_merge_masks_collapses_multiple_targets_into_single_uint8_mask(): + mask_a = np.array([[1, 0], [0, 0]], dtype=np.uint8) + mask_b = np.array([[0, 0], [1, 0]], dtype=np.uint8) + + merged = merge_masks([mask_a, mask_b]) + + assert merged.dtype == np.uint8 + assert merged.tolist() == [[255, 0], [255, 0]] +``` + +- [ ] **Step 2: Run the masking tests to verify they fail** + +Run: `python -m pytest tests\webapp\test_masking_service.py -q` +Expected: FAIL because the masking service does not exist yet. + +- [ ] **Step 3: Implement a service that wraps current SAM interaction utilities** + +```python +from hugging_face.tools.interact_tools import SamControler + + +def merge_masks(masks: list[np.ndarray]) -> np.ndarray: + if not masks: + raise ValueError("at least one mask is required") + merged = np.zeros_like(masks[0], dtype=np.uint8) + for mask in masks: + merged = np.where(mask > 0, 255, merged).astype(np.uint8) + return merged +``` + +- [ ] **Step 4: Add monkeypatched refinement tests and rerun** + +Run: `python -m pytest tests\webapp\test_masking_service.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the masking service** + +```bash +git add matanyone2/webapp/services/masking.py matanyone2/webapp/models.py tests/webapp/test_masking_service.py +git commit -m "feat: extract masking refinement service" +``` + +## Task 6: Wrap MatAnyone2 Inference For Job Execution + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\services\inference.py` +- Create: `D:\my_app\matanyone2\tests\webapp\test_inference_service.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\models.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\worker.py` + +- [ ] **Step 1: Write failing tests for inference output paths and state transitions** + +```python +from pathlib import Path + +from matanyone2.webapp.services.inference import InferenceService + + +def test_run_job_writes_foreground_and_alpha_outputs(tmp_path, monkeypatch): + service = InferenceService(model_name="MatAnyone 2") + job_dir = tmp_path / "job-1" + job_dir.mkdir() + + monkeypatch.setattr(service, "_run_model", lambda **_: (Path(job_dir / "foreground.mp4"), Path(job_dir / "alpha.mp4"))) + result = service.run_job(source_video_path=Path("input.mp4"), mask_path=Path("mask.png"), job_dir=job_dir, template_frame_index=0) + + assert result.foreground_video_path.name == "foreground.mp4" + assert result.alpha_video_path.name == "alpha.mp4" +``` + +- [ ] **Step 2: Run the inference tests to verify they fail** + +Run: `python -m pytest tests\webapp\test_inference_service.py -q` +Expected: FAIL because the inference service does not exist yet. + +- [ ] **Step 3: Implement a thin wrapper around the current repository inference path** + +```python +@dataclass(slots=True) +class InferenceResult: + foreground_video_path: Path + alpha_video_path: Path + + +class InferenceService: + def run_job(self, source_video_path: Path, mask_path: Path, job_dir: Path, template_frame_index: int) -> InferenceResult: + foreground_path = job_dir / "foreground.mp4" + alpha_path = job_dir / "alpha.mp4" + self._run_model( + source_video_path=source_video_path, + mask_path=mask_path, + foreground_path=foreground_path, + alpha_path=alpha_path, + template_frame_index=template_frame_index, + ) + return InferenceResult(foreground_video_path=foreground_path, alpha_video_path=alpha_path) +``` + +- [ ] **Step 4: Integrate the inference wrapper into the worker and rerun tests** + +Run: `python -m pytest tests\webapp\test_inference_service.py tests\webapp\test_worker.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the inference service** + +```bash +git add matanyone2/webapp/services/inference.py matanyone2/webapp/models.py matanyone2/webapp/worker.py tests/webapp/test_inference_service.py +git commit -m "feat: wrap matanyone2 inference for queued jobs" +``` + +## Task 7: Add Transparent Export Packaging + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\services\export.py` +- Create: `D:\my_app\matanyone2\tests\webapp\test_export_service.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\models.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\worker.py` + +- [ ] **Step 1: Write failing tests for RGBA PNG zip success and ProRes warning fallback** + +```python +from pathlib import Path + +from matanyone2.webapp.services.export import ExportService + + +def test_export_assets_creates_png_zip_even_when_prores_fails(tmp_path, monkeypatch): + service = ExportService(enable_prores=True) + foreground = tmp_path / "foreground.mp4" + alpha = tmp_path / "alpha.mp4" + foreground.write_bytes(b"fg") + alpha.write_bytes(b"a") + + monkeypatch.setattr(service, "_extract_frames", lambda *args, **kwargs: ([tmp_path / "fg-0001.png"], [tmp_path / "a-0001.png"])) + monkeypatch.setattr(service, "_write_rgba_pngs", lambda *args, **kwargs: tmp_path / "rgba_png") + monkeypatch.setattr(service, "_zip_directory", lambda *args, **kwargs: tmp_path / "rgba_png.zip") + monkeypatch.setattr(service, "_export_prores", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("ffmpeg failed"))) + + result = service.export_assets(foreground, alpha, tmp_path) + + assert result.png_zip_path.name == "rgba_png.zip" + assert result.warning_text == "ffmpeg failed" +``` + +- [ ] **Step 2: Run the export tests to verify they fail** + +Run: `python -m pytest tests\webapp\test_export_service.py -q` +Expected: FAIL because the export service does not exist yet. + +- [ ] **Step 3: Implement RGBA composition, zip packaging, and warning-path ProRes export** + +```python +@dataclass(slots=True) +class ExportResult: + rgba_png_dir: Path + png_zip_path: Path + prores_path: Path | None + warning_text: str | None + + +def compose_rgba_frame(foreground_rgb: np.ndarray, alpha_gray: np.ndarray) -> Image.Image: + rgba = np.dstack([foreground_rgb, alpha_gray]).astype(np.uint8) + return Image.fromarray(rgba, mode="RGBA") +``` + +- [ ] **Step 4: Update worker final-status logic and rerun export tests** + +Run: `python -m pytest tests\webapp\test_export_service.py tests\webapp\test_worker.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the export layer** + +```bash +git add matanyone2/webapp/services/export.py matanyone2/webapp/models.py matanyone2/webapp/worker.py tests/webapp/test_export_service.py +git commit -m "feat: add transparent export packaging" +``` + +## Task 8: Build The Upload, Annotation, Job, And Download Web Flow + +**Files:** +- Create: `D:\my_app\matanyone2\matanyone2\webapp\api\dependencies.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\api\routes\pages.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\api\routes\uploads.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\api\routes\annotation.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\api\routes\jobs.py` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\templates\base.html` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\templates\upload.html` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\templates\annotate.html` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\templates\job.html` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\static\styles.css` +- Create: `D:\my_app\matanyone2\matanyone2\webapp\static\annotator.js` +- Create: `D:\my_app\matanyone2\tests\webapp\test_api_flow.py` +- Modify: `D:\my_app\matanyone2\matanyone2\webapp\api\app.py` + +- [ ] **Step 1: Write failing API flow tests for upload, annotate, submit, and status** + +```python +from fastapi.testclient import TestClient + + +def test_submit_flow_returns_job_page(app_client: TestClient, sample_video_upload): + upload_response = app_client.post("/api/uploads", files={"video": sample_video_upload}) + assert upload_response.status_code == 200 + draft_id = upload_response.json()["draft_id"] + + annotate_response = app_client.post( + f"/api/drafts/{draft_id}/submit", + json={"template_frame_index": 0, "selected_masks": ["mask_001"]}, + ) + + assert annotate_response.status_code == 200 + assert annotate_response.json()["status"] == "queued" +``` + +- [ ] **Step 2: Run the API tests to verify they fail** + +Run: `python -m pytest tests\webapp\test_api_flow.py -q` +Expected: FAIL because routes and templates do not exist yet. + +- [ ] **Step 3: Implement the three-page web flow and JSON endpoints** + +```python +router = APIRouter() + + +@router.post("/api/uploads") +async def upload_video(video: UploadFile, video_service=Depends(get_video_service)): + draft = await video_service.create_draft_from_upload(video) + return {"draft_id": draft.draft_id, "template_frame_url": f"/api/drafts/{draft.draft_id}/template-frame"} + + +@router.get("/jobs/{job_id}") +def job_page(job_id: str, request: Request, repository=Depends(get_repository)): + job = repository.get_job(job_id) + return templates.TemplateResponse("job.html", {"request": request, "job": job}) +``` + +- [ ] **Step 4: Rerun API tests with fake services monkeypatched into dependencies** + +Run: `python -m pytest tests\webapp\test_api_flow.py tests\webapp\test_app_factory.py -q` +Expected: PASS + +- [ ] **Step 5: Commit the web flow** + +```bash +git add matanyone2/webapp/api matanyone2/webapp/templates matanyone2/webapp/static tests/webapp/test_api_flow.py +git commit -m "feat: add internal web upload and job flow" +``` + +## Task 9: Finish End-To-End Verification And Operator Documentation + +**Files:** +- Modify: `D:\my_app\matanyone2\tests\webapp\conftest.py` +- Modify: `D:\my_app\matanyone2\tests\webapp\test_worker.py` +- Modify: `D:\my_app\matanyone2\tests\webapp\test_api_flow.py` +- Modify: `D:\my_app\matanyone2\README.md` + +- [ ] **Step 1: Add failing tests for restart behavior and queue handoff** + +```python +def test_second_job_waits_until_first_job_finishes(app_client, seeded_jobs): + first_job_id, second_job_id = seeded_jobs + + first_status = app_client.get(f"/api/jobs/{first_job_id}").json() + second_status = app_client.get(f"/api/jobs/{second_job_id}").json() + + assert first_status["status"] == "running" + assert second_status["status"] == "queued" + assert second_status["queue_position"] == 1 +``` + +- [ ] **Step 2: Run the full webapp test suite to verify the new checks fail** + +Run: `python -m pytest tests\webapp -q` +Expected: FAIL on the newly added restart or queue handoff assertions. + +- [ ] **Step 3: Implement the missing restart/status details and update operator docs** + +````markdown +## Internal Web App + +Run the web server: + +```shell +python scripts/run_internal_webapp.py +``` + +Run the worker in a separate process: + +```shell +python scripts/run_internal_worker.py +``` +```` + +- [ ] **Step 4: Run the full webapp suite and a syntax smoke check** + +Run: `python -m pytest tests\webapp -q` +Expected: PASS + +Run: `python -m compileall matanyone2\webapp scripts\run_internal_webapp.py scripts\run_internal_worker.py` +Expected: PASS with no syntax errors + +- [ ] **Step 5: Commit the final plan deliverable** + +```bash +git add tests/webapp README.md +git commit -m "test: cover internal web app flow and docs" +``` + +## Manual Verification Checklist + +- Upload a real short clip from `D:\my_app\matanyone2\test_sample`. +- Select a non-zero template frame and verify the displayed frame changes. +- Add positive and negative clicks, save two masks, and submit a merged multi-target job. +- Submit a second job while the first is running and confirm the second shows queued status. +- Confirm the completed job exposes `foreground.mp4`, `alpha.mp4`, `rgba_png.zip`, and optionally `output_prores4444.mov`. +- Force a ProRes export failure and confirm the job lands in `completed_with_warning`. +- Restart the web process while a queued job exists and confirm it remains queued. +- Restart during a running job and confirm the old job becomes `interrupted`. + +## Review Notes For The Implementer + +- Keep `hugging_face/app.py` unchanged unless a small shared extraction is clearly lower-risk than duplication. +- Prefer monkeypatched lightweight tests over GPU-backed tests inside `pytest`. +- Treat RGBA PNG zip as the primary artifact; do not make ProRes success a hard requirement. +- Keep queue semantics simple: one worker, one running job, FIFO ordering. +- Avoid premature streaming refactors in version 1. Wrap the current memory-heavy inference path first, then optimize later. diff --git a/docs/superpowers/plans/2026-03-27-matanyone2-internal-webapp-ui-redesign-plan.md b/docs/superpowers/plans/2026-03-27-matanyone2-internal-webapp-ui-redesign-plan.md new file mode 100644 index 0000000..21b1e15 --- /dev/null +++ b/docs/superpowers/plans/2026-03-27-matanyone2-internal-webapp-ui-redesign-plan.md @@ -0,0 +1,495 @@ +# MatAnyone2 Internal Web App UI Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Rebuild the internal MatAnyone2 web app into a desktop-first post-production workbench with a professional upload flow, a staged multi-target annotation experience, and a preview-first results page. + +**Architecture:** Keep the existing FastAPI + Jinja2 + vanilla JavaScript stack, but refactor the browser layer around a dedicated workbench state model instead of a single accumulating click session. Treat the redesign as a product and interaction rewrite on top of the current backend pipeline: fix the known API/state bugs first, then rebuild the upload shell, annotation workbench, and results review page with richer JSON contracts and page-specific controllers. + +**Tech Stack:** Python 3.10+, FastAPI, Jinja2 templates, vanilla JavaScript modules, CSS custom properties, SQLite, pytest, Playwright smoke flow, existing SAM-backed masking service, existing internal webapp scripts + +--- + +## File Structure + +### Modify + +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\models.py` + Expand annotation state from one flat click session into explicit target layers, active tool metadata, and richer page/result view models. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\services\masking.py` + Reset click state between saved targets, add layer-aware preview helpers, and persist independent target metadata for the workbench. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\api\routes\uploads.py` + Convert validation failures into user-facing 4xx responses and return richer upload metadata needed by the redesigned upload page. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\api\routes\annotation.py` + Replace the current thin click/save/submit contract with a richer workbench API for stage changes, active target selection, and target-aware previews. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\api\routes\pages.py` + Return 404 for missing jobs and pass richer template context for the redesigned upload, annotation, and results pages. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\base.html` + Introduce the desktop tool shell, status rail, font loading, and script/module entrypoints. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\upload.html` + Replace the plain upload form with the `New Session` layout and system/media summary cards. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\annotate.html` + Replace the current demo-like page with the three-column workbench layout, stage switcher, layer panel, and inspector panel. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\job.html` + Replace the link list page with a preview-first review surface that surfaces artifacts, warnings, and re-entry to annotation. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\styles.css` + Rebuild the visual system using CSS variables, workstation panels, canvas layouts, and preview states. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_api_flow.py` + Extend request-level coverage for upload validation, richer page markup, layer actions, and result page contracts. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_masking_service.py` + Cover independent target saves, active-target resets, and merged multi-target export behavior. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_smoke.py` + Update smoke assertions for the redesigned page flow and richer workbench/result states. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\README.md` + Document the redesigned UI workflow, keyboard controls, and updated smoke/launch expectations. + +### Create + +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\shared.js` + Shared browser helpers for JSON fetches, status messaging, DOM utilities, and cache-busting. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\upload.js` + Upload-page controller for drag-and-drop, file metadata display, and transition into the workbench. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\workbench.js` + Annotation workbench controller for stages, tool modes, target layers, canvas interactions, and submission. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\results.js` + Results-page controller for polling, preview mode switching, artifact rendering, and warning presentation. +- `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_page_templates.py` + Focused assertions for the new upload, workbench, and review HTML shells. + +## Task 1: Fix Review Blockers And Establish Layer-Aware Annotation State + +**Files:** +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\models.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\services\masking.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\api\routes\uploads.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\api\routes\pages.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_masking_service.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_api_flow.py` + +- [ ] **Step 1: Write failing tests for the three known blockers** + +```python +def test_upload_validation_errors_return_400(app_client): + response = app_client.post( + "/api/uploads", + files={"video": ("broken.mp4", b"not-a-video", "video/mp4")}, + ) + + assert response.status_code == 400 + assert response.json()["detail"] == "unable to read video frames" + + +def test_saved_masks_start_with_fresh_click_state(tmp_path): + session = service.create_session(draft) + service.apply_click(session, x=1, y=1, positive=True) + service.save_current_mask(session) + service.apply_click(session, x=8, y=8, positive=True) + + assert session.active_target.click_points == [(8, 8)] + + +def test_missing_job_page_returns_404(app_client): + response = app_client.get("/jobs/missing-job") + + assert response.status_code == 404 +``` + +- [ ] **Step 2: Run only the blocker-focused tests** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_masking_service.py tests\webapp\test_api_flow.py -q` + +Expected: FAIL for upload validation, independent target clicks, and missing job 404 behavior. + +- [ ] **Step 3: Refactor the session model and route error handling** + +```python +@dataclass(slots=True) +class AnnotationTarget: + name: str + click_points: list[tuple[int, int]] = field(default_factory=list) + click_labels: list[int] = field(default_factory=list) + mask_path: Path | None = None + preview_path: Path | None = None + + +def upload_video(...): + try: + draft = video_service.create_draft_from_upload(video) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +def job_page(...): + try: + job = repository.get_job(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="job not found") from exc +``` + +Implementation notes: +- Replace the single `click_points` / `click_labels` fields on `DraftSession` with an explicit active target record. +- `save_current_mask()` must persist the current target and then reset the active target click state before the next target starts. +- Keep merged export behavior unchanged for now: multiple targets still combine into one merged mask when submitted. + +- [ ] **Step 4: Re-run the focused tests and then the full webapp suite** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp -q` + +Expected: PASS, with the blocker regressions covered and the prior suite still green. + +- [ ] **Step 5: Commit the stabilization pass** + +```bash +git add matanyone2/webapp/models.py matanyone2/webapp/services/masking.py matanyone2/webapp/api/routes/uploads.py matanyone2/webapp/api/routes/pages.py tests/webapp/test_masking_service.py tests/webapp/test_api_flow.py +git commit -m "fix: stabilize webapp review blockers" +``` + +## Task 2: Introduce Rich Workbench API Contracts + +**Files:** +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\models.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\services\masking.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\api\routes\annotation.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_api_flow.py` + +- [ ] **Step 1: Write failing tests for workbench-state endpoints** + +```python +def test_annotation_page_exposes_workbench_contract(app_client, sample_video_upload): + draft_id = app_client.post("/api/uploads", files={"video": sample_video_upload}).json()["draft_id"] + + response = app_client.get(f"/drafts/{draft_id}/annotate") + + assert 'data-workbench-endpoint="/api/drafts/' in response.text + assert 'data-targets-endpoint="/api/drafts/' in response.text + + +def test_target_creation_and_selection_round_trip(app_client, sample_video_upload): + draft_id = app_client.post("/api/uploads", files={"video": sample_video_upload}).json()["draft_id"] + + created = app_client.post(f"/api/drafts/{draft_id}/targets", json={"name": "Hero"}).json() + selected = app_client.post( + f"/api/drafts/{draft_id}/targets/{created['target_id']}/select" + ).json() + + assert created["name"] == "Hero" + assert selected["active_target_id"] == created["target_id"] +``` + +- [ ] **Step 2: Run the API-flow tests to confirm the current contract is insufficient** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_api_flow.py -q` + +Expected: FAIL because the workbench endpoints and target-selection JSON do not exist yet. + +- [ ] **Step 3: Add workbench JSON models and endpoints** + +```python +class DraftTargetPayload(BaseModel): + target_id: str + name: str + point_count: int + visible: bool + locked: bool + + +@router.get("/api/drafts/{draft_id}") +def get_workbench_state(...): + return { + "draft_id": draft_id, + "stage": session.stage, + "active_target_id": session.active_target_id, + "targets": [...], + } + + +@router.post("/api/drafts/{draft_id}/targets") +def create_target(...): + ... + + +@router.post("/api/drafts/{draft_id}/targets/{target_id}/select") +def select_target(...): + ... +``` + +Implementation notes: +- Keep the current click/save/submit endpoints, but change their responses to include refreshed workbench state so the browser no longer has to infer state from the DOM. +- Add stage-change support even if the first implementation only records `coarse`, `refine`, and `preview` without algorithmic differences. +- Every target returned to the browser must carry stable ids, names, visibility, and selection state. + +- [ ] **Step 4: Re-run the API tests and a smoke subset** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_api_flow.py tests\webapp\test_smoke.py -q` + +Expected: PASS, proving the new workbench contract is available without breaking smoke expectations. + +- [ ] **Step 5: Commit the workbench API layer** + +```bash +git add matanyone2/webapp/models.py matanyone2/webapp/services/masking.py matanyone2/webapp/api/routes/annotation.py tests/webapp/test_api_flow.py +git commit -m "feat: add workbench annotation state api" +``` + +## Task 3: Rebuild The Shared Shell And Upload Page + +**Files:** +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\base.html` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\upload.html` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\styles.css` +- Create: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\shared.js` +- Create: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\upload.js` +- Create: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_page_templates.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_api_flow.py` + +- [ ] **Step 1: Write failing template tests for the new upload shell** + +```python +def test_upload_page_renders_new_session_shell(app_client): + response = app_client.get("/") + + assert 'class="app-shell"' in response.text + assert 'data-page="upload"' in response.text + assert 'id="dropzone-panel"' in response.text + assert 'id="media-info-card"' in response.text +``` + +- [ ] **Step 2: Run the upload-page tests and confirm the old markup fails** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_page_templates.py tests\webapp\test_api_flow.py -q` + +Expected: FAIL because the current upload page still renders a plain form and loads the legacy monolithic script. + +- [ ] **Step 3: Implement the shell, design tokens, and upload controller** + +```html + +
+
...
+
...
+
+ + +``` + +```js +export function bindUploadPage(root) { + const fileInput = root.querySelector("#video-file"); + const infoCard = root.querySelector("#media-info-card"); + ... +} +``` + +Implementation notes: +- Use CSS variables for the workstation palette, panel elevations, spacing, and semantic colors before styling any page-specific components. +- The upload page must expose system readiness, file metadata, and output expectations in three clear panels. +- Keep the upload action progressive: file selection updates metadata immediately, the primary CTA transitions into the annotation workbench after the draft is created. + +- [ ] **Step 4: Re-run page/template tests plus the upload flow test** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_page_templates.py tests\webapp\test_api_flow.py -q` + +Expected: PASS with the new upload shell and upload-to-annotate transition intact. + +- [ ] **Step 5: Commit the shell and upload redesign** + +```bash +git add matanyone2/webapp/templates/base.html matanyone2/webapp/templates/upload.html matanyone2/webapp/static/styles.css matanyone2/webapp/static/shared.js matanyone2/webapp/static/upload.js tests/webapp/test_page_templates.py tests/webapp/test_api_flow.py +git commit -m "feat: redesign internal webapp upload shell" +``` + +## Task 4: Build The Annotation Workbench UI + +**Files:** +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\annotate.html` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\styles.css` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\shared.js` +- Create: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\workbench.js` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_page_templates.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_api_flow.py` + +- [ ] **Step 1: Write failing tests for the new workbench structure** + +```python +def test_annotation_page_renders_workbench_layout(app_client, sample_video_upload): + draft_id = app_client.post("/api/uploads", files={"video": sample_video_upload}).json()["draft_id"] + + response = app_client.get(f"/drafts/{draft_id}/annotate") + + assert 'class="workbench-shell"' in response.text + assert 'id="tool-rail"' in response.text + assert 'id="canvas-stage"' in response.text + assert 'id="layer-panel"' in response.text + assert 'id="inspector-panel"' in response.text +``` + +- [ ] **Step 2: Run the annotation template and API tests** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_page_templates.py tests\webapp\test_api_flow.py tests\webapp\test_masking_service.py -q` + +Expected: FAIL because the current page still renders the demo controls and cannot drive layered workbench state. + +- [ ] **Step 3: Implement the three-column workbench and modular browser controller** + +```js +const STAGES = ["coarse", "refine", "preview"]; + +function renderTargets(targets, activeTargetId) { + ... +} + +function setStage(stage) { + state.stage = stage; + root.dataset.stage = stage; +} +``` + +Implementation notes: +- The left rail should own tool mode switches only; do not scatter annotation actions across the page. +- The center column should expose stage tabs, canvas controls, and view modes (`source`, `overlay`, `alpha`) without mixing them into the right inspector. +- The right panel should manage targets, current tool settings, history, and contextual help. +- Default to hiding stale click markers when the stage changes out of coarse mode so the canvas stays readable. + +- [ ] **Step 4: Re-run annotation-focused tests and a browser smoke** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_page_templates.py tests\webapp\test_api_flow.py tests\webapp\test_masking_service.py -q` + +Run: `.\.venv\Scripts\python.exe scripts\smoke_internal_webapp.py --copies 1` + +Expected: PASS for the test suite, and the smoke run reaches the redesigned annotate page and submits a job successfully. + +- [ ] **Step 5: Commit the annotation workbench** + +```bash +git add matanyone2/webapp/templates/annotate.html matanyone2/webapp/static/styles.css matanyone2/webapp/static/shared.js matanyone2/webapp/static/workbench.js tests/webapp/test_page_templates.py tests/webapp/test_api_flow.py tests/webapp/test_masking_service.py +git commit -m "feat: rebuild annotation workbench ui" +``` + +## Task 5: Rebuild The Results Review Page + +**Files:** +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\templates\job.html` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\styles.css` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\shared.js` +- Create: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\matanyone2\webapp\static\results.js` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_page_templates.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_api_flow.py` + +- [ ] **Step 1: Write failing tests for the preview-first results page** + +```python +def test_job_page_renders_review_viewport(app_client): + job = app_client.app.state.repository.create_job( + source_video_path="queued.mp4", + template_frame_index=0, + mask_path="queued.png", + params_json="{}", + ) + + response = app_client.get(f"/jobs/{job.job_id}") + + assert 'id="preview-viewport"' in response.text + assert 'id="preview-mode-tabs"' in response.text + assert 'id="artifact-panel"' in response.text +``` + +- [ ] **Step 2: Run the results-page tests and confirm the old page fails** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_page_templates.py tests\webapp\test_api_flow.py -q` + +Expected: FAIL because the current results page only renders status text and a plain artifact list. + +- [ ] **Step 3: Implement the review layout and results controller** + +```js +const PREVIEW_MODES = ["source", "overlay", "alpha", "foreground"]; + +function renderArtifacts(artifacts) { + ... +} + +function applyStatus(payload) { + statusNode.textContent = payload.status; + warningNode.textContent = payload.warning_text || payload.error_text || ""; +} +``` + +Implementation notes: +- Keep status polling, but move the primary emphasis to the preview surface and mode tabs. +- Surface warnings separately from normal status so `completed_with_warning` reads clearly. +- Preserve the existing artifact download routes; the results page is a redesign of presentation, not the transport layer. + +- [ ] **Step 4: Re-run the results tests and end-to-end smoke** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_page_templates.py tests\webapp\test_api_flow.py tests\webapp\test_smoke.py -q` + +Expected: PASS with the redesigned results page still showing downloads and terminal states correctly. + +- [ ] **Step 5: Commit the review page** + +```bash +git add matanyone2/webapp/templates/job.html matanyone2/webapp/static/styles.css matanyone2/webapp/static/shared.js matanyone2/webapp/static/results.js tests/webapp/test_page_templates.py tests/webapp/test_api_flow.py tests/webapp/test_smoke.py +git commit -m "feat: redesign result review page" +``` + +## Task 6: Refresh Documentation, Smoke Coverage, And Final Verification + +**Files:** +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\README.md` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\tests\webapp\test_smoke.py` +- Modify: `D:\my_app\matanyone2\.worktrees\internal-webapp-ui-rebuild\scripts\smoke_internal_webapp.py` + +- [ ] **Step 1: Write failing tests for the redesigned smoke expectations** + +```python +def test_poll_jobs_accepts_completed_with_warning(): + statuses = poll_jobs(...) + assert statuses["job-1"]["status"] in {"completed", "completed_with_warning"} +``` + +- [ ] **Step 2: Run smoke-related tests before adjusting docs/scripts** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp\test_smoke.py tests\webapp\test_service_scripts.py -q` + +Expected: FAIL or require updates because the redesigned pages expose new selectors, labels, and status expectations. + +- [ ] **Step 3: Update smoke helpers and README for the new UI workflow** + +```markdown +1. Launch the desktop workstation shell with `scripts/start_internal_webapp.ps1`. +2. Create a draft from the upload page. +3. Build one or more targets in the annotation workbench. +4. Review `source / overlay / alpha / foreground` before downloading artifacts. +``` + +Implementation notes: +- Keep the smoke runner CLI stable if possible; prefer extending selectors and assertions over changing the operator-facing command. +- Document the keyboard shortcuts introduced by the workbench (`V`, `P`, `N`, `B`, `E`, `[`, `]`, `Ctrl+Z`). +- Update the README screenshots and wording only after the implementation is verified. + +- [ ] **Step 4: Run the full verification set** + +Run: `.\.venv\Scripts\python.exe -m pytest tests\webapp -q` + +Run: `.\.venv\Scripts\python.exe -m pytest tests\test_inference_utils.py -q` + +Run: `.\.venv\Scripts\python.exe -m compileall matanyone2\webapp scripts\run_internal_webapp.py scripts\run_internal_worker.py scripts\smoke_internal_webapp.py` + +Expected: PASS, with the redesigned UI fully covered and no regressions in the inference utility patch. + +- [ ] **Step 5: Commit docs and smoke updates** + +```bash +git add README.md tests/webapp/test_smoke.py scripts/smoke_internal_webapp.py +git commit -m "docs: update webapp redesign workflow" +``` + +## Manual Review Checklist + +- [ ] Upload page reads like an internal tool entrypoint, not a raw form. +- [ ] Annotation page keeps the canvas visually dominant at 1920x1080. +- [ ] Saving one target does not pollute the next target's clicks. +- [ ] `coarse`, `refine`, and `preview` stages are obvious without reading code. +- [ ] Results page makes `source / overlay / alpha / foreground` easy to compare. +- [ ] Warning states are visually distinct from success states. +- [ ] The smoke flow still completes on the `.venv` CUDA environment. diff --git a/docs/superpowers/specs/2026-03-26-matanyone2-internal-web-app-design.md b/docs/superpowers/specs/2026-03-26-matanyone2-internal-web-app-design.md new file mode 100644 index 0000000..06095c8 --- /dev/null +++ b/docs/superpowers/specs/2026-03-26-matanyone2-internal-web-app-design.md @@ -0,0 +1,496 @@ +# MatAnyone2 Internal Web App Design + +**Date:** 2026-03-26 + +**Status:** Approved for planning + +## Summary + +Build a single-machine internal web application for short-form human video matting. The first version is a **semi-automatic high-quality tool**, not a zero-interaction platform. + +Primary flow: + +`upload source video -> choose template frame -> click one or more target people -> submit job -> queue -> generate results -> download transparent deliverables` + +The system runs on a fixed internal machine and is optimized for internal testing on TVC and ad-style clips, typically up to 10 seconds at 1080p, on hardware with 128 GB RAM and an RTX 5090 32 GB GPU. + +## Why This Scope Fits The Current Repository + +The repository already proves the core matting path: + +- Command-line inference takes a video plus a first-frame mask and writes both foreground and alpha outputs. + - `README.md` states that each run requires a video and its first-frame segmentation mask. + - `inference_matanyone2.py` loads `mask_path` and writes `*_fgr.mp4` and `*_pha.mp4`. +- The interactive demo already fills the missing mask-preparation step through SAM-based clicking on a chosen frame. + - `hugging_face/app.py` handles point-click refinement, multi-mask collection, and matting execution. +- Device selection already supports CUDA, MPS, and CPU fallback. + - `matanyone2/utils/device.py` selects the best available device. + +The repository also shows why the first version should stay narrow: + +- `matanyone2/utils/inference_utils.py` reads the full video into memory. +- `inference_matanyone2.py` and `hugging_face/matanyone2_wrapper.py` accumulate frame tensors and output arrays in memory. +- The demo is an interaction prototype, not a production-shaped web service. + +This makes the current codebase a strong base for an internal tool, but not a drop-in zero-touch service. + +## Product Goal + +Create a stable internal web tool that allows a user to: + +1. Upload a short source video. +2. Select a template frame. +3. Click one or more target people on that frame to create the initial mask. +4. Submit the task to a queue. +5. Download transparent deliverables after processing completes. + +## Explicit Version 1 Scope + +### In Scope + +- Fixed internal machine deployment. +- Browser access over the internal network. +- No authentication. +- Single logical worker on one GPU. +- Multiple submitted jobs allowed, with queueing. +- One running inference job at a time. +- One uploaded video per job. +- One combined matte result per job, even when multiple people are selected. +- Template-frame selection before submission. +- Point-based mask creation and refinement on the selected frame. +- Foreground and alpha intermediate outputs. +- Transparent output delivery as: + - `RGBA PNG` sequence + - `RGBA PNG` zip package + - optional `MOV ProRes 4444` +- Basic task status UI and download UI. +- Error messages visible in the web app. + +### Out Of Scope + +- Fully automatic subject detection or auto-selection. +- Account system, SSO, audit trail, or permission tiers. +- Multi-machine scheduling. +- High-concurrency production serving. +- Long-term result gallery or media asset management. +- Real-time preview while inference is running. +- Per-person separate output packages for multi-target jobs. +- Guaranteed support for long videos. + +## Users And Usage Assumptions + +- Users are internal testers, not external customers. +- Input videos are usually TVC or ad-style clips and usually no longer than 10 seconds. +- Quality matters more than throughput. +- Operators can tolerate a short wait, including queue wait time. +- Users need downloadable transparent assets for post-production workflows. + +## Success Criteria + +Version 1 is successful if it can reliably do the following on the target machine: + +- Accept a 1080p clip around 10 seconds long. +- Let the user select one or more target people through clicks on a template frame. +- Queue the job instead of rejecting it when another job is already running. +- Generate a combined matte result for the selected targets. +- Produce downloadable transparent output as `RGBA PNG` sequence zip. +- Produce `foreground mp4` and `alpha mp4` for inspection and fallback. +- Optionally produce `MOV ProRes 4444` when export support is available. +- Show clear job state transitions and actionable failure messages. + +## Product Definition + +### Positioning + +This is an internal **semi-automatic matting workstation** exposed through a web browser. It is not a batch automation system and not a general-purpose media processing platform. + +### Core Output Contract + +Each completed job produces: + +- `foreground.mp4` +- `alpha.mp4` +- `rgba_png/` sequence +- `rgba_png.zip` +- optional `output_prores4444.mov` + +Important distinction: + +- `alpha.mp4` is an alpha-matte video, not the final transparent delivery format. +- `rgba_png.zip` is the primary version-1 delivery artifact. +- `output_prores4444.mov` is a best-effort enhanced export and must not block overall job success if the PNG export succeeded. + +### Multi-Target Behavior + +Version 1 supports selecting multiple people on the template frame, but all selected people are merged into one combined mask before inference submission. The job produces one combined output package, not separate outputs per person. + +This matches the current repository direction more closely than a per-person export design and keeps the first implementation focused. + +## User Experience And Page Flow + +The UI should be a simple 3-step job flow rather than a large admin console. + +### Step 1: Upload + +The upload page allows the user to: + +- upload one video +- see basic validation feedback +- inspect extracted metadata such as duration, fps, resolution, and file size +- proceed to frame selection and mask authoring + +Validation should reject or stop early on: + +- unsupported file format +- empty or unreadable video +- missing frames +- file too large for configured limits +- duration beyond the configured version-1 boundary + +### Step 2: Mark Targets + +The mask-authoring page allows the user to: + +- choose the template frame +- add positive and negative points +- refine the first-frame mask +- add multiple masks +- see a merged mask preview +- confirm the final selection before submission + +This interaction should reuse the behavior already proven in `hugging_face/app.py`, but the implementation should be extracted into service modules rather than reusing the Gradio event graph directly. + +### Step 3: Queue, Status, And Download + +After submission, the user lands on a task page that shows: + +- job id +- current state +- queue position when queued +- timestamps +- warning state for partial export success +- final download links when complete + +Recommended visible states: + +- `queued` +- `preparing` +- `running` +- `exporting` +- `completed` +- `completed_with_warning` +- `failed` +- `interrupted` + +## System Design + +### Architecture Choice + +Use a **single application with clear internal modules**, not multiple deployable services. + +Recommended structure: + +- Web/API layer +- Mask authoring service +- Job service +- Inference service +- Export service +- Worker process +- Runtime storage layer + +This is intentionally more structured than the current Gradio demo but much lighter than a distributed system. + +### Web/API Layer + +Responsibilities: + +- file upload +- request validation +- template-frame preview +- click/mask interaction endpoints +- job submission +- job status queries +- artifact downloads + +This layer must never run long GPU inference inline inside the request/response cycle. + +### Mask Authoring Service + +Responsibilities: + +- store click state +- call SAM-backed refinement +- manage multiple masks +- merge selected masks into one final template mask + +This should be extracted from the current interaction logic in `hugging_face/app.py`. + +### Job Service + +Responsibilities: + +- create job records +- assign job ids +- persist state transitions +- manage working directories +- expose queue position +- mark retries or reruns as new jobs + +### Inference Service + +Responsibilities: + +- load the selected model +- read the submitted source video +- apply the final template mask +- run MatAnyone2 inference +- write intermediate results + +This service should reuse the existing repository inference logic rather than rewriting the model path. + +### Export Service + +Responsibilities: + +- combine foreground and alpha into `RGBA PNG` frames +- zip the PNG sequence +- optionally render `MOV ProRes 4444` +- expose export warnings separately from inference failures + +### Worker Process + +Use a separate worker process for GPU work. + +Reasons: + +- current inference paths can be memory-heavy +- isolated process failure is easier to recover from +- GPU memory cleanup is more predictable +- web responsiveness does not depend on inference timing + +Only one inference worker should run at a time in version 1. + +## Storage Design + +Use a lightweight persistent local design: + +- `SQLite` for job metadata and status +- per-job directories for inputs, outputs, parameters, and logs + +Suggested runtime contents per job: + +- original uploaded video +- selected template-frame index +- click history or final click payload +- final merged mask PNG +- parameter JSON +- processing log +- foreground output +- alpha output +- PNG sequence output +- zip archive +- optional ProRes output + +This allows: + +- queue persistence across web process restarts +- job inspection without reading application memory +- easy cleanup policies later +- future extension to a results page without redesigning storage + +## Queue Behavior + +Version 1 queue policy: + +- allow multiple submissions +- run only one inference job at a time +- show waiting jobs as queued +- show queue order in the UI +- do not deduplicate matching uploads +- do not auto-cancel older jobs + +If the server restarts: + +- `queued` jobs remain queued +- `running` jobs become `interrupted` +- users can manually resubmit or rerun through a new job submission path later + +## Processing Pipeline + +### Draft Stage + +- upload video +- read minimal metadata +- extract preview frame(s) +- collect click state and template frame choice +- generate final merged mask + +### Submitted Job Stage + +- persist source video and mask +- persist selected parameters +- create job row in the database +- enqueue the job + +### Worker Stage + +- move job to `preparing` +- initialize model and resources +- move job to `running` +- generate foreground and alpha outputs +- move job to `exporting` +- generate transparent deliverables +- move job to final status + +## Export Rules + +### Required Exports + +Required for job success: + +- `foreground.mp4` +- `alpha.mp4` +- `rgba_png.zip` + +### Optional Export + +Best-effort only: + +- `output_prores4444.mov` + +### Success Semantics + +- If inference fails, the job fails. +- If PNG sequence generation fails, the job fails. +- If PNG succeeds but ProRes export fails, the job completes with warning. + +This keeps the primary internal post-production path reliable while treating codec-specific export as an enhancement. + +## Error Handling + +### Validation Errors + +Handled before queue submission: + +- unreadable video +- no extracted frames +- no target selected +- invalid parameters +- unsupported file type +- configured duration or size limit exceeded + +### Runtime Failures + +Handled on the worker side: + +- model load failure +- GPU or CUDA failure +- inference exception +- filesystem write failure +- export exception + +The task page should show a clear, human-readable failure reason. Users should not need terminal access. + +### Logging + +Each job should write a processing log that captures: + +- start and end timestamps +- model used +- template frame index +- selected parameters +- state transitions +- exception traceback on failure +- export warnings + +## Non-Functional Requirements + +- Stable on the target internal machine for short 1080p clips. +- Responsive web UI even when the worker is busy. +- Persistent queue and job metadata across normal web process restarts. +- Clean failure behavior without hanging the whole web process. +- No requirement for cloud dependencies. + +## Technical Recommendation + +Recommended implementation stack: + +- `FastAPI` for the web/API server +- lightweight server-rendered or light SPA frontend +- `SQLite` for job and queue metadata +- separate worker process for inference and export +- `ffmpeg` for packaging and transparent video export + +Avoid starting version 1 with a heavy frontend stack unless the UI requirements expand significantly. The difficult work here is inference orchestration and export reliability, not complex client state. + +## Repository Impact + +Do not keep layering product behavior directly into `hugging_face/app.py`. + +Recommended repository direction: + +- keep the existing demo as a demo +- extract reusable masking logic into service modules +- wrap inference into a service boundary +- add dedicated runtime, worker, and web modules for the internal app + +This prevents the production-shaped path from being trapped inside a Gradio event graph. + +## Risks And Boundaries + +### Current Codebase Risk + +The current repository is research-oriented and memory-heavy: + +- full-video reads into memory +- in-memory tensor accumulation +- output arrays accumulated before final write + +This is acceptable for version 1 on the target hardware and target clip length, but it must be treated as a boundary, not as proof that the design scales to longer clips or higher concurrency. + +### Export Risk + +`MOV ProRes 4444` depends on local `ffmpeg` capabilities and container/codec support. It should be implemented as an optional export profile and tested on the target machine before being treated as required. + +### Product Boundary + +The first version is intentionally a high-quality operator-assisted workflow. Any future goal of "upload and auto-pick the main person" is a separate project because it requires subject detection and selection logic not present as a stable product feature in the repository today. + +## Recommended Delivery Order + +Implementation should be planned in this order: + +1. service-wrap inference so a video and merged mask can be submitted programmatically +2. add job persistence and single-worker queueing +3. add upload, status, and download API paths +4. add mask-authoring endpoints and UI +5. add PNG sequence export and zip packaging +6. add optional ProRes 4444 export +7. add integration tests and restart behavior checks + +This order reduces risk by validating the core processing chain before investing in the full UI. + +## Acceptance Checklist + +The implementation plan should satisfy all of the following: + +- one user can upload a short clip and finish a complete job in the browser +- another user can submit a second job while the first is running +- the second job waits in queue and later completes +- the user can select multiple people and receive one combined output package +- the primary downloadable output is an `RGBA PNG` zip package +- the system also preserves `foreground.mp4` and `alpha.mp4` +- the UI shows clear state transitions and clear failures +- a web-process restart does not silently lose queued jobs + +## Decisions Locked By This Spec + +- Version 1 is semi-automatic, not zero-interaction. +- Deployment target is one fixed internal machine. +- No authentication in version 1. +- Queueing is allowed and required. +- Multi-target jobs are supported, but merged into one output. +- Transparent delivery is required. +- `RGBA PNG` zip is the primary final artifact. +- `MOV ProRes 4444` is optional best-effort output. +- The existing Gradio demo is reference material, not the production host surface. diff --git a/docs/superpowers/specs/2026-03-27-matanyone2-internal-webapp-ui-redesign-design.md b/docs/superpowers/specs/2026-03-27-matanyone2-internal-webapp-ui-redesign-design.md new file mode 100644 index 0000000..b2c8425 --- /dev/null +++ b/docs/superpowers/specs/2026-03-27-matanyone2-internal-webapp-ui-redesign-design.md @@ -0,0 +1,566 @@ +# MatAnyone2 Internal Web App UI Redesign Design + +**Date:** 2026-03-27 + +**Status:** Approved for planning + +## Summary + +Redesign the internal MatAnyone2 web app as a **desktop-first professional matting workstation** rather than a thin wrapper around the current demo flow. + +The redesign covers all three primary pages: + +- upload +- annotation +- results review + +The new UI direction is: + +- dark, post-production-tool visual language +- desktop-first layout for mouse and keyboard on large screens +- annotation as a real workbench, not a simple point-click page +- multi-target layer management +- a staged workflow: + - coarse subject selection + - edge refinement + - result preview + +This redesign is explicitly a **product and interaction redesign**, not an algorithm rewrite. It must make the current system more controllable, more legible, and more credible as an internal tool, while staying compatible with the current MatAnyone2-based backend. + +## Why A Full UI Redesign Is Needed + +The current UI has three structural problems: + +1. It looks like a technical demo, not an internal production tool. +2. It exposes low-level interaction details directly to the user. +3. It does not provide a clear model for refining difficult edges such as hair. + +Specific symptoms in the current implementation: + +- The annotation page keeps accumulating click points in one continuous session state. +- Old points remain visually present and conceptually active, which makes the page feel noisy and uncontrolled. +- The UI does not separate coarse object identification from edge refinement. +- Multi-target editing is not represented as a first-class layer workflow. +- The results page behaves more like a download list than a review surface. + +This makes even valid model behavior feel unreliable, because the product does not communicate what the user is doing at each step or what kind of result should be expected from that step. + +## Product Goal + +Turn the current internal web app into a **desktop annotation-and-review workstation** for short-form human video matting. + +The redesigned workflow should help a user: + +1. Upload a short video and confirm it is valid. +2. Enter a dedicated annotation workbench. +3. Build one or more target layers. +4. Move through a staged masking workflow with clear intent. +5. Review the generated result in a visually credible way. +6. Download deliverables when satisfied. + +## Explicit Scope + +### In Scope + +- Full redesign of: + - upload page + - annotation page + - results page +- New desktop-first layout system +- New visual system for a professional post-production-tool aesthetic +- Annotation workbench redesign +- Multi-target layer UI model +- Staged interaction model: + - coarse selection + - edge refinement + - preview +- Improved status, feedback, and review surfaces +- Keyboard shortcut design for annotation tools +- Frontend state model changes needed to support the redesigned workflow +- Light backend/API extensions required for: + - richer draft state + - layer metadata + - history-oriented interaction + - preview state switching + +### Out Of Scope + +- New matting model architecture +- Guaranteed hair-quality improvements from algorithm changes alone +- Full Photoshop-class matte painting tools +- Mobile-first experience +- Touch-first tablet workflows +- Long-term asset library or project management +- Multi-user collaborative editing + +## Design Principles + +### 1. Canvas First + +The image and its matte state are the center of the product. Tooling, metadata, and status should frame the canvas, not compete with it. + +### 2. Stage Clarity + +Users must always know whether they are: + +- selecting the subject +- refining edges +- previewing the result + +The system should never collapse these into one undifferentiated interaction state. + +### 3. Layer-Based Mental Model + +Each target person is an independent layer. The UI must not treat multiple targets as an invisible merged state during editing, even if downstream export initially remains combined. + +### 4. Professional Restraint + +The UI should feel precise and trustworthy, not flashy. Deep color, quiet structure, and strong hierarchy are preferred over decorative effects. + +### 5. Feedback Over Logs + +The product should communicate workflow meaning, not implementation details. Users should see messages such as "Subject updated" or "Edge refinement applied", not coordinate-heavy click logs as the primary feedback language. + +## Users And Usage Assumptions + +- Internal operators +- Mouse and keyboard +- Large-screen desktops, typically 1920x1080 or larger +- Short-form videos, usually within the existing internal tool constraints +- Users care about visual judgment, especially around silhouette quality and hair +- Users are comfortable with tool-like interfaces if the workflow is clear + +## Information Architecture + +The redesigned product remains a three-page flow, but each page becomes a purpose-built surface instead of a minimal form. + +### 1. Upload Page + +Purpose: + +- begin a new session +- validate the media +- show system readiness +- set expectations before entering annotation + +Primary areas: + +- session header +- large drag-and-drop upload panel +- media information card +- output summary +- primary action bar + +### 2. Annotation Workbench + +Purpose: + +- construct and refine target masks in a controlled environment + +Primary layout: + +- left tool rail +- central canvas stage +- right layer and inspector panel + +Primary page zones: + +- top workbench bar +- stage switcher +- canvas and view controls +- target layer list +- tool inspector +- contextual help + +### 3. Results Review Page + +Purpose: + +- inspect the generated outputs +- judge whether the matte is acceptable +- download deliverables +- return to annotation when needed + +Primary areas: + +- result top bar +- large preview viewport +- preview mode tabs +- artifact panel +- warnings panel +- target review controls + +## Visual System + +### Visual Direction + +The visual reference point is a professional post-production desktop tool, not a generic SaaS admin and not a consumer media site. + +The interface should read as: + +- stable +- dark +- structured +- detail-oriented + +### Color System + +Use a restrained dark palette with semantic highlights: + +- background: + - deep graphite + - cold charcoal +- panel levels: + - base + - raised + - floating +- text: + - warm off-white to controlled mid-gray +- semantic accents: + - active: cool cyan-blue + - success: restrained green + - warning: amber + - error: orange-red + +Mask and annotation colors: + +- active mask fill: blue-violet translucent fill +- edge highlight: warm orange outline +- positive points: green +- negative points: magenta-violet + +These colors are functional. They should not be reused as decorative branding flourishes. + +### Typography + +Typography should feel tool-oriented: + +- compact, legible UI typography +- restrained title sizing +- strong small labels and panel headings +- tabular numbers for fps, frame counts, durations, and queue-related data + +Avoid: + +- marketing-style oversized headlines +- excessive weight contrast +- ornamental font choices + +### Panels And Surfaces + +Panels should feel like parts of one workstation shell: + +- narrow, disciplined radius +- subtle separators +- consistent spacing scale +- low-noise backgrounds +- strong visual priority for the canvas + +The UI should avoid looking like a stack of unrelated web cards. + +### Motion + +Motion should only clarify state changes: + +- stage switching +- layer selection +- preview mode changes +- job progress transitions +- compact hover and pressed feedback + +Animation rules: + +- use transform and opacity only +- respect `prefers-reduced-motion` +- do not animate layout dimensions +- avoid decorative ambient motion + +## Upload Page Design + +### Purpose + +The upload page should confirm media readiness before annotation starts. + +### Core Components + +- `SessionHeader` + - product name + - machine/GPU readiness + - queue status +- `DropzonePanel` + - drag/drop target + - click-to-select fallback +- `MediaInfoCard` + - file name + - duration + - resolution + - frame rate + - file size + - first-frame thumbnail +- `OutputSummary` + - alpha + - foreground + - PNG sequence zip + - ProRes 4444 +- `PrimaryActionBar` + - primary: enter annotation workbench + - secondary: reselect media + +### UX Rules + +- immediate validation feedback +- no tiny form controls as the main interaction +- no hidden assumptions about supported usage +- all key constraints visible before the user commits + +## Annotation Workbench Design + +### Core Layout + +The annotation page becomes a three-column workbench: + +- left: tool rail +- center: canvas stage +- right: layers and inspector + +### Top Workbench Bar + +Displays: + +- source video name +- current stage +- active target layer +- save state +- submit action + +This is the user's orientation anchor. + +### Stage Model + +The annotation flow is explicitly staged: + +1. `Coarse Selection` +2. `Edge Refinement` +3. `Preview` + +Each stage changes which controls are emphasized and how the canvas is interpreted. + +### Tool Rail + +The left rail contains primary tools: + +- browse / pan +- positive point +- negative point +- add region +- subtract region +- edge refinement brush +- compare / inspection + +Shortcuts should be visible and standardized. + +### Canvas Stage + +The center canvas supports: + +- zoom +- pan +- fit to screen +- 100% and 200% inspection +- mask overlay opacity +- original / matte / composite / compare modes +- optional transparency grid + +The canvas must be visually dominant. + +### Layer Panel + +The right-side layer panel manages multiple targets: + +- create target +- rename target +- show/hide +- lock +- delete +- solo current target +- choose active editing target + +Each target is edited independently at the UI level. + +### Inspector Panel + +The inspector changes based on the active stage and tool. + +Examples: + +- current tool description +- point counts +- brush size +- edge intensity or refinement settings +- history list +- undo / redo +- clear current stage + +### Why The Current "Too Many Points" Problem Happens + +The current implementation keeps accumulating clicks into one persistent interaction state and visually exposes that history too directly. + +The redesign fixes this by changing the model: + +- clicks belong to the active target layer +- clicks are stage-aware +- only active or recent points are shown by default +- history is managed in the inspector, not sprayed permanently across the canvas +- refinement is not treated as "add more points forever" + +### Annotation Interaction Rules + +- coarse selection is for identifying the subject, not perfect edges +- edge refinement is where the product focuses the user on hair, shoulders, and semi-transparent boundaries +- preview hides annotation clutter and focuses on output judgment +- feedback messages describe workflow meaning, not raw implementation logs + +## Results Review Page Design + +### Purpose + +The result page is a review surface first and a download surface second. + +### Core Components + +- `ResultTopbar` + - job status + - back to annotation + - re-export or rerun entry +- `PreviewViewport` + - large preview area +- `PreviewModeTabs` + - original + - alpha + - foreground + - composite +- `ArtifactPanel` + - direct downloads and export metadata +- `WarningPanel` + - export caveats + - degraded outputs +- `TargetReviewPanel` + - per-target review toggles if supported by the frontend state +- `JobTimeline` + - queued + - running + - exporting + - completed + +### UX Rules + +- preview is visually prioritized over raw metadata +- status is visible without pushing the preview off screen +- download links are organized and named like deliverables, not implementation leftovers +- users can clearly return to editing if quality is not acceptable + +## Frontend State Model Changes + +The redesign requires a richer frontend state model than the current flat page logic. + +The annotation workbench should track: + +- active stage +- active tool +- active target layer +- target layer collection +- per-layer click state +- per-layer visibility and lock state +- history entries +- canvas view mode +- canvas zoom and pan state +- current preview assets +- unsaved change state + +This is a structural shift away from the current single-page event handling model. + +## Backend And API Implications + +The redesign should avoid unnecessary backend rewrites, but some API growth is expected. + +Recommended additions or adjustments: + +- draft/session endpoints should be able to return richer editor state +- layer metadata should be serializable +- history-oriented interaction should be supported +- preview resources should support multiple view modes cleanly +- submission should preserve the distinction between UI layers and downstream merged masks + +The backend does not need to become algorithmically smarter in the first phase, but it must support the product model cleanly. + +## Naming And Language Adjustments + +The redesign should reduce technical demo language in the UI. + +De-emphasize or replace phrases such as: + +- "Create Draft" +- "Save Mask" +- coordinate-heavy click logs + +Prefer workflow-oriented language such as: + +- "Start Session" +- "Save Target" +- "Refine Edge" +- "Preview Result" +- "Submit For Processing" + +## Accessibility And Input Rules + +The redesign must keep strong desktop usability: + +- visible focus states +- keyboard navigability +- minimum usable target sizes +- semantic labeling +- error messages near the related area +- contrast-safe overlays and panel text + +Desktop-first does not remove the need for accessibility discipline. + +## What This Redesign Does Not Promise + +This redesign will improve: + +- clarity +- controllability +- workflow quality +- confidence in result review + +It does not, by itself, guarantee materially better hair quality at the algorithm level. + +The product should make that boundary clear by: + +- separating coarse selection from edge refinement +- surfacing better preview modes +- making quality judgment easier +- leaving room for later algorithm or matte-editing upgrades + +## Acceptance Criteria + +The redesign is successful when: + +- the product no longer feels like a technical demo +- the upload page clearly communicates readiness and constraints +- the annotation page feels like a controlled workbench +- multi-target editing is visually and conceptually layer-based +- users can distinguish coarse subject selection from edge refinement +- the interface no longer overwhelms the user with persistent point clutter +- the results page supports visual review before download +- the three pages feel like one coherent tool, not three unrelated templates + +## Recommended Implementation Order + +1. Rebuild the base shell, layout grid, and global visual system. +2. Rebuild the annotation frontend state model and workbench layout. +3. Add layer-oriented target management and staged tool surfaces. +4. Rebuild the results page as a review-first surface. +5. Only after the above, refine supporting API responses where required. + +This order ensures the redesign fixes the interaction model first instead of only repainting the existing behavior. diff --git a/matanyone2/utils/get_default_model.py b/matanyone2/utils/get_default_model.py index 714a351..5ab6110 100644 --- a/matanyone2/utils/get_default_model.py +++ b/matanyone2/utils/get_default_model.py @@ -3,13 +3,20 @@ """ from omegaconf import open_dict from hydra import compose, initialize +from hydra.core.global_hydra import GlobalHydra import torch from matanyone2.model.matanyone2 import MatAnyone2 def get_matanyone2_model(ckpt_path, device=None) -> MatAnyone2: + global_hydra = GlobalHydra.instance() + if global_hydra.is_initialized(): + global_hydra.clear() initialize(version_base='1.3.2', config_path="../config", job_name="eval_our_config") - cfg = compose(config_name="eval_matanyone_config") + try: + cfg = compose(config_name="eval_matanyone_config") + finally: + GlobalHydra.instance().clear() with open_dict(cfg): cfg['weights'] = ckpt_path diff --git a/matanyone2/utils/inference_utils.py b/matanyone2/utils/inference_utils.py index c5bb6a0..e16af15 100644 --- a/matanyone2/utils/inference_utils.py +++ b/matanyone2/utils/inference_utils.py @@ -7,7 +7,16 @@ import torchvision IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.JPG', '.JPEG', '.PNG') -VIDEO_EXTENSIONS = ('.mp4', '.mov', '.avi', '.MP4', '.MOV', '.AVI') +VIDEO_EXTENSIONS = ( + '.mp4', + '.mov', + '.avi', + '.m4v', + '.MP4', + '.MOV', + '.AVI', + '.M4V', +) def read_frame_from_videos(frame_root): if frame_root.endswith(VIDEO_EXTENSIONS): # Video file path @@ -51,4 +60,4 @@ def gen_erosion(alpha, min_kernel_size, max_kernel_size): kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size,kernel_size)) fg = np.array(np.equal(alpha, 255).astype(np.float32)) erode = cv2.erode(fg, kernel, iterations=1)*255 - return erode.astype(np.float32) \ No newline at end of file + return erode.astype(np.float32) diff --git a/matanyone2/webapp/__init__.py b/matanyone2/webapp/__init__.py new file mode 100644 index 0000000..ee99cb0 --- /dev/null +++ b/matanyone2/webapp/__init__.py @@ -0,0 +1 @@ +"""Internal web app package for MatAnyone2.""" diff --git a/matanyone2/webapp/api/__init__.py b/matanyone2/webapp/api/__init__.py new file mode 100644 index 0000000..ac4c204 --- /dev/null +++ b/matanyone2/webapp/api/__init__.py @@ -0,0 +1 @@ +"""FastAPI entrypoints for the internal web app.""" diff --git a/matanyone2/webapp/api/app.py b/matanyone2/webapp/api/app.py new file mode 100644 index 0000000..9ca382d --- /dev/null +++ b/matanyone2/webapp/api/app.py @@ -0,0 +1,48 @@ +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates + +from matanyone2.webapp.config import WebAppSettings +from matanyone2.webapp.db import init_database +from matanyone2.webapp.api.routes import annotation, jobs, pages, uploads +from matanyone2.webapp.queue import QueueCoordinator +from matanyone2.webapp.repository import JobRepository +from matanyone2.webapp.services.masking import MaskingService +from matanyone2.webapp.services.video import VideoDraftService + + +def create_app(settings=None) -> FastAPI: + if settings is None: + settings = WebAppSettings() + init_database(settings.database_path) + app = FastAPI(title="MatAnyone2 Internal Web App") + app.state.settings = settings + app.state.repository = JobRepository.from_path(settings.database_path) + app.state.queue = QueueCoordinator(app.state.repository) + app.state.video_service = VideoDraftService( + runtime_root=settings.runtime_root, + max_video_seconds=settings.max_video_seconds, + max_upload_bytes=settings.max_upload_bytes, + ) + app.state.masking_service = MaskingService( + runtime_root=settings.runtime_root, + sam_backend=settings.sam_backend, + sam_model_type=settings.sam_model_type, + sam2_variant=settings.sam2_variant, + sam2_checkpoint_path=settings.sam2_checkpoint_path, + sam3_checkpoint_path=settings.sam3_checkpoint_path, + ) + app.state.drafts = {} + app.state.templates = Jinja2Templates(directory="matanyone2/webapp/templates") + app.state.queue.recover_interrupted_jobs() + app.mount("/static", StaticFiles(directory="matanyone2/webapp/static"), name="static") + app.include_router(pages.router) + app.include_router(uploads.router) + app.include_router(annotation.router) + app.include_router(jobs.router) + + @app.get("/healthz") + def healthcheck() -> dict[str, str]: + return {"status": "ok"} + + return app diff --git a/matanyone2/webapp/api/dependencies.py b/matanyone2/webapp/api/dependencies.py new file mode 100644 index 0000000..c4976fe --- /dev/null +++ b/matanyone2/webapp/api/dependencies.py @@ -0,0 +1,25 @@ +from fastapi import Request + + +def get_settings(request: Request): + return request.app.state.settings + + +def get_repository(request: Request): + return request.app.state.repository + + +def get_queue(request: Request): + return request.app.state.queue + + +def get_video_service(request: Request): + return request.app.state.video_service + + +def get_draft_store(request: Request): + return request.app.state.drafts + + +def get_masking_service(request: Request): + return request.app.state.masking_service diff --git a/matanyone2/webapp/api/routes/__init__.py b/matanyone2/webapp/api/routes/__init__.py new file mode 100644 index 0000000..0ef8ec4 --- /dev/null +++ b/matanyone2/webapp/api/routes/__init__.py @@ -0,0 +1 @@ +"""Route modules for the internal web app.""" diff --git a/matanyone2/webapp/api/routes/annotation.py b/matanyone2/webapp/api/routes/annotation.py new file mode 100644 index 0000000..e30d9e7 --- /dev/null +++ b/matanyone2/webapp/api/routes/annotation.py @@ -0,0 +1,646 @@ +import json +import mimetypes + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import FileResponse +from pydantic import BaseModel + +from matanyone2.webapp.api.dependencies import ( + get_draft_store, + get_masking_service, + get_repository, + get_video_service, +) + + +class DraftClickPayload(BaseModel): + x: int + y: int + positive: bool = True + + +class DraftSubmitPayload(BaseModel): + process_start_frame_index: int + process_end_frame_index: int + template_frame_index: int + selected_masks: list[str] + + +class DraftTargetCreatePayload(BaseModel): + name: str | None = None + + +class DraftTargetUpdatePayload(BaseModel): + name: str | None = None + visible: bool | None = None + locked: bool | None = None + refine_preset: str | None = None + preset_strength: float | None = None + motion_strength: float | None = None + temporal_stability: float | None = None + edge_feather_radius: float | None = None + + +class DraftStagePayload(BaseModel): + stage: str + + +class DraftBrushPayload(BaseModel): + mode: str + radius: int + points: list[tuple[int, int]] + + +class DraftTemplateFramePayload(BaseModel): + frame_index: int + + +class DraftProcessingRangePayload(BaseModel): + start_frame_index: int + end_frame_index: int + + +class DraftWorkflowStepPayload(BaseModel): + workflow_step: str + + +router = APIRouter() + +WORKFLOW_STEPS = ("clip", "mask", "refine", "review") + +STAGE_PRESENTATION = { + "coarse": { + "stage_label": "Coarse Selection", + "canvas_mode_label": "Guided silhouette pass", + "stage_note": ( + "Establish each person with a few positive and negative clicks. " + "Save the target once the silhouette is broadly correct." + ), + }, + "refine": { + "stage_label": "Edge Refinement", + "canvas_mode_label": "Contour tightening", + "stage_note": ( + "Stay near hairlines, shoulders, and translucent edges. " + "Refinement keeps the current target isolated while you tighten the contour." + ), + }, + "preview": { + "stage_label": "Preview", + "canvas_mode_label": "Read-only review", + "stage_note": ( + "Point editing is locked in preview. Review saved targets, confirm export masks, " + "then queue the matting job." + ), + }, +} + + +def _require_session(draft_store, draft_id: str): + session = draft_store.get(draft_id) + if session is None: + raise HTTPException(status_code=404, detail="draft not found") + return session + + +def _target_payload(target, session): + return { + "target_id": target.target_id, + "name": target.name, + "point_count": len(target.click_points), + "visible": target.visible, + "locked": target.locked, + "refine_preset": target.refine_preset, + "preset_strength": target.preset_strength, + "motion_strength": target.motion_strength, + "temporal_stability": target.temporal_stability, + "edge_feather_radius": target.edge_feather_radius, + "saved_mask_name": target.saved_mask_name, + "selected": target.target_id == session.active_target_id, + } + + +def _active_mask_url(session, draft_id: str): + saved_mask_name = session.active_target.saved_mask_name + if saved_mask_name and saved_mask_name in session.saved_masks: + return f"/api/drafts/{draft_id}/masks/{saved_mask_name}" + if session.current_mask_path is not None: + return f"/api/drafts/{draft_id}/current-mask" + return None + + +def _active_review_job_url(session) -> str | None: + if not session.latest_job_id: + return None + return f"/api/jobs/{session.latest_job_id}" + + +def _normalize_sidebar_tab(tab_name: str | None) -> str: + if tab_name in {"targets", "refine", "export"}: + return tab_name + return "targets" + + +def _step_index(step_name: str) -> int: + try: + return WORKFLOW_STEPS.index(step_name) + except ValueError: + return 0 + + +def _sync_workflow_state(session, workflow_step: str) -> None: + if workflow_step not in WORKFLOW_STEPS: + raise ValueError(f"unknown workflow step: {workflow_step}") + + session.workflow_step = workflow_step + if workflow_step == "clip": + session.active_sidebar_tab = "targets" + session.stage = "coarse" + elif workflow_step == "mask": + session.active_sidebar_tab = "targets" + session.stage = "coarse" + elif workflow_step == "refine": + session.active_sidebar_tab = "refine" + session.stage = "refine" + else: + session.active_sidebar_tab = "export" + session.stage = "preview" + + +def _set_sidebar_tab(session, tab_name: str | None) -> None: + session.active_sidebar_tab = _normalize_sidebar_tab(tab_name) + + +def _workbench_payload(session, draft_id: str): + stage_meta = STAGE_PRESENTATION[session.stage] + active_target = session.active_target + target_locked = active_target.locked + has_template_frame = session.draft.template_frame_index is not None + workflow_step = session.workflow_step if session.workflow_step in WORKFLOW_STEPS else "clip" + step_index = _step_index(workflow_step) + return { + "draft_id": draft_id, + "stage": session.stage, + "workflow_step": workflow_step, + "available_steps": list(WORKFLOW_STEPS), + "can_go_back": step_index > 0, + "can_go_next": step_index < len(WORKFLOW_STEPS) - 1, + "active_sidebar_tab": _normalize_sidebar_tab(session.active_sidebar_tab), + "compare_enabled": bool(session.compare_enabled), + "latest_job_id": session.latest_job_id, + "review_status_url": _active_review_job_url(session), + "stage_label": stage_meta["stage_label"], + "canvas_mode_label": stage_meta["canvas_mode_label"], + "stage_note": stage_meta["stage_note"], + "can_apply_clicks": session.stage != "preview" and not target_locked and has_template_frame, + "can_create_target": session.stage != "preview", + "can_save_current_target": ( + session.stage != "preview" + and session.current_mask_path is not None + and not target_locked + and has_template_frame + ), + "can_undo_clicks": ( + session.stage != "preview" and not target_locked and len(session.click_points) > 0 and has_template_frame + ), + "can_reset_target": ( + session.stage != "preview" and not target_locked and len(session.click_points) > 0 and has_template_frame + ), + "can_submit": bool(session.selected_mask_names) and has_template_frame, + "can_apply_range": True, + "can_apply_template_frame": True, + "active_target_id": session.active_target_id, + "process_start_frame_index": session.draft.process_start_frame_index, + "process_end_frame_index": session.draft.process_end_frame_index, + "template_frame_index": session.draft.template_frame_index, + "frame_count": session.draft.frame_count, + "fps": session.draft.fps, + "duration_seconds": session.draft.duration_seconds, + "source_video_url": f"/api/drafts/{draft_id}/source-video", + "can_change_template_frame": session.stage != "preview", + "template_frame_url": f"/api/drafts/{draft_id}/template-frame", + "current_mask_url": ( + f"/api/drafts/{draft_id}/current-mask" + if session.current_mask_path is not None + else None + ), + "active_mask_url": _active_mask_url(session, draft_id), + "current_preview_url": ( + f"/api/drafts/{draft_id}/current-preview" + if session.current_preview_path is not None + else None + ), + "mask_names": sorted(session.saved_masks), + "selected_mask_names": sorted(session.selected_mask_names), + "targets": [ + _target_payload(target, session) + for target in session.targets.values() + ], + } + + +@router.get("/api/drafts/{draft_id}") +def get_workbench_state(draft_id: str, draft_store=Depends(get_draft_store)): + session = _require_session(draft_store, draft_id) + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/workflow-step") +def set_workflow_step( + draft_id: str, + payload: DraftWorkflowStepPayload, + draft_store=Depends(get_draft_store), +): + session = _require_session(draft_store, draft_id) + try: + _sync_workflow_state(session, payload.workflow_step) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/targets") +def create_target( + draft_id: str, + payload: DraftTargetCreatePayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + target = masking_service.create_target(session, name=payload.name) + response = _workbench_payload(session, draft_id) + response.update(_target_payload(target, session)) + return response + + +@router.post("/api/drafts/{draft_id}/targets/{target_id}/select") +def select_target( + draft_id: str, + target_id: str, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + try: + masking_service.select_target(session, target_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail=f"target not found: {target_id}") from exc + return _workbench_payload(session, draft_id) + + +@router.patch("/api/drafts/{draft_id}/targets/{target_id}") +def update_target( + draft_id: str, + target_id: str, + payload: DraftTargetUpdatePayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + try: + masking_service.update_target( + session, + target_id, + name=payload.name, + visible=payload.visible, + locked=payload.locked, + refine_preset=payload.refine_preset, + preset_strength=payload.preset_strength, + motion_strength=payload.motion_strength, + temporal_stability=payload.temporal_stability, + edge_feather_radius=payload.edge_feather_radius, + ) + except KeyError as exc: + raise HTTPException(status_code=404, detail=f"target not found: {target_id}") from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/template-frame") +def set_template_frame( + draft_id: str, + payload: DraftTemplateFramePayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), + video_service=Depends(get_video_service), +): + session = _require_session(draft_store, draft_id) + try: + video_service.select_template_frame(session.draft, payload.frame_index) + masking_service.reset_session_for_template_frame( + session, + frame_index=payload.frame_index, + ) + _sync_workflow_state(session, "mask") + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/processing-range") +def set_processing_range( + draft_id: str, + payload: DraftProcessingRangePayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), + video_service=Depends(get_video_service), +): + session = _require_session(draft_store, draft_id) + try: + video_service.select_processing_range( + session.draft, + start_frame_index=payload.start_frame_index, + end_frame_index=payload.end_frame_index, + ) + masking_service.reset_session_for_processing_range(session) + session.latest_job_id = None + session.compare_enabled = False + _sync_workflow_state(session, "clip") + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/stage") +def set_stage( + draft_id: str, + payload: DraftStagePayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + try: + masking_service.set_stage(session, payload.stage) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if payload.stage == "coarse": + _set_sidebar_tab(session, "targets") + session.workflow_step = "mask" if session.draft.template_frame_index is not None else "clip" + elif payload.stage == "refine": + _sync_workflow_state(session, "refine") + else: + _sync_workflow_state(session, "review" if session.latest_job_id else "review") + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/click") +def apply_click( + draft_id: str, + payload: DraftClickPayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + try: + result = masking_service.apply_click( + session, + x=payload.x, + y=payload.y, + positive=payload.positive, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + response = _workbench_payload(session, draft_id) + response.update( + { + "current_mask_path": str(result.current_mask_path), + "current_preview_path": str(result.current_preview_path), + "current_mask_url": f"/api/drafts/{draft_id}/current-mask", + "current_preview_url": f"/api/drafts/{draft_id}/current-preview", + } + ) + return response + + +@router.post("/api/drafts/{draft_id}/brush") +def apply_brush( + draft_id: str, + payload: DraftBrushPayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + try: + result = masking_service.apply_brush( + session, + points=payload.points, + mode=payload.mode, + radius=payload.radius, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + response = _workbench_payload(session, draft_id) + response.update( + { + "current_mask_path": str(result.current_mask_path), + "current_preview_path": str(result.current_preview_path), + "current_mask_url": f"/api/drafts/{draft_id}/current-mask", + "current_preview_url": f"/api/drafts/{draft_id}/current-preview", + } + ) + return response + + +@router.post("/api/drafts/{draft_id}/masks") +def save_mask( + draft_id: str, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + try: + mask_name = masking_service.save_current_mask(session) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + response = _workbench_payload(session, draft_id) + response.update({"mask_name": mask_name}) + return response + + +@router.post("/api/drafts/{draft_id}/undo") +def undo_click( + draft_id: str, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + masking_service.undo_last_click(session) + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/reset-target") +def reset_target( + draft_id: str, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + session = _require_session(draft_store, draft_id) + masking_service.reset_active_target(session) + return _workbench_payload(session, draft_id) + + +@router.post("/api/drafts/{draft_id}/submit") +def submit_draft( + draft_id: str, + payload: DraftSubmitPayload, + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), + repository=Depends(get_repository), +): + session = _require_session(draft_store, draft_id) + if payload.process_start_frame_index != session.draft.process_start_frame_index: + raise HTTPException(status_code=400, detail="submitted processing range does not match the current draft") + if payload.process_end_frame_index != session.draft.process_end_frame_index: + raise HTTPException(status_code=400, detail="submitted processing range does not match the current draft") + if session.draft.template_frame_index is None: + raise HTTPException(status_code=400, detail="apply a template frame inside the processing range before submitting") + if payload.template_frame_index != session.draft.template_frame_index: + raise HTTPException(status_code=400, detail="submitted template frame does not match the current processing range") + if not ( + session.draft.process_start_frame_index + <= payload.template_frame_index + <= session.draft.process_end_frame_index + ): + raise HTTPException(status_code=400, detail="template frame must fall inside the processing range") + try: + mask_path = masking_service.write_merged_mask(session, payload.selected_masks) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except KeyError as exc: + raise HTTPException(status_code=400, detail=f"unknown mask: {exc.args[0]}") from exc + + job = repository.create_job( + source_video_path=str(session.draft.video_path), + template_frame_index=payload.template_frame_index, + mask_path=str(mask_path), + params_json=json.dumps( + { + "template_frame_index": payload.template_frame_index, + "process_start_frame_index": payload.process_start_frame_index, + "process_end_frame_index": payload.process_end_frame_index, + "process_range_duration_seconds": ( + (payload.process_end_frame_index - payload.process_start_frame_index + 1) + / max(session.draft.fps, 1.0) + ), + "source_fps": session.draft.fps, + "selected_masks": payload.selected_masks, + "selected_mask_presets": { + mask_name: session.saved_mask_presets.get(mask_name, "balanced") + for mask_name in payload.selected_masks + }, + "selected_mask_controls": { + mask_name: { + "preset_strength": next( + ( + target.preset_strength + for target in session.targets.values() + if target.saved_mask_name == mask_name + ), + 0.5, + ), + "motion_strength": next( + ( + target.motion_strength + for target in session.targets.values() + if target.saved_mask_name == mask_name + ), + 0.35, + ), + "temporal_stability": next( + ( + target.temporal_stability + for target in session.targets.values() + if target.saved_mask_name == mask_name + ), + 0.0, + ), + "edge_feather_radius": next( + ( + target.edge_feather_radius + for target in session.targets.values() + if target.saved_mask_name == mask_name + ), + 0.0, + ), + } + for mask_name in payload.selected_masks + }, + } + ), + ) + session.latest_job_id = job.job_id + session.compare_enabled = False + _sync_workflow_state(session, "review") + return { + "job_id": job.job_id, + "status": job.status.value, + "workflow_step": session.workflow_step, + } + + +@router.get("/api/drafts/{draft_id}/current-preview") +def get_current_preview(draft_id: str, draft_store=Depends(get_draft_store)): + session = _require_session(draft_store, draft_id) + if session.current_preview_path is None: + raise HTTPException(status_code=404, detail="current preview not found") + return FileResponse( + session.current_preview_path, + media_type="image/png", + filename=session.current_preview_path.name, + ) + + +@router.get("/api/drafts/{draft_id}/source-video") +def get_source_video( + draft_id: str, + draft_store=Depends(get_draft_store), + video_service=Depends(get_video_service), +): + session = _require_session(draft_store, draft_id) + video_path = session.draft.video_path + preview_path = session.draft.browser_preview_path + if preview_path is None or not preview_path.exists(): + try: + preview_path = video_service.ensure_browser_preview( + video_path, + preview_path=video_path.parent / "preview_source.mp4", + ) + session.draft.browser_preview_path = preview_path + except RuntimeError: + preview_path = None + serving_path = preview_path or video_path + return FileResponse( + serving_path, + media_type=mimetypes.guess_type(serving_path.name)[0] or "video/mp4", + filename=serving_path.name, + ) + + +@router.get("/api/drafts/{draft_id}/current-mask") +def get_current_mask(draft_id: str, draft_store=Depends(get_draft_store)): + session = _require_session(draft_store, draft_id) + if session.current_mask_path is None: + raise HTTPException(status_code=404, detail="current mask not found") + return FileResponse( + session.current_mask_path, + media_type="image/png", + filename=session.current_mask_path.name, + ) + + +@router.get("/api/drafts/{draft_id}/masks/{mask_name}") +def get_saved_mask( + draft_id: str, + mask_name: str, + draft_store=Depends(get_draft_store), +): + session = _require_session(draft_store, draft_id) + mask_path = session.saved_masks.get(mask_name) + if mask_path is None: + raise HTTPException(status_code=404, detail=f"saved mask not found: {mask_name}") + return FileResponse( + mask_path, + media_type="image/png", + filename=mask_path.name, + ) diff --git a/matanyone2/webapp/api/routes/jobs.py b/matanyone2/webapp/api/routes/jobs.py new file mode 100644 index 0000000..80d37c7 --- /dev/null +++ b/matanyone2/webapp/api/routes/jobs.py @@ -0,0 +1,288 @@ +import json +from pathlib import Path + +import mimetypes + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import FileResponse + +from matanyone2.webapp.api.dependencies import ( + get_repository, + get_settings, + get_video_service, +) +from matanyone2.webapp.models import JobStatus + + +router = APIRouter() + + +ARTIFACT_SPECS = ( + ("foreground.mp4", "Foreground pass", "foreground"), + ("alpha.mp4", "Alpha matte", "alpha"), + ("rgba_png.zip", "RGBA PNG sequence", "png_sequence"), + ("output_prores4444.mov", "ProRes 4444", "prores"), +) + +PREVIEW_ARTIFACT_SPECS = ( + ("foreground", "preview_foreground.mp4"), + ("alpha", "preview_alpha.mp4"), +) + +TIMELINE_STEPS = ( + ("queued", "Queued"), + ("preparing", "Preparing"), + ("running", "Matting"), + ("exporting", "Export"), +) + +TIMELINE_STATE_INDEX = { + JobStatus.QUEUED: 0, + JobStatus.PREPARING: 1, + JobStatus.RUNNING: 2, + JobStatus.EXPORTING: 3, + JobStatus.COMPLETED: 3, + JobStatus.COMPLETED_WITH_WARNING: 3, + JobStatus.FAILED: 2, + JobStatus.INTERRUPTED: 2, +} + + +def _format_status_label(status: JobStatus) -> str: + return status.value.replace("_", " ").capitalize() + + +def _format_bytes(size_bytes: int | None) -> str | None: + if size_bytes is None: + return None + if size_bytes < 1024: + return f"{size_bytes} B" + if size_bytes < 1024 * 1024: + return f"{size_bytes / 1024:.1f} KB" + if size_bytes < 1024 * 1024 * 1024: + return f"{size_bytes / (1024 * 1024):.1f} MB" + return f"{size_bytes / (1024 * 1024 * 1024):.1f} GB" + + +def _parse_job_params(params_json: str) -> dict: + try: + payload = json.loads(params_json) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _build_timeline(status: JobStatus) -> list[dict[str, str]]: + current_index = TIMELINE_STATE_INDEX[status] + timeline = [] + for index, (key, label) in enumerate(TIMELINE_STEPS): + if index < current_index: + state = "complete" + elif index == current_index: + state = "current" + else: + state = "upcoming" + timeline.append({"key": key, "label": label, "state": state}) + return timeline + + +def _build_job_summary(job) -> dict[str, object]: + params = _parse_job_params(job.params_json) + selected_masks = params.get("selected_masks") + if not isinstance(selected_masks, list): + selected_masks = [] + selected_mask_presets = params.get("selected_mask_presets") + if not isinstance(selected_mask_presets, dict): + selected_mask_presets = {} + + process_start_frame_index = params.get("process_start_frame_index") + process_end_frame_index = params.get("process_end_frame_index") + process_range_duration_seconds = params.get("process_range_duration_seconds") + source_fps = params.get("source_fps") + + return { + "source_name": Path(job.source_video_path).name, + "template_frame_index": params.get( + "template_frame_index", + job.template_frame_index, + ), + "process_start_frame_index": process_start_frame_index, + "process_end_frame_index": process_end_frame_index, + "process_range_duration_seconds": process_range_duration_seconds, + "source_fps": source_fps, + "selected_mask_count": len(selected_masks), + "selected_masks": selected_masks, + "selected_mask_presets": selected_mask_presets, + "mask_name": Path(job.mask_path).name, + } + + +def _build_artifact_payload(job_id: str, runtime_root: Path) -> tuple[dict[str, str], dict[str, dict[str, object]]]: + job_dir = Path(runtime_root) / "jobs" / job_id + artifacts: dict[str, str] = {} + artifact_details: dict[str, dict[str, object]] = {} + + for artifact_name, label, kind in ARTIFACT_SPECS: + artifact_path = job_dir / artifact_name + available = artifact_path.exists() and artifact_path.is_file() + url = f"/api/jobs/{job_id}/artifacts/{artifact_name}" if available else None + size_bytes = artifact_path.stat().st_size if available else None + if available and url is not None: + artifacts[artifact_name] = url + artifact_details[artifact_name] = { + "name": artifact_name, + "label": label, + "kind": kind, + "available": available, + "url": url, + "size_bytes": size_bytes, + "size_label": _format_bytes(size_bytes), + } + return artifacts, artifact_details + + +def _artifact_urls(job_id: str, runtime_root: Path) -> dict[str, str]: + job_dir = Path(runtime_root) / "jobs" / job_id + artifact_names = [ + "foreground.mp4", + "alpha.mp4", + "rgba_png.zip", + "output_prores4444.mov", + ] + artifacts = {} + for artifact_name in artifact_names: + artifact_path = job_dir / artifact_name + if artifact_path.exists(): + artifacts[artifact_name] = f"/api/jobs/{job_id}/artifacts/{artifact_name}" + return artifacts + + +def _build_preview_payload(job_id: str, runtime_root: Path) -> dict[str, str]: + job_dir = Path(runtime_root) / "jobs" / job_id + preview_artifacts: dict[str, str] = {} + for preview_kind, artifact_name in PREVIEW_ARTIFACT_SPECS: + artifact_path = job_dir / artifact_name + if artifact_path.exists() and artifact_path.is_file(): + preview_artifacts[preview_kind] = f"/api/jobs/{job_id}/artifacts/{artifact_name}" + return preview_artifacts + + +@router.get("/api/jobs/{job_id}") +def get_job_status( + job_id: str, + repository=Depends(get_repository), + settings=Depends(get_settings), +): + try: + job = repository.get_job(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="job not found") from exc + queue_position = ( + repository.get_queue_position(job.job_id) + if job.status.value == "queued" + else None + ) + artifacts, artifact_details = _build_artifact_payload(job.job_id, settings.runtime_root) + return { + "job_id": job.job_id, + "status": job.status.value, + "status_label": _format_status_label(job.status), + "queue_position": queue_position, + "warning_text": job.warning_text, + "error_text": job.error_text, + "source_video_url": f"/api/jobs/{job.job_id}/source-video", + "artifacts": artifacts, + "preview_artifacts": _build_preview_payload(job.job_id, settings.runtime_root), + "artifact_details": artifact_details, + "job_summary": _build_job_summary(job), + "timeline": _build_timeline(job.status), + } + + +@router.get("/api/jobs/{job_id}/source-video") +def get_source_video( + job_id: str, + repository=Depends(get_repository), + settings=Depends(get_settings), + video_service=Depends(get_video_service), +): + try: + job = repository.get_job(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="job not found") from exc + + params = _parse_job_params(job.params_json) + job_dir = Path(settings.runtime_root) / "jobs" / job.job_id + source_path = Path(job.source_video_path) + if not source_path.exists() or not source_path.is_file(): + raise HTTPException(status_code=404, detail="source video not found") + + processing_range_start = params.get("process_start_frame_index") + processing_range_end = params.get("process_end_frame_index") + clip_source_path = source_path + preview_path = None + + if ( + isinstance(processing_range_start, int) + and isinstance(processing_range_end, int) + and processing_range_start >= 0 + and processing_range_end >= processing_range_start + ): + clip_candidate = job_dir / "processing_range.mp4" + if clip_candidate.exists() and clip_candidate.is_file(): + clip_source_path = clip_candidate + else: + try: + clip_source_path, _ = video_service.write_processing_range_clip( + source_path, + start_frame_index=processing_range_start, + end_frame_index=processing_range_end, + output_path=clip_candidate, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + preview_candidate = job_dir / "preview_source.mp4" + if preview_candidate.exists() and preview_candidate.is_file(): + preview_path = preview_candidate + else: + try: + preview_path = video_service.ensure_browser_preview( + clip_source_path, + preview_path=preview_candidate, + ) + except RuntimeError: + preview_path = None + else: + try: + preview_path = video_service.ensure_browser_preview( + source_path, + preview_path=source_path.parent / "preview_source.mp4", + ) + except RuntimeError: + preview_path = None + + serving_path = preview_path or clip_source_path + return FileResponse( + path=serving_path, + filename=serving_path.name, + media_type=mimetypes.guess_type(serving_path.name)[0] or "video/mp4", + ) + + +@router.get("/api/jobs/{job_id}/artifacts/{artifact_name}") +def download_artifact( + job_id: str, + artifact_name: str, + repository=Depends(get_repository), + settings=Depends(get_settings), +): + try: + repository.get_job(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="job not found") from exc + + artifact_path = Path(settings.runtime_root) / "jobs" / job_id / artifact_name + if not artifact_path.exists() or not artifact_path.is_file(): + raise HTTPException(status_code=404, detail="artifact not found") + return FileResponse(path=artifact_path, filename=artifact_name) diff --git a/matanyone2/webapp/api/routes/pages.py b/matanyone2/webapp/api/routes/pages.py new file mode 100644 index 0000000..aeee984 --- /dev/null +++ b/matanyone2/webapp/api/routes/pages.py @@ -0,0 +1,57 @@ +from fastapi import APIRouter, Depends, HTTPException, Request + +from matanyone2.webapp.api.dependencies import get_draft_store, get_repository + + +router = APIRouter() + + +@router.get("/") +def upload_page(request: Request): + templates = request.app.state.templates + return templates.TemplateResponse(request, "upload.html") + + +def _workspace_context(draft_id: str, session): + return { + "draft_id": draft_id, + "draft": session.draft, + "saved_masks": sorted(session.saved_masks), + "latest_job_id": session.latest_job_id, + } + + +@router.get("/drafts/{draft_id}/workspace") +def workspace_page(request: Request, draft_id: str, draft_store=Depends(get_draft_store)): + session = draft_store.get(draft_id) + if session is None: + raise HTTPException(status_code=404, detail="draft not found") + templates = request.app.state.templates + return templates.TemplateResponse( + request, + "workspace.html", + _workspace_context(draft_id, session), + ) + + +@router.get("/drafts/{draft_id}/annotate") +def annotate_page(request: Request, draft_id: str, draft_store=Depends(get_draft_store)): + session = draft_store.get(draft_id) + if session is None: + raise HTTPException(status_code=404, detail="draft not found") + templates = request.app.state.templates + return templates.TemplateResponse( + request, + "workspace.html", + _workspace_context(draft_id, session), + ) + + +@router.get("/jobs/{job_id}") +def job_page(request: Request, job_id: str, repository=Depends(get_repository)): + templates = request.app.state.templates + try: + job = repository.get_job(job_id) + except KeyError as exc: + raise HTTPException(status_code=404, detail="job not found") from exc + return templates.TemplateResponse(request, "job.html", {"job": job}) diff --git a/matanyone2/webapp/api/routes/uploads.py b/matanyone2/webapp/api/routes/uploads.py new file mode 100644 index 0000000..b4abf0d --- /dev/null +++ b/matanyone2/webapp/api/routes/uploads.py @@ -0,0 +1,44 @@ +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from fastapi.responses import FileResponse + +from matanyone2.webapp.api.dependencies import ( + get_draft_store, + get_masking_service, + get_video_service, +) + + +router = APIRouter() + + +@router.post("/api/uploads") +def upload_video( + video: UploadFile = File(...), + video_service=Depends(get_video_service), + draft_store=Depends(get_draft_store), + masking_service=Depends(get_masking_service), +): + try: + draft = video_service.create_draft_from_upload(video) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + draft_store[draft.draft_id] = masking_service.create_session(draft) + return { + "draft_id": draft.draft_id, + "template_frame_url": f"/api/drafts/{draft.draft_id}/template-frame", + } + + +@router.get("/api/drafts/{draft_id}/template-frame") +def get_template_frame( + draft_id: str, + draft_store=Depends(get_draft_store), +): + session = draft_store.get(draft_id) + if session is None: + raise HTTPException(status_code=404, detail="draft not found") + return FileResponse( + session.draft.template_frame_path, + media_type="image/png", + filename=session.draft.template_frame_path.name, + ) diff --git a/matanyone2/webapp/config.py b/matanyone2/webapp/config.py new file mode 100644 index 0000000..05ec7c6 --- /dev/null +++ b/matanyone2/webapp/config.py @@ -0,0 +1,52 @@ +from dataclasses import dataclass +from pathlib import Path +import os + + +def _default_sam3_checkpoint_path() -> str: + candidates = [ + os.getenv("MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH"), + r"D:\my_app\lens_hunter2\models\sam3\checkpoints\sam3.pt", + str(Path("pretrained_models") / "sam3.pt"), + ] + for candidate in candidates: + if not candidate: + continue + if Path(candidate).exists(): + return str(candidate) + for candidate in candidates: + if candidate: + return str(candidate) + return str(Path("pretrained_models") / "sam3.pt") + + +@dataclass(slots=True) +class WebAppSettings: + runtime_root: Path = Path( + os.getenv("MATANYONE2_WEBAPP_RUNTIME_ROOT", "runtime/webapp") + ) + database_path: Path = Path( + os.getenv("MATANYONE2_WEBAPP_DATABASE_PATH", "runtime/webapp/jobs.db") + ) + max_video_seconds: int = int( + os.getenv("MATANYONE2_WEBAPP_MAX_VIDEO_SECONDS", "10") + ) + max_upload_bytes: int = int( + os.getenv( + "MATANYONE2_WEBAPP_MAX_UPLOAD_BYTES", + str(2 * 1024 * 1024 * 1024), + ) + ) + enable_prores_export: bool = ( + os.getenv("MATANYONE2_WEBAPP_ENABLE_PRORES", "1") == "1" + ) + sam_backend: str = os.getenv("MATANYONE2_WEBAPP_SAM_BACKEND", "sam3") + sam_model_type: str = os.getenv("MATANYONE2_WEBAPP_SAM_MODEL_TYPE", "vit_h") + sam2_variant: str = os.getenv( + "MATANYONE2_WEBAPP_SAM2_VARIANT", + "sam2.1_hiera_large", + ) + sam2_checkpoint_path: str | None = os.getenv( + "MATANYONE2_WEBAPP_SAM2_CHECKPOINT_PATH" + ) + sam3_checkpoint_path: str = _default_sam3_checkpoint_path() diff --git a/matanyone2/webapp/db.py b/matanyone2/webapp/db.py new file mode 100644 index 0000000..43c21f7 --- /dev/null +++ b/matanyone2/webapp/db.py @@ -0,0 +1,32 @@ +from pathlib import Path +import sqlite3 + +from matanyone2.webapp.runtime_paths import ensure_parent_dir + + +def init_database(database_path: Path) -> None: + ensure_parent_dir(database_path) + with sqlite3.connect(database_path) as connection: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + status TEXT NOT NULL, + source_video_path TEXT NOT NULL, + mask_path TEXT NOT NULL, + template_frame_index INTEGER NOT NULL, + params_json TEXT NOT NULL, + warning_text TEXT, + error_text TEXT, + created_at TEXT NOT NULL + ) + """ + ) + connection.commit() + + +def connect(database_path: Path) -> sqlite3.Connection: + ensure_parent_dir(database_path) + connection = sqlite3.connect(database_path) + connection.row_factory = sqlite3.Row + return connection diff --git a/matanyone2/webapp/models.py b/matanyone2/webapp/models.py new file mode 100644 index 0000000..69a5491 --- /dev/null +++ b/matanyone2/webapp/models.py @@ -0,0 +1,151 @@ +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path + +try: + from enum import StrEnum +except ImportError: # pragma: no cover - Python < 3.11 compatibility + class StrEnum(str, Enum): + pass + + +class JobStatus(StrEnum): + QUEUED = "queued" + PREPARING = "preparing" + RUNNING = "running" + EXPORTING = "exporting" + COMPLETED = "completed" + COMPLETED_WITH_WARNING = "completed_with_warning" + FAILED = "failed" + INTERRUPTED = "interrupted" + + +@dataclass(slots=True) +class JobRecord: + job_id: str + status: JobStatus + source_video_path: str + mask_path: str + template_frame_index: int + params_json: str + warning_text: str | None + error_text: str | None + + +@dataclass(slots=True) +class DraftRecord: + draft_id: str + video_path: Path + template_frame_path: Path + width: int + height: int + fps: float + frame_count: int + duration_seconds: float + process_start_frame_index: int = 0 + process_end_frame_index: int = 0 + template_frame_index: int | None = 0 + browser_preview_path: Path | None = None + + +@dataclass(slots=True) +class AnnotationTarget: + target_id: str + name: str + click_points: list[tuple[int, int]] = field(default_factory=list) + click_labels: list[int] = field(default_factory=list) + saved_mask_name: str | None = None + visible: bool = True + locked: bool = False + refine_preset: str = "balanced" + preset_strength: float = 0.5 + motion_strength: float = 0.35 + temporal_stability: float = 0.0 + edge_feather_radius: float = 0.0 + + +@dataclass(slots=True) +class DraftSession: + draft: DraftRecord + session_dir: Path + targets: dict[str, AnnotationTarget] = field(default_factory=dict) + active_target_id: str | None = None + saved_masks: dict[str, Path] = field(default_factory=dict) + saved_mask_presets: dict[str, str] = field(default_factory=dict) + selected_mask_names: set[str] = field(default_factory=set) + current_mask_base_path: Path | None = None + current_mask_path: Path | None = None + current_preview_path: Path | None = None + stage: str = "coarse" + workflow_step: str = "clip" + active_sidebar_tab: str = "targets" + compare_enabled: bool = False + latest_job_id: str | None = None + _target_sequence: int = 0 + + def __post_init__(self): + if not self.targets: + self.create_target() + elif self.active_target_id is None: + self.active_target_id = next(iter(self.targets)) + self._target_sequence = len(self.targets) + + @property + def active_target(self) -> AnnotationTarget: + if self.active_target_id is None or self.active_target_id not in self.targets: + return self.create_target() + return self.targets[self.active_target_id] + + @property + def click_points(self) -> list[tuple[int, int]]: + return self.active_target.click_points + + @click_points.setter + def click_points(self, value: list[tuple[int, int]]): + self.active_target.click_points = list(value) + + @property + def click_labels(self) -> list[int]: + return self.active_target.click_labels + + @click_labels.setter + def click_labels(self, value: list[int]): + self.active_target.click_labels = list(value) + + def create_target(self, name: str | None = None) -> AnnotationTarget: + self._target_sequence += 1 + target = AnnotationTarget( + target_id=f"target-{self._target_sequence:03d}", + name=name or f"Target {self._target_sequence}", + ) + self.targets[target.target_id] = target + self.active_target_id = target.target_id + return target + + def select_target(self, target_id: str) -> AnnotationTarget: + if target_id not in self.targets: + raise KeyError(target_id) + self.active_target_id = target_id + return self.targets[target_id] + + +@dataclass(slots=True) +class MaskingResult: + current_mask_path: Path + current_preview_path: Path + + +@dataclass(slots=True) +class InferenceResult: + foreground_video_path: Path + alpha_video_path: Path + + +@dataclass(slots=True) +class ExportResult: + rgba_png_dir: Path + png_zip_path: Path + preview_foreground_path: Path | None + preview_alpha_path: Path | None + prores_path: Path | None + warning_text: str | None diff --git a/matanyone2/webapp/queue.py b/matanyone2/webapp/queue.py new file mode 100644 index 0000000..f058949 --- /dev/null +++ b/matanyone2/webapp/queue.py @@ -0,0 +1,13 @@ +from matanyone2.webapp.repository import JobRepository + + +class QueueCoordinator: + def __init__(self, repository: JobRepository): + self.repository = repository + + def recover_interrupted_jobs(self) -> None: + self.repository.mark_running_jobs_interrupted() + + def next_job_id(self) -> str | None: + next_job = self.repository.next_queued_job() + return None if next_job is None else next_job.job_id diff --git a/matanyone2/webapp/repository.py b/matanyone2/webapp/repository.py new file mode 100644 index 0000000..ec6d04a --- /dev/null +++ b/matanyone2/webapp/repository.py @@ -0,0 +1,155 @@ +from datetime import datetime, timezone +from pathlib import Path +import sqlite3 +import uuid + +from matanyone2.webapp.db import connect, init_database +from matanyone2.webapp.models import JobRecord, JobStatus + + +class JobRepository: + def __init__(self, database_path: Path): + self.database_path = database_path + init_database(database_path) + + @classmethod + def from_path(cls, database_path: Path) -> "JobRepository": + return cls(database_path=Path(database_path)) + + def create_job( + self, + *, + source_video_path: str, + template_frame_index: int, + mask_path: str, + params_json: str, + ) -> JobRecord: + job = JobRecord( + job_id=uuid.uuid4().hex, + status=JobStatus.QUEUED, + source_video_path=source_video_path, + mask_path=mask_path, + template_frame_index=template_frame_index, + params_json=params_json, + warning_text=None, + error_text=None, + ) + with connect(self.database_path) as connection: + connection.execute( + """ + INSERT INTO jobs ( + job_id, + status, + source_video_path, + mask_path, + template_frame_index, + params_json, + warning_text, + error_text, + created_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + job.job_id, + job.status.value, + job.source_video_path, + job.mask_path, + job.template_frame_index, + job.params_json, + job.warning_text, + job.error_text, + datetime.now(timezone.utc).isoformat(), + ), + ) + connection.commit() + return job + + def get_job(self, job_id: str) -> JobRecord: + with connect(self.database_path) as connection: + row = connection.execute( + """ + SELECT job_id, status, source_video_path, mask_path, template_frame_index, + params_json, warning_text, error_text + FROM jobs + WHERE job_id = ? + """, + (job_id,), + ).fetchone() + if row is None: + raise KeyError(job_id) + return self._row_to_job(row) + + def get_queue_position(self, job_id: str) -> int: + with connect(self.database_path) as connection: + row = connection.execute( + """ + SELECT COUNT(*) + FROM jobs AS q + WHERE q.status = ? + AND q.created_at <= ( + SELECT created_at + FROM jobs + WHERE job_id = ? + ) + """, + (JobStatus.QUEUED.value, job_id), + ).fetchone() + return int(row[0]) + + def update_status( + self, + job_id: str, + status: JobStatus, + *, + warning_text: str | None = None, + error_text: str | None = None, + ) -> None: + with connect(self.database_path) as connection: + connection.execute( + """ + UPDATE jobs + SET status = ?, warning_text = ?, error_text = ? + WHERE job_id = ? + """, + (status.value, warning_text, error_text, job_id), + ) + connection.commit() + + def mark_running_jobs_interrupted(self) -> None: + with connect(self.database_path) as connection: + connection.execute( + "UPDATE jobs SET status = ? WHERE status = ?", + (JobStatus.INTERRUPTED.value, JobStatus.RUNNING.value), + ) + connection.commit() + + def next_queued_job(self) -> JobRecord | None: + with connect(self.database_path) as connection: + row = connection.execute( + """ + SELECT job_id, status, source_video_path, mask_path, template_frame_index, + params_json, warning_text, error_text + FROM jobs + WHERE status = ? + ORDER BY created_at ASC + LIMIT 1 + """, + (JobStatus.QUEUED.value,), + ).fetchone() + if row is None: + return None + return self._row_to_job(row) + + @staticmethod + def _row_to_job(row: sqlite3.Row) -> JobRecord: + return JobRecord( + job_id=row["job_id"], + status=JobStatus(row["status"]), + source_video_path=row["source_video_path"], + mask_path=row["mask_path"], + template_frame_index=row["template_frame_index"], + params_json=row["params_json"], + warning_text=row["warning_text"], + error_text=row["error_text"], + ) diff --git a/matanyone2/webapp/runtime_paths.py b/matanyone2/webapp/runtime_paths.py new file mode 100644 index 0000000..1644a22 --- /dev/null +++ b/matanyone2/webapp/runtime_paths.py @@ -0,0 +1,11 @@ +from pathlib import Path + + +def ensure_dir(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + return path + + +def ensure_parent_dir(path: Path) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + return path diff --git a/matanyone2/webapp/services/__init__.py b/matanyone2/webapp/services/__init__.py new file mode 100644 index 0000000..db4d55d --- /dev/null +++ b/matanyone2/webapp/services/__init__.py @@ -0,0 +1 @@ +"""Service layer for the internal web app.""" diff --git a/matanyone2/webapp/services/export.py b/matanyone2/webapp/services/export.py new file mode 100644 index 0000000..8d3eaac --- /dev/null +++ b/matanyone2/webapp/services/export.py @@ -0,0 +1,403 @@ +from pathlib import Path +import shutil +import subprocess +import zipfile + +import cv2 +import numpy as np +from PIL import Image + +from matanyone2.webapp.models import ExportResult +from matanyone2.webapp.runtime_paths import ensure_dir + + +def compose_rgba_frame(foreground_rgb: np.ndarray, alpha_gray: np.ndarray) -> Image.Image: + rgba = np.dstack([foreground_rgb, alpha_gray]).astype(np.uint8) + return Image.fromarray(rgba, mode="RGBA") + + +class ExportService: + def __init__(self, enable_prores: bool = True): + self.enable_prores = enable_prores + + def export_assets( + self, + foreground_video_path: Path, + alpha_video_path: Path, + job_dir: Path, + *, + motion_strength: float = 0.0, + temporal_stability: float = 0.0, + edge_feather_radius: float = 0.0, + ) -> ExportResult: + foreground_frames, alpha_frames, fps = self._extract_frames( + foreground_video_path, + alpha_video_path, + job_dir, + ) + processed_alpha_frames = self._process_alpha_frames( + alpha_frames, + motion_strength=motion_strength, + temporal_stability=temporal_stability, + edge_feather_radius=edge_feather_radius, + ) + self._overwrite_alpha_frames(alpha_frames, processed_alpha_frames) + self._write_alpha_video(processed_alpha_frames, alpha_video_path, fps=fps) + rgba_png_dir = self._write_rgba_pngs( + foreground_frames, + processed_alpha_frames, + job_dir, + ) + png_zip_path = self._zip_directory(rgba_png_dir, job_dir / "rgba_png.zip") + preview_foreground_path = None + preview_alpha_path = None + + warning_text = None + try: + preview_foreground_path, preview_alpha_path = self._export_preview_videos( + foreground_video_path, + alpha_video_path, + job_dir, + ) + except RuntimeError as exc: + warning_text = str(exc) + + prores_path = None + if self.enable_prores: + try: + prores_path = self._export_prores( + rgba_png_dir, + job_dir / "output_prores4444.mov", + fps=fps, + ) + except RuntimeError as exc: + warning_text = self._merge_warning_text(warning_text, str(exc)) + + return ExportResult( + rgba_png_dir=rgba_png_dir, + png_zip_path=png_zip_path, + preview_foreground_path=preview_foreground_path, + preview_alpha_path=preview_alpha_path, + prores_path=prores_path, + warning_text=warning_text, + ) + + def _extract_frames( + self, + foreground_video_path: Path, + alpha_video_path: Path, + job_dir: Path, + ) -> tuple[list[Path], list[Path], float]: + foreground_dir = ensure_dir(job_dir / "foreground_frames") + alpha_dir = ensure_dir(job_dir / "alpha_frames") + foreground_paths: list[Path] = [] + alpha_paths: list[Path] = [] + + foreground_capture = cv2.VideoCapture(str(foreground_video_path)) + alpha_capture = cv2.VideoCapture(str(alpha_video_path)) + fps = float(foreground_capture.get(cv2.CAP_PROP_FPS) or 0.0) + + try: + frame_index = 0 + while True: + fg_ok, fg_frame = foreground_capture.read() + alpha_ok, alpha_frame = alpha_capture.read() + + if not fg_ok and not alpha_ok: + break + if fg_ok != alpha_ok: + raise RuntimeError("foreground and alpha frame counts do not match") + + foreground_path = foreground_dir / f"{frame_index:04d}.png" + alpha_path = alpha_dir / f"{frame_index:04d}.png" + cv2.imwrite(str(foreground_path), fg_frame) + cv2.imwrite(str(alpha_path), alpha_frame) + foreground_paths.append(foreground_path) + alpha_paths.append(alpha_path) + frame_index += 1 + finally: + foreground_capture.release() + alpha_capture.release() + + if not foreground_paths: + raise RuntimeError("no frames extracted from foreground video") + if fps <= 0: + fps = 24.0 + return foreground_paths, alpha_paths, fps + + def _process_alpha_frames( + self, + alpha_frames: list[Path], + *, + motion_strength: float, + temporal_stability: float, + edge_feather_radius: float, + ) -> list[np.ndarray]: + processed_alpha_frames: list[np.ndarray] = [] + for alpha_path in alpha_frames: + alpha_image = cv2.imread(str(alpha_path), cv2.IMREAD_UNCHANGED) + if alpha_image is None: + raise RuntimeError(f"unable to read alpha frame: {alpha_path}") + if alpha_image.ndim == 3: + alpha_gray = cv2.cvtColor(alpha_image, cv2.COLOR_BGR2GRAY) + else: + alpha_gray = alpha_image + processed_alpha_frames.append( + self._apply_motion_softness(alpha_gray, motion_strength=motion_strength) + ) + + processed_alpha_frames = self._stabilize_alpha_frames( + processed_alpha_frames, + temporal_stability=temporal_stability, + ) + return [ + self._apply_edge_feather(alpha_frame, feather_radius=edge_feather_radius) + for alpha_frame in processed_alpha_frames + ] + + def _overwrite_alpha_frames( + self, + alpha_frames: list[Path], + processed_alpha_frames: list[np.ndarray], + ) -> None: + for alpha_path, alpha_frame in zip(alpha_frames, processed_alpha_frames, strict=True): + cv2.imwrite(str(alpha_path), alpha_frame) + + def _write_rgba_pngs( + self, + foreground_frames: list[Path], + processed_alpha_frames: list[np.ndarray], + job_dir: Path, + ) -> Path: + rgba_dir = ensure_dir(job_dir / "rgba_png") + for index, (foreground_path, alpha_gray) in enumerate( + zip(foreground_frames, processed_alpha_frames, strict=True) + ): + foreground_rgb = cv2.cvtColor( + cv2.imread(str(foreground_path), cv2.IMREAD_COLOR), + cv2.COLOR_BGR2RGB, + ) + rgba_frame = compose_rgba_frame(foreground_rgb, alpha_gray) + rgba_frame.save(rgba_dir / f"{index:04d}.png") + return rgba_dir + + def _apply_motion_softness( + self, + alpha_frame: np.ndarray, + *, + motion_strength: float, + ) -> np.ndarray: + if motion_strength <= 0: + return alpha_frame.astype(np.uint8) + radius = max(1, int(round(1 + (motion_strength * 4)))) + kernel_size = radius if radius % 2 == 1 else radius + 1 + blurred = cv2.GaussianBlur(alpha_frame, (kernel_size, kernel_size), sigmaX=0) + return blurred.astype(np.uint8) + + def _stabilize_alpha_frames( + self, + alpha_frames: list[np.ndarray], + *, + temporal_stability: float, + ) -> list[np.ndarray]: + if temporal_stability <= 0 or len(alpha_frames) <= 1: + return [frame.astype(np.uint8) for frame in alpha_frames] + + stabilized: list[np.ndarray] = [] + window_radius = max(1, int(round(1 + (temporal_stability * 2)))) + for index, current in enumerate(alpha_frames): + start = max(0, index - window_radius) + end = min(len(alpha_frames), index + window_radius + 1) + window = np.stack(alpha_frames[start:end]).astype(np.float32) + median_frame = np.median(window, axis=0) + mean_frame = np.mean(window, axis=0) + current_float = current.astype(np.float32) + + binary = np.where(current > 127, 255, 0).astype(np.uint8) + kernel_size = self._odd_kernel_size(3 + int(round(temporal_stability * 4))) + kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size)) + edge_band = cv2.morphologyEx(binary, cv2.MORPH_GRADIENT, kernel) + semitransparent = ( + ((current > 8) & (current < 247)) + | ((median_frame > 8) & (median_frame < 247)) + ) + edge_mask = (edge_band > 0) | semitransparent + + core_blend = cv2.addWeighted( + current_float, + 1.0 - (temporal_stability * 0.25), + mean_frame, + temporal_stability * 0.25, + 0.0, + ) + edge_blend = cv2.addWeighted( + current_float, + 1.0 - temporal_stability, + median_frame, + temporal_stability, + 0.0, + ) + stabilized_frame = core_blend + stabilized_frame[edge_mask] = edge_blend[edge_mask] + stabilized.append(np.clip(stabilized_frame, 0, 255).astype(np.uint8)) + return stabilized + + def _apply_edge_feather( + self, + alpha_frame: np.ndarray, + *, + feather_radius: float, + ) -> np.ndarray: + feather_radius = max(0.0, float(feather_radius)) + base = np.clip(alpha_frame, 0, 255).astype(np.uint8) + if feather_radius <= 0: + return base + + kernel_size = self._odd_kernel_size(max(3, int(round(min(feather_radius, 3.0))))) + blur_size = self._odd_kernel_size(max(3, int(round((feather_radius * 2.0) + 1.0)))) + kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size)) + binary = np.where(base > 127, 255, 0).astype(np.uint8) + edge_band = cv2.morphologyEx(binary, cv2.MORPH_GRADIENT, kernel) + edge_weight = edge_band.astype(np.float32) / 255.0 + edge_mask = (edge_band > 0) | ((base > 0) & (base < 255)) + blurred = cv2.GaussianBlur( + base, + (blur_size, blur_size), + sigmaX=max(0.8, feather_radius / 2.0), + ) + feathered = base.astype(np.float32) + feathered[edge_mask] = ( + (base.astype(np.float32)[edge_mask] * (1.0 - edge_weight[edge_mask])) + + (blurred.astype(np.float32)[edge_mask] * edge_weight[edge_mask]) + ) + return np.clip(feathered, 0, 255).astype(np.uint8) + + def _write_alpha_video( + self, + alpha_frames: list[np.ndarray], + output_path: Path, + *, + fps: float, + ) -> Path: + if not alpha_frames: + raise RuntimeError("cannot write an empty alpha video") + height, width = alpha_frames[0].shape[:2] + writer = cv2.VideoWriter( + str(output_path), + cv2.VideoWriter_fourcc(*"mp4v"), + fps if fps > 0 else 24.0, + (width, height), + ) + try: + for frame in alpha_frames: + writer.write(cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)) + finally: + writer.release() + return output_path + + def _zip_directory(self, source_dir: Path, zip_path: Path) -> Path: + with zipfile.ZipFile(zip_path, "w") as archive: + for path in sorted(source_dir.rglob("*")): + if path.is_file(): + archive.write(path, arcname=path.relative_to(source_dir)) + return zip_path + + def _export_preview_videos( + self, + foreground_video_path: Path, + alpha_video_path: Path, + job_dir: Path, + ) -> tuple[Path, Path]: + ffmpeg_binary = shutil.which("ffmpeg") + if ffmpeg_binary is None: + raise RuntimeError("ffmpeg is not installed") + + preview_foreground_path = job_dir / "preview_foreground.mp4" + preview_alpha_path = job_dir / "preview_alpha.mp4" + self._export_browser_preview_video( + ffmpeg_binary, + foreground_video_path, + preview_foreground_path, + ) + self._export_browser_preview_video( + ffmpeg_binary, + alpha_video_path, + preview_alpha_path, + ) + return preview_foreground_path, preview_alpha_path + + def _export_browser_preview_video( + self, + ffmpeg_binary: str, + input_path: Path, + output_path: Path, + ) -> Path: + output_path.parent.mkdir(parents=True, exist_ok=True) + command = [ + ffmpeg_binary, + "-y", + "-i", + str(input_path), + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(output_path), + ] + self._run_ffmpeg_command(command) + if not output_path.exists(): + raise RuntimeError(f"ffmpeg did not produce browser preview output: {output_path.name}") + return output_path + + def _export_prores(self, rgba_png_dir: Path, output_path: Path, *, fps: float) -> Path: + first_frame = rgba_png_dir / "0000.png" + if not first_frame.exists(): + raise RuntimeError("rgba png sequence is empty") + + ffmpeg_binary = shutil.which("ffmpeg") + if ffmpeg_binary is None: + raise RuntimeError("ffmpeg is not installed") + + output_path.parent.mkdir(parents=True, exist_ok=True) + command = [ + ffmpeg_binary, + "-y", + "-framerate", + str(fps), + "-i", + str(rgba_png_dir / "%04d.png"), + "-c:v", + "prores_ks", + "-profile:v", + "4444", + "-pix_fmt", + "yuva444p10le", + str(output_path), + ] + self._run_ffmpeg_command(command) + if not output_path.exists(): + raise RuntimeError("ffmpeg did not produce prores output") + return output_path + + def _run_ffmpeg_command(self, command: list[str]) -> None: + completed = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + stderr = completed.stderr.strip() or completed.stdout.strip() or "ffmpeg failed" + raise RuntimeError(stderr) + + @staticmethod + def _merge_warning_text(existing: str | None, incoming: str) -> str: + if not existing: + return incoming + return f"{existing}\n{incoming}" + + @staticmethod + def _odd_kernel_size(size: int) -> int: + return size if size % 2 == 1 else size + 1 diff --git a/matanyone2/webapp/services/inference.py b/matanyone2/webapp/services/inference.py new file mode 100644 index 0000000..ea87c4d --- /dev/null +++ b/matanyone2/webapp/services/inference.py @@ -0,0 +1,287 @@ +from pathlib import Path +import shutil + +import cv2 + +from matanyone2.webapp.models import InferenceResult + + +class InferenceService: + def __init__(self, model_name: str = "MatAnyone 2"): + self.model_name = model_name + + def run_job( + self, + *, + source_video_path: Path, + mask_path: Path, + job_dir: Path, + template_frame_index: int, + process_start_frame_index: int = 0, + process_end_frame_index: int | None = None, + selected_mask_controls: dict[str, dict] | None = None, + selected_mask_presets: dict[str, str] | None = None, + ) -> InferenceResult: + source_video_path = Path(source_video_path) + mask_path = Path(mask_path) + job_dir = Path(job_dir) + clip_source_video_path, relative_anchor_frame_index, _ = self._prepare_processing_clip( + source_video_path=source_video_path, + job_dir=job_dir, + process_start_frame_index=process_start_frame_index, + process_end_frame_index=process_end_frame_index, + template_frame_index=template_frame_index, + ) + inference_hyperparameters = self._resolve_inference_hyperparameters( + source_video_path=clip_source_video_path, + selected_mask_controls=selected_mask_controls or {}, + selected_mask_presets=selected_mask_presets or {}, + ) + if relative_anchor_frame_index > 0: + foreground_path, alpha_path = self._run_bidirectional_job( + source_video_path=clip_source_video_path, + mask_path=mask_path, + job_dir=job_dir, + template_frame_index=relative_anchor_frame_index, + **inference_hyperparameters, + ) + else: + foreground_path, alpha_path = self._run_model( + source_video_path=clip_source_video_path, + mask_path=mask_path, + job_dir=job_dir, + template_frame_index=relative_anchor_frame_index, + **inference_hyperparameters, + ) + return InferenceResult( + foreground_video_path=foreground_path, + alpha_video_path=alpha_path, + ) + + def _prepare_processing_clip( + self, + *, + source_video_path: Path, + job_dir: Path, + process_start_frame_index: int, + process_end_frame_index: int | None, + template_frame_index: int, + ) -> tuple[Path, int, float | None]: + if process_end_frame_index is None and process_start_frame_index == 0: + return source_video_path, template_frame_index, None + + frames, fps = self._read_video_frames(source_video_path) + if not frames: + raise RuntimeError("source video has no readable frames") + + resolved_end = process_end_frame_index if process_end_frame_index is not None else len(frames) - 1 + if process_start_frame_index < 0 or process_start_frame_index > resolved_end: + raise ValueError("processing range start must be before the end") + if resolved_end >= len(frames): + raise ValueError("processing range is out of range for source video") + if not (process_start_frame_index <= template_frame_index <= resolved_end): + raise ValueError("template frame must fall inside the processing range") + + clipped_frames = frames[process_start_frame_index : resolved_end + 1] + clip_path = job_dir / "processing_range.mp4" + self._write_video_frames(clipped_frames, clip_path, fps=fps) + relative_anchor = template_frame_index - process_start_frame_index + return clip_path, relative_anchor, fps + + def _run_bidirectional_job( + self, + *, + source_video_path: Path, + mask_path: Path, + job_dir: Path, + template_frame_index: int, + n_warmup: int, + r_erode: int, + r_dilate: int, + ) -> tuple[Path, Path]: + frames, fps = self._read_video_frames(source_video_path) + if not frames: + raise RuntimeError("source video has no readable frames") + if template_frame_index >= len(frames): + raise ValueError("template frame index is out of range for source video") + + forward_frames = frames[template_frame_index:] + backward_frames = list(reversed(frames[:template_frame_index + 1])) + + staging_dir = job_dir / "bidirectional" + staging_dir.mkdir(parents=True, exist_ok=True) + forward_input = staging_dir / "forward_input.mp4" + backward_input = staging_dir / "backward_input.mp4" + self._write_video_frames(forward_frames, forward_input, fps=fps) + self._write_video_frames(backward_frames, backward_input, fps=fps) + + forward_job_dir = staging_dir / "forward_run" + backward_job_dir = staging_dir / "backward_run" + forward_foreground, forward_alpha = self._run_model( + source_video_path=forward_input, + mask_path=mask_path, + job_dir=forward_job_dir, + template_frame_index=0, + n_warmup=n_warmup, + r_erode=r_erode, + r_dilate=r_dilate, + ) + backward_foreground, backward_alpha = self._run_model( + source_video_path=backward_input, + mask_path=mask_path, + job_dir=backward_job_dir, + template_frame_index=0, + n_warmup=n_warmup, + r_erode=r_erode, + r_dilate=r_dilate, + ) + + foreground_path = job_dir / "foreground.mp4" + alpha_path = job_dir / "alpha.mp4" + self._stitch_pass_outputs( + backward_video_path=backward_foreground, + forward_video_path=forward_foreground, + output_path=foreground_path, + fps=fps, + ) + self._stitch_pass_outputs( + backward_video_path=backward_alpha, + forward_video_path=forward_alpha, + output_path=alpha_path, + fps=fps, + ) + return foreground_path, alpha_path + + def _run_model( + self, + *, + source_video_path: Path, + mask_path: Path, + job_dir: Path, + template_frame_index: int, + n_warmup: int, + r_erode: int, + r_dilate: int, + ) -> tuple[Path, Path]: + del template_frame_index + from inference_matanyone2 import main as run_inference + + job_dir.mkdir(parents=True, exist_ok=True) + + run_inference( + input_path=str(source_video_path), + mask_path=str(mask_path), + output_path=str(job_dir), + ckpt_path="pretrained_models/matanyone2.pth", + n_warmup=n_warmup, + r_erode=r_erode, + r_dilate=r_dilate, + ) + + stem = source_video_path.stem + generated_foreground = job_dir / f"{stem}_fgr.mp4" + generated_alpha = job_dir / f"{stem}_pha.mp4" + foreground_path = job_dir / "foreground.mp4" + alpha_path = job_dir / "alpha.mp4" + shutil.move(generated_foreground, foreground_path) + shutil.move(generated_alpha, alpha_path) + return foreground_path, alpha_path + + def _resolve_inference_hyperparameters( + self, + *, + source_video_path: Path, + selected_mask_controls: dict[str, dict], + selected_mask_presets: dict[str, str], + ) -> dict[str, int]: + width, height = self._read_video_size(source_video_path) + max_side = max(width, height) + min_side = min(width, height) + is_high_resolution = min_side >= 1000 or max_side >= 1700 + + options = { + "n_warmup": 10 if is_high_resolution else 1, + "r_erode": 15 if is_high_resolution else 4, + "r_dilate": 15 if is_high_resolution else 4, + } + + preset_values = set(selected_mask_presets.values()) + max_feather_radius = max( + (float(control.get("edge_feather_radius", 0.0)) for control in selected_mask_controls.values()), + default=0.0, + ) + max_temporal_stability = max( + (float(control.get("temporal_stability", 0.0)) for control in selected_mask_controls.values()), + default=0.0, + ) + + if "hair" in preset_values: + options["n_warmup"] = max(options["n_warmup"], 10) + 2 + options["r_dilate"] += 2 + if "edge" in preset_values: + options["r_erode"] += 1 + if "motion" in preset_values: + options["r_dilate"] += 1 + if max_feather_radius >= 4.0: + options["r_dilate"] += 1 + if max_temporal_stability >= 0.5: + options["n_warmup"] += 1 + + options["n_warmup"] = min(options["n_warmup"], 16) + options["r_erode"] = min(options["r_erode"], 21) + options["r_dilate"] = min(options["r_dilate"], 21) + return options + + def _read_video_frames(self, video_path: Path) -> tuple[list, float]: + capture = cv2.VideoCapture(str(video_path)) + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) + frames = [] + try: + while True: + ok, frame = capture.read() + if not ok: + break + frames.append(frame) + finally: + capture.release() + return frames, fps if fps > 0 else 24.0 + + def _read_video_size(self, video_path: Path) -> tuple[int, int]: + capture = cv2.VideoCapture(str(video_path)) + try: + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + finally: + capture.release() + return width, height + + def _write_video_frames(self, frames: list, output_path: Path, *, fps: float) -> Path: + if not frames: + raise RuntimeError("cannot write an empty video") + height, width = frames[0].shape[:2] + writer = cv2.VideoWriter( + str(output_path), + cv2.VideoWriter_fourcc(*"mp4v"), + fps if fps > 0 else 24.0, + (width, height), + ) + try: + for frame in frames: + writer.write(frame) + finally: + writer.release() + return output_path + + def _stitch_pass_outputs( + self, + *, + backward_video_path: Path, + forward_video_path: Path, + output_path: Path, + fps: float, + ) -> Path: + backward_frames, _ = self._read_video_frames(backward_video_path) + forward_frames, _ = self._read_video_frames(forward_video_path) + restored_backward_frames = list(reversed(backward_frames)) + stitched_frames = restored_backward_frames[:-1] + forward_frames + return self._write_video_frames(stitched_frames, output_path, fps=fps) diff --git a/matanyone2/webapp/services/masking.py b/matanyone2/webapp/services/masking.py new file mode 100644 index 0000000..297fa55 --- /dev/null +++ b/matanyone2/webapp/services/masking.py @@ -0,0 +1,780 @@ +import numpy as np +from pathlib import Path +import sys +import types +from importlib.machinery import ModuleSpec + +from PIL import Image, ImageDraw, ImageFilter + +from matanyone2.webapp.models import AnnotationTarget, DraftRecord, DraftSession, MaskingResult +from matanyone2.webapp.runtime_paths import ensure_dir + + +SAM2_CHECKPOINTS = { + "sam2.1_hiera_tiny": { + "config": "configs/sam2.1/sam2.1_hiera_t.yaml", + "filename": "sam2.1_hiera_tiny.pt", + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_tiny.pt", + }, + "sam2.1_hiera_small": { + "config": "configs/sam2.1/sam2.1_hiera_s.yaml", + "filename": "sam2.1_hiera_small.pt", + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_small.pt", + }, + "sam2.1_hiera_base_plus": { + "config": "configs/sam2.1/sam2.1_hiera_b+.yaml", + "filename": "sam2.1_hiera_base_plus.pt", + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_base_plus.pt", + }, + "sam2.1_hiera_large": { + "config": "configs/sam2.1/sam2.1_hiera_l.yaml", + "filename": "sam2.1_hiera_large.pt", + "url": "https://dl.fbaipublicfiles.com/segment_anything_2/092824/sam2.1_hiera_large.pt", + }, +} + +PRESET_LABELS = { + "balanced": "Balanced", + "hair": "Hair Priority", + "edge": "Edge Priority", + "motion": "Motion Blur", +} + + +def _rewrite_sam3_editable_mapping_if_needed() -> None: + local_repo_root = Path(r"D:\my_app\lens_hunter2\models\sam3\sam3_repo\sam3") + if not local_repo_root.exists(): + return + try: + import __editable___sam3_0_1_0_finder as sam3_editable_finder + except ImportError: + return + + current_root = sam3_editable_finder.MAPPING.get("sam3") + if current_root == str(local_repo_root): + return + + sam3_editable_finder.MAPPING["sam3"] = str(local_repo_root) + for namespace, paths in list(sam3_editable_finder.NAMESPACES.items()): + rewritten = [] + for path in paths: + if current_root and path.startswith(current_root): + rewritten.append(path.replace(current_root, str(local_repo_root), 1)) + else: + rewritten.append(path) + sam3_editable_finder.NAMESPACES[namespace] = rewritten + + +def _install_triton_stub_if_needed() -> None: + try: + import triton # noqa: F401 + import triton.language # noqa: F401 + return + except ImportError: + pass + + triton_module = types.ModuleType("triton") + triton_language_module = types.ModuleType("triton.language") + + def _jit(function=None, **_kwargs): + if function is None: + def _decorator(inner): + return inner + return _decorator + return function + + class _MissingTritonSymbol: + def __call__(self, *args, **kwargs): + raise RuntimeError("SAM3 requested a Triton-only code path that is unavailable on this runtime") + + def _missing_attr(_name): + return _MissingTritonSymbol() + + triton_module.jit = _jit + triton_module.__spec__ = ModuleSpec("triton", loader=None) + triton_language_module.constexpr = object() + triton_language_module.__getattr__ = _missing_attr + triton_language_module.__spec__ = ModuleSpec("triton.language", loader=None) + + sys.modules.setdefault("triton", triton_module) + sys.modules.setdefault("triton.language", triton_language_module) + + +def merge_masks(masks: list[np.ndarray]) -> np.ndarray: + if not masks: + raise ValueError("at least one mask is required") + + merged = np.zeros_like(masks[0], dtype=np.uint8) + for mask in masks: + mask_uint8 = mask.astype(np.uint8) + if mask_uint8.max() <= 1: + mask_uint8 = mask_uint8 * 255 + merged = np.maximum(merged, mask_uint8) + return merged + + +class SamMaskController: + def __init__(self, checkpoint_path: str, model_type: str, device: str): + from hugging_face.tools.interact_tools import SamControler + + self._controller = SamControler(checkpoint_path, model_type, device) + + def first_frame_click(self, image, points, labels, multimask=True): + self._controller.sam_controler.reset_image() + self._controller.sam_controler.set_image(image) + return self._controller.first_frame_click( + image=image, + points=points, + labels=labels, + multimask=multimask, + ) + + +def _render_mask_preview(image: np.ndarray, mask: np.ndarray, points, labels) -> Image.Image: + base = Image.fromarray(image.astype(np.uint8), mode="RGB").convert("RGBA") + overlay = np.zeros((mask.shape[0], mask.shape[1], 4), dtype=np.uint8) + overlay[mask > 0] = np.array([110, 132, 255, 118], dtype=np.uint8) + composite = Image.alpha_composite(base, Image.fromarray(overlay, mode="RGBA")) + draw = ImageDraw.Draw(composite) + + for (x, y), label in zip(points.tolist(), labels.tolist()): + color = (118, 230, 136, 255) if label == 1 else (232, 105, 196, 255) + outline = (255, 143, 96, 255) + draw.ellipse((x - 8, y - 8, x + 8, y + 8), fill=outline) + draw.ellipse((x - 5, y - 5, x + 5, y + 5), fill=color) + + return composite.convert("RGB") + + +class Sam2MaskController: + def __init__(self, predictor): + self._predictor = predictor + + def first_frame_click(self, image, points, labels, multimask=True): + self._predictor.set_image(image) + masks, scores, _ = self._predictor.predict( + point_coords=points, + point_labels=labels, + multimask_output=multimask, + ) + + masks = np.asarray(masks) + scores = np.asarray(scores) + if masks.ndim == 3: + best_index = int(np.argmax(scores)) if scores.size else 0 + mask = masks[best_index] + else: + mask = masks + mask = mask.astype(np.uint8) + painted_image = _render_mask_preview(image, mask, points, labels) + return mask, scores, painted_image + + +class Sam3MaskController: + def __init__(self, predictor_or_model, processor=None): + self._predictor = predictor_or_model if processor is None else None + self._model = predictor_or_model if processor is not None else None + self._processor = processor + + def first_frame_click(self, image, points, labels, multimask=True): + if self._processor is None: + self._predictor.set_image(image) + masks, scores, _ = self._predictor.predict( + point_coords=points, + point_labels=labels, + multimask_output=multimask, + ) + else: + inference_state = self._processor.set_image(Image.fromarray(image.astype(np.uint8))) + masks, scores, _ = self._model.predict_inst( + inference_state, + point_coords=points, + point_labels=labels, + multimask_output=multimask, + ) + + masks = np.asarray(masks) + scores = np.asarray(scores) + if masks.ndim == 3: + best_index = int(np.argmax(scores)) if scores.size else 0 + mask = masks[best_index] + else: + mask = masks + mask = mask.astype(np.uint8) + painted_image = _render_mask_preview(image, mask, points, labels) + return mask, scores, painted_image + + +class MaskingService: + VALID_STAGES = {"coarse", "refine", "preview"} + VALID_REFINE_PRESETS = {"balanced", "hair", "edge", "motion"} + VALID_BRUSH_MODES = {"add", "remove", "feather"} + + def __init__( + self, + *, + runtime_root: Path, + controller_factory=None, + sam_backend: str = "sam3", + sam_model_type: str = "vit_h", + sam2_variant: str = "sam2.1_hiera_large", + sam2_checkpoint_path: str | None = None, + sam3_checkpoint_path: str | None = None, + ): + self.runtime_root = Path(runtime_root) + self.controller_factory = controller_factory + self.sam_backend = sam_backend + self.sam_model_type = sam_model_type + self.sam2_variant = sam2_variant + self.sam2_checkpoint_path = sam2_checkpoint_path + self.sam3_checkpoint_path = sam3_checkpoint_path + self._controller = None + + def create_session(self, draft: DraftRecord) -> DraftSession: + session_dir = ensure_dir(self.runtime_root / "drafts" / draft.draft_id / "annotation") + return DraftSession(draft=draft, session_dir=session_dir) + + def create_target(self, session: DraftSession, name: str | None = None) -> AnnotationTarget: + self._clear_current_render(session) + return session.create_target(name=name) + + def select_target(self, session: DraftSession, target_id: str) -> AnnotationTarget: + target = session.select_target(target_id) + self._hydrate_target_render(session) + return target + + def update_target( + self, + session: DraftSession, + target_id: str, + *, + name: str | None = None, + visible: bool | None = None, + locked: bool | None = None, + refine_preset: str | None = None, + preset_strength: float | None = None, + motion_strength: float | None = None, + temporal_stability: float | None = None, + edge_feather_radius: float | None = None, + ) -> AnnotationTarget: + target = session.targets.get(target_id) + if target is None: + raise KeyError(target_id) + if name is not None: + stripped = name.strip() + if not stripped: + raise ValueError("target name cannot be empty") + target.name = stripped + if visible is not None: + target.visible = visible + if locked is not None: + target.locked = locked + if refine_preset is not None: + if refine_preset not in self.VALID_REFINE_PRESETS: + raise ValueError(f"unknown refine preset: {refine_preset}") + target.refine_preset = refine_preset + if preset_strength is not None: + target.preset_strength = self._validate_unit_interval( + preset_strength, + field_name="preset_strength", + ) + if motion_strength is not None: + target.motion_strength = self._validate_unit_interval( + motion_strength, + field_name="motion_strength", + ) + if temporal_stability is not None: + target.temporal_stability = self._validate_unit_interval( + temporal_stability, + field_name="temporal_stability", + ) + if edge_feather_radius is not None: + target.edge_feather_radius = self._validate_non_negative( + edge_feather_radius, + field_name="edge_feather_radius", + maximum=24.0, + ) + if ( + refine_preset is not None + or preset_strength is not None + or motion_strength is not None + or temporal_stability is not None + or edge_feather_radius is not None + ): + self._rerender_current_from_base(session) + return target + + def set_stage(self, session: DraftSession, stage: str) -> str: + if stage not in self.VALID_STAGES: + raise ValueError(f"unknown stage: {stage}") + session.stage = stage + return session.stage + + def apply_click( + self, + session: DraftSession, + *, + x: int, + y: int, + positive: bool, + ) -> MaskingResult: + if session.draft.template_frame_index is None: + raise ValueError("apply a template frame inside the processing range before editing") + session.click_points = session.click_points + [(x, y)] + session.click_labels = session.click_labels + [1 if positive else 0] + return self._render_active_target(session) + + def apply_brush( + self, + session: DraftSession, + *, + points: list[tuple[int, int]], + mode: str, + radius: int, + ) -> MaskingResult: + if session.draft.template_frame_index is None: + raise ValueError("apply a template frame inside the processing range before editing") + if session.stage == "preview": + raise ValueError("preview mode is read-only") + if session.active_target.locked: + raise ValueError("active target is locked") + if mode not in self.VALID_BRUSH_MODES: + raise ValueError(f"unknown brush mode: {mode}") + if radius < 1: + raise ValueError("brush radius must be at least 1") + if not points: + raise ValueError("at least one brush point is required") + + mask = self._load_editable_mask(session) + mask_image = Image.fromarray(mask, mode="L") + stroke_mask = Image.new("L", mask_image.size, 0) + stroke_draw = ImageDraw.Draw(stroke_mask) + + for x, y in points: + bounds = (x - radius, y - radius, x + radius, y + radius) + stroke_draw.ellipse(bounds, fill=255) + + if mode == "add": + ImageDraw.Draw(mask_image).bitmap((0, 0), stroke_mask, fill=255) + elif mode == "remove": + ImageDraw.Draw(mask_image).bitmap((0, 0), stroke_mask, fill=0) + else: + blurred = mask_image.filter(ImageFilter.GaussianBlur(radius=max(1, radius // 4))) + mask_image = Image.composite(blurred, mask_image, stroke_mask) + + result_mask = np.where(np.array(mask_image, dtype=np.uint8) > 127, 255, 0).astype(np.uint8) + return self._write_current_render(session, result_mask) + + def undo_last_click(self, session: DraftSession) -> MaskingResult | None: + if not session.click_points: + self._clear_current_render(session) + return None + + session.click_points = session.click_points[:-1] + session.click_labels = session.click_labels[:-1] + if not session.click_points: + self._clear_current_render(session) + return None + return self._render_active_target(session) + + def reset_active_target(self, session: DraftSession) -> None: + session.click_points = [] + session.click_labels = [] + self._clear_current_render(session) + + def apply_refine_preset( + self, + mask: np.ndarray, + preset: str, + *, + preset_strength: float = 0.5, + motion_strength: float = 0.35, + ) -> np.ndarray: + if preset not in self.VALID_REFINE_PRESETS: + raise ValueError(f"unknown refine preset: {preset}") + + preset_strength = self._validate_unit_interval( + preset_strength, + field_name="preset_strength", + ) + motion_strength = self._validate_unit_interval( + motion_strength, + field_name="motion_strength", + ) + binary_mask = np.where(mask > 127, 255, 0).astype(np.uint8) + if preset == "balanced": + if motion_strength <= 0: + return binary_mask + softened = Image.fromarray(binary_mask, mode="L").filter( + ImageFilter.GaussianBlur(radius=0.4 + (motion_strength * 1.4)) + ) + return np.where(np.array(softened, dtype=np.uint8) >= 112, 255, 0).astype(np.uint8) + + mask_image = Image.fromarray(binary_mask, mode="L") + if preset == "hair": + filter_size = self._odd_kernel_size(3 + int(round(preset_strength * 4))) + processed = mask_image.filter(ImageFilter.MaxFilter(filter_size)) + elif preset == "edge": + filter_size = self._odd_kernel_size(3 + int(round(preset_strength * 4))) + processed = mask_image.filter(ImageFilter.MinFilter(filter_size)) + else: + blur_radius = 0.8 + (motion_strength * 2.4) + processed = mask_image.filter(ImageFilter.GaussianBlur(radius=blur_radius)) + threshold = max(36, int(round(112 - (preset_strength * 36)))) + return np.where(np.array(processed, dtype=np.uint8) >= threshold, 255, 0).astype(np.uint8) + + return np.where(np.array(processed, dtype=np.uint8) > 127, 255, 0).astype(np.uint8) + + def apply_temporal_stability_preview( + self, + mask: np.ndarray, + *, + temporal_stability: float = 0.0, + ) -> np.ndarray: + temporal_stability = self._validate_unit_interval( + temporal_stability, + field_name="temporal_stability", + ) + binary_mask = np.where(mask > 127, 255, 0).astype(np.uint8) + if temporal_stability <= 0: + return binary_mask + + blur_radius = 0.6 + (temporal_stability * 2.8) + blurred = Image.fromarray(binary_mask, mode="L").filter( + ImageFilter.GaussianBlur(radius=blur_radius) + ) + threshold = max(72, int(round(152 - (temporal_stability * 64)))) + stabilized = np.where(np.array(blurred, dtype=np.uint8) >= threshold, 255, 0).astype(np.uint8) + + filter_size = self._odd_kernel_size(3 + int(round(temporal_stability * 4))) + stabilized_image = Image.fromarray(stabilized, mode="L") + stabilized_image = stabilized_image.filter(ImageFilter.MaxFilter(filter_size)) + stabilized_image = stabilized_image.filter(ImageFilter.MinFilter(filter_size)) + return np.where(np.array(stabilized_image, dtype=np.uint8) > 127, 255, 0).astype(np.uint8) + + def apply_edge_feather( + self, + mask: np.ndarray, + *, + feather_radius: float = 0.0, + ) -> np.ndarray: + feather_radius = self._validate_non_negative( + feather_radius, + field_name="edge_feather_radius", + maximum=24.0, + ) + base_mask = np.clip(mask, 0, 255).astype(np.uint8) + if feather_radius <= 0: + return base_mask + + mask_image = Image.fromarray(base_mask, mode="L") + blurred_image = mask_image.filter(ImageFilter.GaussianBlur(radius=feather_radius)) + binary_mask = np.where(base_mask > 127, 255, 0).astype(np.uint8) + binary_image = Image.fromarray(binary_mask, mode="L") + kernel_size = self._odd_kernel_size(max(3, int(round(min(feather_radius, 3.0))))) + dilated = np.array(binary_image.filter(ImageFilter.MaxFilter(kernel_size)), dtype=np.uint8) + eroded = np.array(binary_image.filter(ImageFilter.MinFilter(kernel_size)), dtype=np.uint8) + edge_band = np.clip(dilated.astype(np.int16) - eroded.astype(np.int16), 0, 255).astype(np.uint8) + edge_weight = edge_band.astype(np.float32) / 255.0 + + base_float = base_mask.astype(np.float32) + blurred_float = np.array(blurred_image, dtype=np.uint8).astype(np.float32) + feathered = (base_float * (1.0 - edge_weight)) + (blurred_float * edge_weight) + return np.clip(feathered, 0, 255).astype(np.uint8) + + def apply_target_controls( + self, + mask: np.ndarray, + target: AnnotationTarget, + ) -> np.ndarray: + refined_mask = self.apply_refine_preset( + mask, + target.refine_preset, + preset_strength=target.preset_strength, + motion_strength=target.motion_strength, + ) + temporal_input = refined_mask if np.any(refined_mask > 0) else np.where(mask > 127, 255, 0).astype(np.uint8) + stabilized_mask = self.apply_temporal_stability_preview( + temporal_input, + temporal_stability=target.temporal_stability, + ) + return self.apply_edge_feather( + stabilized_mask, + feather_radius=target.edge_feather_radius, + ) + + def reset_session_for_template_frame(self, session: DraftSession, *, frame_index: int) -> None: + session.targets = {} + session.active_target_id = None + session.saved_masks = {} + session.saved_mask_presets = {} + session.selected_mask_names = set() + session.stage = "coarse" + session._target_sequence = 0 + session.draft.template_frame_index = frame_index + self._clear_current_render(session) + session.create_target() + + def reset_session_for_processing_range(self, session: DraftSession) -> None: + session.targets = {} + session.active_target_id = None + session.saved_masks = {} + session.saved_mask_presets = {} + session.selected_mask_names = set() + session.stage = "coarse" + session._target_sequence = 0 + session.draft.template_frame_index = None + self._clear_current_render(session) + session.create_target() + + def _render_active_target(self, session: DraftSession) -> MaskingResult: + image = np.array(Image.open(session.draft.template_frame_path).convert("RGB")) + points = np.array(session.click_points, dtype=np.int32) + labels = np.array(session.click_labels, dtype=np.int32) + + mask, _, _ = self._get_controller().first_frame_click( + image=image, + points=points, + labels=labels, + multimask=True, + ) + + session.click_points = [(int(px), int(py)) for px, py in points.tolist()] + session.click_labels = [int(label) for label in labels.tolist()] + return self._write_current_render(session, np.where(mask > 0, 255, 0).astype(np.uint8)) + + def _clear_current_render(self, session: DraftSession) -> None: + session.current_mask_base_path = None + session.current_mask_path = None + session.current_preview_path = None + + def save_current_mask(self, session: DraftSession) -> str: + if session.current_mask_path is None: + raise ValueError("no current mask to save") + mask_name = f"mask_{len(session.saved_masks) + 1:03d}" + saved_mask_path = session.session_dir / f"{mask_name}.png" + current_mask = self._load_current_base_mask(session) + processed_mask = self.apply_target_controls(current_mask, session.active_target) + Image.fromarray(processed_mask, mode="L").save(saved_mask_path) + session.saved_masks[mask_name] = saved_mask_path + session.saved_mask_presets[mask_name] = session.active_target.refine_preset + session.selected_mask_names.add(mask_name) + session.active_target.saved_mask_name = mask_name + session.click_points = [] + session.click_labels = [] + self._hydrate_target_render(session) + return mask_name + + def write_merged_mask(self, session: DraftSession, selected_masks: list[str]) -> Path: + if not selected_masks: + raise ValueError("at least one selected mask is required") + masks = [] + for mask_name in selected_masks: + mask_path = session.saved_masks.get(mask_name) + if mask_path is None: + raise KeyError(mask_name) + masks.append(np.array(Image.open(mask_path).convert("L"))) + merged_mask = merge_masks(masks) + merged_mask_path = session.session_dir / "merged_mask.png" + Image.fromarray(merged_mask, mode="L").save(merged_mask_path) + return merged_mask_path + + def _get_controller(self): + if self._controller is None: + factory = self.controller_factory or self._build_default_controller + self._controller = factory() + return self._controller + + def _load_editable_mask(self, session: DraftSession) -> np.ndarray: + if session.current_mask_base_path is not None and session.current_mask_base_path.exists(): + return np.array(Image.open(session.current_mask_base_path).convert("L"), dtype=np.uint8) + + saved_mask_name = session.active_target.saved_mask_name + if saved_mask_name: + saved_mask_path = session.saved_masks.get(saved_mask_name) + if saved_mask_path is not None and saved_mask_path.exists(): + return np.array(Image.open(saved_mask_path).convert("L"), dtype=np.uint8) + + if session.current_mask_path is not None and session.current_mask_path.exists(): + return np.array(Image.open(session.current_mask_path).convert("L"), dtype=np.uint8) + + height = session.draft.height + width = session.draft.width + return np.zeros((height, width), dtype=np.uint8) + + def _load_current_base_mask(self, session: DraftSession) -> np.ndarray: + if session.current_mask_base_path is not None and session.current_mask_base_path.exists(): + return np.array(Image.open(session.current_mask_base_path).convert("L"), dtype=np.uint8) + return self._load_editable_mask(session) + + def _write_current_render( + self, + session: DraftSession, + mask: np.ndarray, + ) -> MaskingResult: + current_mask_base_path = session.session_dir / "current_mask_base.png" + current_mask_path = session.session_dir / "current_mask.png" + current_preview_path = session.session_dir / "current_preview.png" + base_mask = np.where(mask > 0, 255, 0).astype(np.uint8) + Image.fromarray(base_mask, mode="L").save(current_mask_base_path) + display_mask = self.apply_target_controls(base_mask, session.active_target) + Image.fromarray(display_mask, mode="L").save(current_mask_path) + + image = np.array(Image.open(session.draft.template_frame_path).convert("RGB")) + points = np.array(session.click_points, dtype=np.int32) + labels = np.array(session.click_labels, dtype=np.int32) + painted_image = _render_mask_preview(image, display_mask, points, labels) + + painted_image.save(current_preview_path) + session.current_mask_base_path = current_mask_base_path + session.current_mask_path = current_mask_path + session.current_preview_path = current_preview_path + return MaskingResult( + current_mask_path=current_mask_path, + current_preview_path=current_preview_path, + ) + + def _rerender_current_from_base(self, session: DraftSession) -> None: + if session.current_mask_base_path is not None and session.current_mask_base_path.exists(): + base_mask = np.array(Image.open(session.current_mask_base_path).convert("L"), dtype=np.uint8) + self._write_current_render(session, base_mask) + return + if session.current_mask_path is not None and session.current_mask_path.exists(): + base_mask = np.array(Image.open(session.current_mask_path).convert("L"), dtype=np.uint8) + self._write_current_render(session, base_mask) + return + saved_mask_name = session.active_target.saved_mask_name + if saved_mask_name: + saved_mask_path = session.saved_masks.get(saved_mask_name) + if saved_mask_path is not None and saved_mask_path.exists(): + saved_mask = np.array(Image.open(saved_mask_path).convert("L"), dtype=np.uint8) + self._write_current_render(session, saved_mask) + return + + def _hydrate_target_render(self, session: DraftSession) -> None: + if session.click_points: + self._render_active_target(session) + return + + saved_mask_name = session.active_target.saved_mask_name + if saved_mask_name: + saved_mask_path = session.saved_masks.get(saved_mask_name) + if saved_mask_path is not None and saved_mask_path.exists(): + saved_mask = np.array(Image.open(saved_mask_path).convert("L"), dtype=np.uint8) + self._write_current_render(session, saved_mask) + return + self._clear_current_render(session) + + def _build_default_controller(self): + if self.sam_backend == "sam3": + return self._build_sam3_controller() + if self.sam_backend == "sam2": + return self._build_sam2_controller() + if self.sam_backend == "sam1": + return self._build_sam1_controller() + raise ValueError(f"unknown sam backend: {self.sam_backend}") + + def _build_sam1_controller(self): + from hugging_face.tools.download_util import load_file_from_url + from hugging_face.tools.misc import get_device + + checkpoint_urls = { + "vit_h": "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth", + "vit_l": "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth", + "vit_b": "https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth", + } + checkpoint_path = load_file_from_url( + checkpoint_urls[self.sam_model_type], + model_dir=str(Path("pretrained_models")), + ) + return SamMaskController( + checkpoint_path=checkpoint_path, + model_type=self.sam_model_type, + device=str(get_device()), + ) + + def _build_sam2_controller(self): + from hugging_face.tools.download_util import load_file_from_url + from hugging_face.tools.misc import get_device + + try: + from sam2.build_sam import build_sam2 + from sam2.sam2_image_predictor import SAM2ImagePredictor + except ImportError as exc: + raise RuntimeError( + "SAM2 backend is selected but the 'SAM-2' package is not installed" + ) from exc + + variant = SAM2_CHECKPOINTS.get(self.sam2_variant) + if variant is None: + raise ValueError(f"unknown SAM2 variant: {self.sam2_variant}") + + checkpoint_path = self.sam2_checkpoint_path + if checkpoint_path is None: + checkpoint_path = load_file_from_url( + variant["url"], + model_dir=str(Path("pretrained_models")), + ) + + predictor = SAM2ImagePredictor( + build_sam2( + variant["config"], + checkpoint_path, + device=str(get_device()), + ) + ) + return Sam2MaskController(predictor) + + def _build_sam3_controller(self): + from hugging_face.tools.misc import get_device + + _rewrite_sam3_editable_mapping_if_needed() + _install_triton_stub_if_needed() + try: + from sam3 import build_sam3_image_model + from sam3.model.sam3_image_processor import Sam3Processor + except ImportError as exc: + raise RuntimeError( + "SAM3 backend is selected but the local 'sam3' runtime could not be imported" + ) from exc + + checkpoint_path = self.sam3_checkpoint_path + if not checkpoint_path: + raise RuntimeError("SAM3 backend requires a checkpoint path") + + model = build_sam3_image_model( + checkpoint_path=str(checkpoint_path), + device=str(get_device()), + load_from_HF=False, + enable_inst_interactivity=True, + ) + predictor = getattr(model, "inst_interactive_predictor", None) + if predictor is None: + raise RuntimeError("SAM3 image model did not expose an interactive predictor") + processor = Sam3Processor(model, device=str(get_device())) + return Sam3MaskController(model, processor) + + @staticmethod + def _validate_unit_interval(value: float, *, field_name: str) -> float: + numeric = float(value) + if numeric < 0.0 or numeric > 1.0: + raise ValueError(f"{field_name} must be between 0.0 and 1.0") + return numeric + + @staticmethod + def _validate_non_negative( + value: float, + *, + field_name: str, + maximum: float | None = None, + ) -> float: + numeric = float(value) + if numeric < 0.0: + raise ValueError(f"{field_name} must be non-negative") + if maximum is not None and numeric > maximum: + raise ValueError(f"{field_name} must be at most {maximum}") + return numeric + + @staticmethod + def _odd_kernel_size(size: int) -> int: + return size if size % 2 == 1 else size + 1 diff --git a/matanyone2/webapp/services/video.py b/matanyone2/webapp/services/video.py new file mode 100644 index 0000000..8618c7c --- /dev/null +++ b/matanyone2/webapp/services/video.py @@ -0,0 +1,209 @@ +from pathlib import Path +import shutil +import subprocess +import uuid + +import cv2 +from fastapi import UploadFile + +from matanyone2.webapp.models import DraftRecord +from matanyone2.webapp.runtime_paths import ensure_dir + + +class VideoDraftService: + def __init__( + self, + *, + runtime_root: Path, + max_video_seconds: int, + max_upload_bytes: int, + ): + self.runtime_root = Path(runtime_root) + self.max_video_seconds = max_video_seconds + self.max_upload_bytes = max_upload_bytes + + def create_draft(self, source_video_path: Path) -> DraftRecord: + source_video_path = Path(source_video_path) + if source_video_path.stat().st_size > self.max_upload_bytes: + raise ValueError("video exceeds max upload size") + + draft_id = uuid.uuid4().hex + draft_dir = ensure_dir(self.runtime_root / "drafts" / draft_id) + staged_video_path = draft_dir / source_video_path.name + shutil.copy2(source_video_path, staged_video_path) + + cap = cv2.VideoCapture(str(staged_video_path)) + try: + fps = float(cap.get(cv2.CAP_PROP_FPS) or 0.0) + frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0) + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + ok, frame = cap.read() + finally: + cap.release() + + if not ok or frame_count <= 0 or fps <= 0: + raise ValueError("unable to read video frames") + + duration_seconds = frame_count / fps + if duration_seconds > self.max_video_seconds: + raise ValueError("video exceeds max duration") + + template_frame_path = draft_dir / "template_frame.png" + cv2.imwrite(str(template_frame_path), frame) + browser_preview_path = draft_dir / "preview_source.mp4" + try: + browser_preview_path = self.ensure_browser_preview( + staged_video_path, + preview_path=browser_preview_path, + ) + except RuntimeError: + browser_preview_path = None + + return DraftRecord( + draft_id=draft_id, + video_path=staged_video_path, + browser_preview_path=browser_preview_path, + template_frame_path=template_frame_path, + width=width, + height=height, + fps=fps, + frame_count=frame_count, + duration_seconds=duration_seconds, + process_start_frame_index=0, + process_end_frame_index=frame_count - 1, + template_frame_index=0, + ) + + def create_draft_from_upload(self, upload_file: UploadFile) -> DraftRecord: + uploads_dir = ensure_dir(self.runtime_root / "uploads") + upload_path = uploads_dir / upload_file.filename + upload_path.write_bytes(upload_file.file.read()) + return self.create_draft(upload_path) + + def select_template_frame(self, draft: DraftRecord, frame_index: int) -> DraftRecord: + if frame_index < 0 or frame_index >= draft.frame_count: + raise ValueError("frame index is out of range") + if not (draft.process_start_frame_index <= frame_index <= draft.process_end_frame_index): + raise ValueError("template frame must fall inside the processing range") + frame = self._read_frame(draft.video_path, frame_index) + template_frame_path = draft.template_frame_path.parent / "template_frame.png" + if not cv2.imwrite(str(template_frame_path), frame): + raise ValueError("unable to write template frame") + draft.template_frame_path = template_frame_path + draft.template_frame_index = frame_index + return draft + + def select_processing_range( + self, + draft: DraftRecord, + *, + start_frame_index: int, + end_frame_index: int, + ) -> DraftRecord: + if start_frame_index < 0 or end_frame_index < 0: + raise ValueError("processing range cannot be negative") + if start_frame_index > end_frame_index: + raise ValueError("processing range start must be before the end") + if end_frame_index >= draft.frame_count: + raise ValueError("processing range is out of range") + draft.process_start_frame_index = start_frame_index + draft.process_end_frame_index = end_frame_index + draft.template_frame_index = None + return draft + + def ensure_browser_preview( + self, + video_path: Path, + *, + preview_path: Path | None = None, + ) -> Path: + preview_path = preview_path or self._default_preview_path(video_path) + if preview_path.exists() and preview_path.is_file(): + return preview_path + + ffmpeg_binary = shutil.which("ffmpeg") + if ffmpeg_binary is None: + raise RuntimeError("ffmpeg is not installed") + + preview_path.parent.mkdir(parents=True, exist_ok=True) + command = [ + ffmpeg_binary, + "-y", + "-i", + str(video_path), + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(preview_path), + ] + self._run_ffmpeg_command(command) + if not preview_path.exists(): + raise RuntimeError("ffmpeg did not produce browser preview output") + return preview_path + + def write_processing_range_clip( + self, + video_path: Path, + *, + start_frame_index: int, + end_frame_index: int, + output_path: Path, + ) -> tuple[Path, float]: + capture = cv2.VideoCapture(str(video_path)) + fps = float(capture.get(cv2.CAP_PROP_FPS) or 0.0) + width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH) or 0) + height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT) or 0) + output_path.parent.mkdir(parents=True, exist_ok=True) + writer = cv2.VideoWriter( + str(output_path), + cv2.VideoWriter_fourcc(*"mp4v"), + fps if fps > 0 else 24.0, + (width, height), + ) + frames_written = 0 + try: + capture.set(cv2.CAP_PROP_POS_FRAMES, start_frame_index) + for _ in range(start_frame_index, end_frame_index + 1): + ok, frame = capture.read() + if not ok: + break + writer.write(frame) + frames_written += 1 + finally: + writer.release() + capture.release() + if frames_written == 0: + raise ValueError("unable to extract the requested processing range") + return output_path, fps if fps > 0 else 24.0 + + @staticmethod + def _read_frame(video_path: Path, frame_index: int): + capture = cv2.VideoCapture(str(video_path)) + try: + capture.set(cv2.CAP_PROP_POS_FRAMES, frame_index) + ok, frame = capture.read() + finally: + capture.release() + if not ok: + raise ValueError("unable to read requested frame") + return frame + + @staticmethod + def _default_preview_path(video_path: Path) -> Path: + return video_path.parent / "preview_source.mp4" + + @staticmethod + def _run_ffmpeg_command(command: list[str]) -> None: + completed = subprocess.run( + command, + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + stderr = completed.stderr.strip() or completed.stdout.strip() or "ffmpeg failed" + raise RuntimeError(stderr) diff --git a/matanyone2/webapp/smoke.py b/matanyone2/webapp/smoke.py new file mode 100644 index 0000000..fb288f6 --- /dev/null +++ b/matanyone2/webapp/smoke.py @@ -0,0 +1,367 @@ +from dataclasses import dataclass +from io import BytesIO +from pathlib import Path +import argparse +import os +import subprocess +import sys +import time + +from PIL import Image +import requests + + +TERMINAL_STATUSES = { + "completed", + "completed_with_warning", + "failed", + "interrupted", +} +SUCCESS_STATUSES = {"completed", "completed_with_warning"} + + +@dataclass(slots=True) +class SmokeResult: + runtime_root: Path + job_statuses: dict[str, dict] + + +def wait_for_server( + session: requests.Session, + base_url: str, + *, + timeout_seconds: float, + poll_interval_seconds: float, + sleep=time.sleep, + monotonic=time.monotonic, +) -> None: + deadline = monotonic() + timeout_seconds + while True: + try: + response = session.get(f"{base_url}/", timeout=10) + if response.status_code == 200: + return + except requests.RequestException: + pass + + if monotonic() >= deadline: + raise TimeoutError(f"webapp did not become ready: {base_url}") + sleep(poll_interval_seconds) + + +def submit_job( + session: requests.Session, + base_url: str, + video_path: Path, + *, + click_point: tuple[int, int] | None = None, +) -> str: + with Path(video_path).open("rb") as source: + upload_response = session.post( + f"{base_url}/api/uploads", + files={"video": (video_path.name, source, "video/mp4")}, + timeout=120, + ) + upload_response.raise_for_status() + upload_payload = upload_response.json() + draft_id = upload_payload["draft_id"] + + template_response = session.get( + f"{base_url}{upload_payload['template_frame_url']}", + timeout=120, + ) + template_response.raise_for_status() + with Image.open(BytesIO(template_response.content)) as image: + width, height = image.size + if click_point is None: + click_point = (width // 2, height // 2) + + click_response = session.post( + f"{base_url}/api/drafts/{draft_id}/click", + json={"x": click_point[0], "y": click_point[1], "positive": True}, + timeout=300, + ) + click_response.raise_for_status() + + save_response = session.post( + f"{base_url}/api/drafts/{draft_id}/masks", + timeout=120, + ) + save_response.raise_for_status() + mask_name = save_response.json()["mask_name"] + + submit_response = session.post( + f"{base_url}/api/drafts/{draft_id}/submit", + json={ + "template_frame_index": 0, + "selected_masks": [mask_name], + }, + timeout=120, + ) + submit_response.raise_for_status() + return submit_response.json()["job_id"] + + +def poll_jobs( + session: requests.Session, + base_url: str, + job_ids: list[str], + *, + timeout_seconds: float, + poll_interval_seconds: float, + sleep=time.sleep, + monotonic=time.monotonic, +) -> dict[str, dict]: + deadline = monotonic() + timeout_seconds + statuses = {} + queued_seen = {job_id: False for job_id in job_ids[1:]} + + while True: + all_terminal = True + retry_iteration = False + for index, job_id in enumerate(job_ids): + try: + response = session.get(f"{base_url}/api/jobs/{job_id}", timeout=30) + except requests.RequestException: + retry_iteration = True + all_terminal = False + break + if response.status_code >= 400: + raise RuntimeError(f"failed to fetch status for {job_id}: {response.status_code}") + payload = response.json() + statuses[job_id] = payload + status = payload["status"] + if index > 0 and status == "queued": + queued_seen[job_id] = True + if status not in TERMINAL_STATUSES: + all_terminal = False + + if retry_iteration: + if monotonic() >= deadline: + raise TimeoutError(f"jobs did not finish before timeout: {job_ids}") + sleep(poll_interval_seconds) + continue + if all_terminal: + break + if monotonic() >= deadline: + raise TimeoutError(f"jobs did not finish before timeout: {job_ids}") + sleep(poll_interval_seconds) + + for job_id, seen in queued_seen.items(): + if not seen: + raise AssertionError(f"job {job_id} never entered queued status") + + for job_id, payload in statuses.items(): + if payload["status"] not in SUCCESS_STATUSES: + raise RuntimeError(f"job {job_id} ended with status {payload['status']}") + return statuses + + +def build_service_env(runtime_root: Path, *, enable_prores: bool) -> dict[str, str]: + runtime_root = Path(runtime_root) + env = os.environ.copy() + env["MATANYONE2_WEBAPP_RUNTIME_ROOT"] = str(runtime_root) + env["MATANYONE2_WEBAPP_DATABASE_PATH"] = str(runtime_root / "jobs.db") + env["MATANYONE2_WEBAPP_ENABLE_PRORES"] = "1" if enable_prores else "0" + env["MATANYONE2_WEBAPP_SAM_BACKEND"] = env.get( + "MATANYONE2_WEBAPP_SAM_BACKEND", + "sam3", + ) + if "MATANYONE2_WEBAPP_SAM_MODEL_TYPE" in os.environ: + env["MATANYONE2_WEBAPP_SAM_MODEL_TYPE"] = os.environ[ + "MATANYONE2_WEBAPP_SAM_MODEL_TYPE" + ] + if "MATANYONE2_WEBAPP_SAM2_VARIANT" in os.environ: + env["MATANYONE2_WEBAPP_SAM2_VARIANT"] = os.environ[ + "MATANYONE2_WEBAPP_SAM2_VARIANT" + ] + if "MATANYONE2_WEBAPP_SAM2_CHECKPOINT_PATH" in os.environ: + env["MATANYONE2_WEBAPP_SAM2_CHECKPOINT_PATH"] = os.environ[ + "MATANYONE2_WEBAPP_SAM2_CHECKPOINT_PATH" + ] + if "MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH" in os.environ: + env["MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH"] = os.environ[ + "MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH" + ] + env["PYTHONIOENCODING"] = "utf-8" + return env + + +def start_services( + *, + project_root: Path, + runtime_root: Path, + port: int, + enable_prores: bool, + python_executable: Path | None = None, +) -> tuple[subprocess.Popen, subprocess.Popen]: + runtime_root = Path(runtime_root) + logs_dir = runtime_root / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + python_path = str(python_executable or sys.executable) + env = build_service_env(runtime_root, enable_prores=enable_prores) + + webapp_stdout = (logs_dir / "webapp.out.log").open("w", encoding="utf-8") + webapp_stderr = (logs_dir / "webapp.err.log").open("w", encoding="utf-8") + worker_stdout = (logs_dir / "worker.out.log").open("w", encoding="utf-8") + worker_stderr = (logs_dir / "worker.err.log").open("w", encoding="utf-8") + + webapp_process = subprocess.Popen( + [ + python_path, + "-m", + "uvicorn", + "scripts.run_internal_webapp:app", + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=project_root, + env=env, + stdout=webapp_stdout, + stderr=webapp_stderr, + ) + worker_process = subprocess.Popen( + [python_path, "scripts/run_internal_worker.py"], + cwd=project_root, + env=env, + stdout=worker_stdout, + stderr=worker_stderr, + ) + return webapp_process, worker_process + + +def stop_process_tree(process: subprocess.Popen) -> None: + if process.poll() is not None: + return + if os.name == "nt": + subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return + + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + + +def run_smoke( + *, + project_root: Path, + video_path: Path, + runtime_root: Path, + port: int, + copies: int, + timeout_seconds: float, + poll_interval_seconds: float, + enable_prores: bool, + python_executable: Path | None = None, +) -> SmokeResult: + base_url = f"http://127.0.0.1:{port}" + session = requests.Session() + webapp_process, worker_process = start_services( + project_root=project_root, + runtime_root=runtime_root, + port=port, + enable_prores=enable_prores, + python_executable=python_executable, + ) + + try: + wait_for_server( + session, + base_url, + timeout_seconds=60.0, + poll_interval_seconds=1.0, + ) + job_ids = [submit_job(session, base_url, video_path) for _ in range(copies)] + statuses = poll_jobs( + session, + base_url, + job_ids, + timeout_seconds=timeout_seconds, + poll_interval_seconds=poll_interval_seconds, + ) + return SmokeResult(runtime_root=Path(runtime_root), job_statuses=statuses) + finally: + session.close() + stop_process_tree(worker_process) + stop_process_tree(webapp_process) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run a queue smoke test for the internal webapp.") + parser.add_argument( + "--video", + default="inputs/video/test-sample2.mp4", + help="Path to the sample video used for smoke submission.", + ) + parser.add_argument( + "--runtime-root", + default=None, + help="Runtime directory used for this smoke run.", + ) + parser.add_argument( + "--port", + type=int, + default=8010, + help="Port used for the temporary webapp process.", + ) + parser.add_argument( + "--copies", + type=int, + default=2, + help="How many jobs to submit back-to-back.", + ) + parser.add_argument( + "--timeout-seconds", + type=float, + default=900.0, + help="Maximum total wait time for all jobs to finish.", + ) + parser.add_argument( + "--poll-interval-seconds", + type=float, + default=5.0, + help="Polling interval for job status checks.", + ) + parser.add_argument( + "--disable-prores", + action="store_true", + help="Skip ProRes export during smoke.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + project_root = Path(__file__).resolve().parents[2] + runtime_root = Path(args.runtime_root) if args.runtime_root else ( + project_root / "runtime" / f"smoke-{time.strftime('%Y%m%d-%H%M%S')}" + ) + result = run_smoke( + project_root=project_root, + video_path=project_root / args.video, + runtime_root=runtime_root, + port=args.port, + copies=args.copies, + timeout_seconds=args.timeout_seconds, + poll_interval_seconds=args.poll_interval_seconds, + enable_prores=not args.disable_prores, + ) + + print(f"runtime_root={result.runtime_root}") + for job_id, payload in result.job_statuses.items(): + print(f"{job_id} {payload['status']} {sorted(payload['artifacts'])}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/matanyone2/webapp/static/annotator.js b/matanyone2/webapp/static/annotator.js new file mode 100644 index 0000000..c8898b8 --- /dev/null +++ b/matanyone2/webapp/static/annotator.js @@ -0,0 +1,239 @@ +(function () { + function setStatus(target, message, isError) { + if (!target) { + return; + } + target.textContent = message; + target.dataset.state = isError ? "error" : "info"; + } + + async function parseJson(response) { + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(payload.detail || "Request failed"); + } + return payload; + } + + function withCacheBust(url) { + const separator = url.includes("?") ? "&" : "?"; + return `${url}${separator}t=${Date.now()}`; + } + + function renderSavedMasks(container, maskNames) { + if (!container) { + return; + } + container.innerHTML = ""; + maskNames.forEach((maskName) => { + const label = document.createElement("label"); + const input = document.createElement("input"); + input.type = "checkbox"; + input.name = "mask_name"; + input.value = maskName; + input.checked = true; + label.appendChild(input); + label.append(` ${maskName}`); + container.appendChild(label); + }); + } + + function bindUploadForm() { + const form = document.getElementById("upload-form"); + if (!form) { + return; + } + + const status = document.getElementById("upload-status"); + form.addEventListener("submit", async (event) => { + event.preventDefault(); + const fileInput = form.querySelector('input[type="file"]'); + const file = fileInput?.files?.[0]; + if (!file) { + setStatus(status, "Select a video before submitting.", true); + return; + } + + const body = new FormData(); + body.append("video", file); + setStatus(status, "Uploading video and preparing draft...", false); + + try { + const payload = await parseJson( + await fetch(form.dataset.uploadEndpoint, { + method: "POST", + body, + }) + ); + window.location.assign(`/drafts/${payload.draft_id}/annotate`); + } catch (error) { + setStatus(status, error.message, true); + } + }); + } + + function bindAnnotator() { + const root = document.getElementById("annotator-app"); + if (!root) { + return; + } + + const status = document.getElementById("annotator-status"); + const image = document.getElementById("annotation-image"); + const saveButton = document.getElementById("save-mask"); + const submitButton = document.getElementById("submit-job"); + const savedMaskList = document.getElementById("saved-mask-list"); + const positiveButton = document.getElementById("positive-mode"); + const negativeButton = document.getElementById("negative-mode"); + let positiveMode = true; + + function setMode(nextPositiveMode) { + positiveMode = nextPositiveMode; + positiveButton?.toggleAttribute("data-active", positiveMode); + negativeButton?.toggleAttribute("data-active", !positiveMode); + } + + setMode(true); + + positiveButton?.addEventListener("click", () => setMode(true)); + negativeButton?.addEventListener("click", () => setMode(false)); + + image?.addEventListener("click", async (event) => { + const bounds = image.getBoundingClientRect(); + const scaleX = image.naturalWidth / bounds.width; + const scaleY = image.naturalHeight / bounds.height; + const x = Math.round((event.clientX - bounds.left) * scaleX); + const y = Math.round((event.clientY - bounds.top) * scaleY); + + setStatus(status, "Updating mask preview...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.clickEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ x, y, positive: positiveMode }), + }) + ); + image.src = withCacheBust(payload.current_preview_url); + setStatus(status, `Added ${positiveMode ? "positive" : "negative"} click at ${x}, ${y}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + saveButton?.addEventListener("click", async () => { + setStatus(status, "Saving current mask...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.saveEndpoint, { + method: "POST", + }) + ); + renderSavedMasks(savedMaskList, payload.mask_names || []); + setStatus(status, `Saved ${payload.mask_name}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + submitButton?.addEventListener("click", async () => { + const selectedMasks = Array.from( + root.querySelectorAll('input[name="mask_name"]:checked') + ).map((input) => input.value); + setStatus(status, "Submitting queued job...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.submitEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + template_frame_index: 0, + selected_masks: selectedMasks, + }), + }) + ); + window.location.assign(`${root.dataset.jobPagePrefix}${payload.job_id}`); + } catch (error) { + setStatus(status, error.message, true); + } + }); + } + + function renderArtifacts(container, artifacts) { + if (!container) { + return; + } + container.innerHTML = ""; + Object.entries(artifacts || {}).forEach(([name, url]) => { + const item = document.createElement("li"); + const link = document.createElement("a"); + link.href = url; + link.textContent = name; + item.appendChild(link); + container.appendChild(item); + }); + } + + function bindJobPage() { + const root = document.getElementById("job-app"); + if (!root) { + return; + } + + const status = document.getElementById("job-status"); + const queuePosition = document.getElementById("job-queue-position"); + const message = document.getElementById("job-message"); + const artifactList = document.getElementById("artifact-list"); + const terminalStates = new Set([ + "completed", + "completed_with_warning", + "failed", + "interrupted", + ]); + + async function refreshStatus() { + try { + const payload = await parseJson(await fetch(root.dataset.statusEndpoint)); + if (status) { + status.textContent = payload.status; + } + if (queuePosition) { + queuePosition.textContent = payload.queue_position + ? `Queue position: ${payload.queue_position}` + : ""; + } + if (message) { + message.textContent = payload.warning_text || payload.error_text || ""; + } + renderArtifacts(artifactList, payload.artifacts); + return terminalStates.has(payload.status); + } catch (error) { + setStatus(message, error.message, true); + return false; + } + } + + refreshStatus().then((isTerminal) => { + if (isTerminal) { + return; + } + const intervalMs = Number(root.dataset.pollIntervalMs || "2000"); + const timer = window.setInterval(async () => { + const shouldStop = await refreshStatus(); + if (shouldStop) { + window.clearInterval(timer); + } + }, intervalMs); + }); + } + + document.addEventListener("DOMContentLoaded", () => { + bindUploadForm(); + bindAnnotator(); + bindJobPage(); + }); + + window.MatAnyone2Annotator = { + version: "0.2.0", + }; +})(); diff --git a/matanyone2/webapp/static/results.js b/matanyone2/webapp/static/results.js new file mode 100644 index 0000000..38ff2b5 --- /dev/null +++ b/matanyone2/webapp/static/results.js @@ -0,0 +1,589 @@ +import { + formatDuration, + parseJson, + setStatus, + withCacheBust, +} from "/static/shared.js"; + +const TERMINAL_STATES = new Set([ + "completed", + "completed_with_warning", + "failed", + "interrupted", +]); + +function bindResultsPage() { + const root = document.getElementById("job-app"); + if (!root) { + return; + } + + const statusNode = document.getElementById("job-status"); + const queueNode = document.getElementById("job-queue-position"); + const messageNode = document.getElementById("job-message"); + const artifactList = document.getElementById("artifact-summary-list"); + const targetReviewList = document.getElementById("target-review-list"); + const reviewSummaryList = document.getElementById("review-summary-list"); + const timelineList = document.getElementById("job-timeline"); + const warningPanel = document.getElementById("warning-panel"); + const warningTitle = document.getElementById("warning-title"); + const warningCopy = document.getElementById("warning-copy"); + const previewVideo = document.getElementById("preview-video"); + const previewPlaceholder = document.getElementById("preview-placeholder"); + const previewCaption = document.getElementById("preview-caption"); + const overlayCanvas = document.getElementById("preview-overlay-canvas"); + const overlayForegroundVideo = document.getElementById("overlay-foreground-video"); + const overlayAlphaVideo = document.getElementById("overlay-alpha-video"); + const tabs = Array.from(root.querySelectorAll(".preview-tab")); + + const state = { + mode: "source", + payload: null, + overlayFrameHandle: null, + overlayForegroundUrl: null, + overlayAlphaUrl: null, + currentVideoUrl: null, + overlayVideosBound: false, + foregroundCanvas: document.createElement("canvas"), + alphaCanvas: document.createElement("canvas"), + }; + + const foregroundContext = state.foregroundCanvas.getContext("2d", { willReadFrequently: true }); + const alphaContext = state.alphaCanvas.getContext("2d", { willReadFrequently: true }); + const overlayContext = overlayCanvas?.getContext("2d", { willReadFrequently: true }) || null; + + function previewUrlFor(payload) { + switch (state.mode) { + case "foreground": + return payload.preview_artifacts?.foreground || null; + case "alpha": + return payload.preview_artifacts?.alpha || null; + case "source": + default: + return payload.source_video_url || root.dataset.sourceVideoEndpoint || null; + } + } + + function previewCaptionFor(payload) { + if (state.mode === "overlay") { + return payload.preview_artifacts?.foreground && payload.preview_artifacts?.alpha + ? "Overlay preview is compositing browser-safe foreground and alpha streams over the source plate." + : "Overlay preview becomes available after browser preview streams finish transcoding."; + } + if (state.mode === "alpha") { + return payload.preview_artifacts?.alpha + ? "Alpha preview is showing the browser-safe grayscale matte stream." + : "Alpha preview becomes available after the browser preview stream is ready."; + } + if (state.mode === "foreground") { + return payload.preview_artifacts?.foreground + ? "Foreground preview is showing the browser-safe rendered foreground pass." + : "Foreground preview becomes available after the browser preview stream is ready."; + } + return "Source preview is using a browser-safe transcode for quick comparison against the matte outputs."; + } + + function renderReviewSummary(summary, payload) { + if (!reviewSummaryList || !summary) { + return; + } + + const presetMap = summary.selected_mask_presets || {}; + const presetEntries = Object.entries(presetMap); + const startFrame = summary.process_start_frame_index; + const endFrame = summary.process_end_frame_index; + const rangeDuration = summary.process_range_duration_seconds; + const sourceFps = Number(summary.source_fps || 0); + const hasRange = Number.isInteger(startFrame) && Number.isInteger(endFrame); + + let processRangeLabel = null; + if (hasRange) { + if (Number.isFinite(sourceFps) && sourceFps > 0) { + processRangeLabel = `Frame ${startFrame}-${endFrame} | ${formatDuration(startFrame / sourceFps)} - ${formatDuration(endFrame / sourceFps)}`; + } else { + processRangeLabel = `Frame ${startFrame}-${endFrame}`; + } + } + + const rows = [ + ["Source", summary.source_name || "Unknown source"], + ["Status", payload.status_label || payload.status], + ...(processRangeLabel ? [["Process range", processRangeLabel]] : []), + ["Template frame", `Frame ${summary.template_frame_index ?? 0}`], + ...(rangeDuration ? [["Processed duration", formatDuration(rangeDuration)]] : []), + [ + "Selected targets", + summary.selected_mask_count + ? `${summary.selected_mask_count} matte${summary.selected_mask_count > 1 ? "s" : ""}` + : "No saved target selected", + ], + [ + "Mask set", + Array.isArray(summary.selected_masks) && summary.selected_masks.length + ? summary.selected_masks.join(", ") + : summary.mask_name || "Awaiting export masks", + ], + ]; + + if (presetEntries.length > 0) { + rows.push([ + "Preset strategy", + presetEntries.map(([maskName, preset]) => `${maskName}: ${preset}`).join(" | "), + ]); + } + + if (payload.queue_position) { + rows.splice(2, 0, ["Queue", `Position ${payload.queue_position}`]); + } + + reviewSummaryList.innerHTML = ""; + rows.forEach(([label, value]) => { + const row = document.createElement("div"); + const dt = document.createElement("dt"); + dt.textContent = label; + const dd = document.createElement("dd"); + dd.textContent = value; + row.append(dt, dd); + reviewSummaryList.appendChild(row); + }); + } + + function renderTimeline(timeline) { + if (!timelineList) { + return; + } + timelineList.innerHTML = ""; + (timeline || []).forEach((step) => { + const item = document.createElement("li"); + item.className = "timeline-step"; + item.dataset.state = step.state; + + const dot = document.createElement("span"); + dot.className = "timeline-step__dot"; + dot.setAttribute("aria-hidden", "true"); + + const copy = document.createElement("div"); + copy.className = "timeline-step__copy"; + + const label = document.createElement("p"); + label.className = "timeline-step__label"; + label.textContent = step.label; + + const state = document.createElement("p"); + state.className = "timeline-step__state"; + state.textContent = step.state; + + copy.append(label, state); + item.append(dot, copy); + timelineList.appendChild(item); + }); + } + + function renderWarningPanel(payload) { + if (!warningPanel || !warningTitle || !warningCopy) { + return; + } + + const copy = payload.error_text || payload.warning_text; + if (!copy) { + warningPanel.hidden = true; + warningPanel.dataset.state = "neutral"; + warningTitle.textContent = ""; + warningCopy.textContent = ""; + return; + } + + const hasError = Boolean(payload.error_text); + warningPanel.hidden = false; + warningPanel.dataset.state = hasError ? "error" : "warning"; + warningTitle.textContent = hasError ? "Failure reported" : "Warning"; + warningCopy.textContent = copy; + } + + function renderArtifacts(artifactDetails) { + if (!artifactList) { + return; + } + artifactList.innerHTML = ""; + Object.values(artifactDetails || {}).forEach((artifact) => { + const item = document.createElement("li"); + item.className = "artifact-card"; + item.dataset.available = artifact.available ? "true" : "false"; + + const header = document.createElement("div"); + header.className = "artifact-card__header"; + + const titleGroup = document.createElement("div"); + const label = document.createElement("p"); + label.className = "artifact-card__label"; + label.textContent = artifact.label; + const name = document.createElement("p"); + name.className = "artifact-card__name"; + name.textContent = artifact.name; + titleGroup.append(label, name); + + const stateChip = document.createElement("span"); + stateChip.className = "artifact-card__state"; + stateChip.textContent = artifact.available ? "Ready" : "Pending"; + header.append(titleGroup, stateChip); + + const meta = document.createElement("p"); + meta.className = "artifact-card__meta"; + meta.textContent = artifact.available + ? `${artifact.kind.replace("_", " ")} ยท ${artifact.size_label || "Available"}` + : `${artifact.kind.replace("_", " ")} ยท Waiting for export`; + + item.append(header, meta); + + if (artifact.available && artifact.url) { + const link = document.createElement("a"); + link.className = "artifact-card__link"; + link.href = artifact.url; + link.textContent = "Download"; + item.appendChild(link); + } + + artifactList.appendChild(item); + }); + } + + function renderTargetReview(summary) { + if (!targetReviewList) { + return; + } + + const selectedMasks = Array.isArray(summary?.selected_masks) ? summary.selected_masks : []; + const presetMap = summary?.selected_mask_presets || {}; + targetReviewList.innerHTML = ""; + + if (selectedMasks.length === 0) { + const empty = document.createElement("li"); + empty.className = "target-review-card target-review-card--empty"; + empty.textContent = "No saved targets were selected for this job."; + targetReviewList.appendChild(empty); + return; + } + + selectedMasks.forEach((maskName, index) => { + const item = document.createElement("li"); + item.className = "target-review-card"; + + const header = document.createElement("div"); + header.className = "target-review-card__header"; + + const nameGroup = document.createElement("div"); + const title = document.createElement("p"); + title.className = "target-review-card__title"; + title.textContent = `Target ${index + 1}`; + const subtitle = document.createElement("p"); + subtitle.className = "target-review-card__subtitle"; + subtitle.textContent = maskName; + nameGroup.append(title, subtitle); + + const chip = document.createElement("span"); + chip.className = "artifact-card__state"; + chip.textContent = "Included"; + + header.append(nameGroup, chip); + + const meta = document.createElement("dl"); + meta.className = "target-review-meta"; + + [ + ["Mask", maskName], + ["Preset", presetMap[maskName] || "balanced"], + ["Export", "Merged into current job"], + ].forEach(([labelText, valueText]) => { + const row = document.createElement("div"); + const dt = document.createElement("dt"); + const dd = document.createElement("dd"); + dt.textContent = labelText; + dd.textContent = valueText; + row.append(dt, dd); + meta.appendChild(row); + }); + + item.append(header, meta); + targetReviewList.appendChild(item); + }); + } + + function cancelOverlayLoop() { + if (state.overlayFrameHandle !== null) { + window.cancelAnimationFrame(state.overlayFrameHandle); + state.overlayFrameHandle = null; + } + } + + function clearOverlayCanvas() { + cancelOverlayLoop(); + if (overlayContext && overlayCanvas) { + overlayContext.clearRect(0, 0, overlayCanvas.width, overlayCanvas.height); + } + if (overlayCanvas) { + overlayCanvas.hidden = true; + } + if (overlayForegroundVideo) { + overlayForegroundVideo.pause(); + } + if (overlayAlphaVideo) { + overlayAlphaVideo.pause(); + } + } + + function ensureMediaSource(video, url) { + if (!video || !url) { + return false; + } + if (video.dataset.assetUrl === url) { + return false; + } + video.dataset.assetUrl = url; + video.src = withCacheBust(url); + video.load(); + return true; + } + + function ensurePreviewVideo(url) { + if (!previewVideo || !url) { + return; + } + if (state.currentVideoUrl === url) { + return; + } + state.currentVideoUrl = url; + previewVideo.src = withCacheBust(url); + previewVideo.load(); + } + + function syncOverlayPlayback() { + if (!previewVideo || !overlayForegroundVideo || !overlayAlphaVideo || state.mode !== "overlay") { + return; + } + const targetTime = previewVideo.currentTime || 0; + const tolerance = 0.08; + + [overlayForegroundVideo, overlayAlphaVideo].forEach((video) => { + try { + if (Math.abs((video.currentTime || 0) - targetTime) > tolerance) { + video.currentTime = targetTime; + } + } catch (error) { + // Ignore sync jitter while metadata is still loading. + } + video.playbackRate = previewVideo.playbackRate || 1; + if (previewVideo.paused) { + video.pause(); + } else { + video.play().catch(() => {}); + } + }); + } + + function drawOverlayFrame() { + if ( + state.mode !== "overlay" || + !overlayCanvas || + !overlayContext || + !previewVideo || + !overlayForegroundVideo || + !overlayAlphaVideo + ) { + cancelOverlayLoop(); + return; + } + + if ( + overlayForegroundVideo.readyState < 2 || + overlayAlphaVideo.readyState < 2 || + previewVideo.readyState < 2 + ) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + return; + } + + const width = overlayForegroundVideo.videoWidth || previewVideo.videoWidth; + const height = overlayForegroundVideo.videoHeight || previewVideo.videoHeight; + if (!width || !height) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + return; + } + + if (overlayCanvas.width !== width || overlayCanvas.height !== height) { + overlayCanvas.width = width; + overlayCanvas.height = height; + state.foregroundCanvas.width = width; + state.foregroundCanvas.height = height; + state.alphaCanvas.width = width; + state.alphaCanvas.height = height; + } + + foregroundContext.clearRect(0, 0, width, height); + alphaContext.clearRect(0, 0, width, height); + foregroundContext.drawImage(overlayForegroundVideo, 0, 0, width, height); + alphaContext.drawImage(overlayAlphaVideo, 0, 0, width, height); + + const foregroundFrame = foregroundContext.getImageData(0, 0, width, height); + const alphaFrame = alphaContext.getImageData(0, 0, width, height); + const composed = foregroundFrame.data; + const matte = alphaFrame.data; + + for (let index = 0; index < composed.length; index += 4) { + composed[index + 3] = matte[index]; + } + + overlayContext.clearRect(0, 0, width, height); + overlayContext.putImageData(foregroundFrame, 0, 0); + + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + } + + function bindOverlayVideoSync() { + if (!previewVideo || state.overlayVideosBound) { + return; + } + + const syncAndMaybeDraw = () => { + syncOverlayPlayback(); + if (state.mode === "overlay" && state.overlayFrameHandle === null) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + } + }; + + ["play", "pause", "seeking", "seeked", "timeupdate", "ratechange", "loadeddata"].forEach((eventName) => { + previewVideo.addEventListener(eventName, syncAndMaybeDraw); + }); + + state.overlayVideosBound = true; + } + + function renderOverlayPreview(payload) { + const sourceUrl = payload.source_video_url || root.dataset.sourceVideoEndpoint || null; + const foregroundUrl = payload.preview_artifacts?.foreground || null; + const alphaUrl = payload.preview_artifacts?.alpha || null; + + if (!sourceUrl || !foregroundUrl || !alphaUrl || !previewVideo || !overlayCanvas) { + clearOverlayCanvas(); + previewVideo?.removeAttribute("src"); + previewVideo?.load(); + if (previewVideo) { + previewVideo.hidden = true; + } + if (previewPlaceholder) { + previewPlaceholder.hidden = false; + previewPlaceholder.textContent = "Waiting for foreground and alpha streams for overlay preview."; + } + return; + } + + bindOverlayVideoSync(); + ensurePreviewVideo(sourceUrl); + ensureMediaSource(overlayForegroundVideo, foregroundUrl); + ensureMediaSource(overlayAlphaVideo, alphaUrl); + if (previewPlaceholder) { + previewPlaceholder.hidden = true; + } + previewVideo.hidden = false; + overlayCanvas.hidden = false; + syncOverlayPlayback(); + if (state.overlayFrameHandle === null) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + } + } + + function renderStandardPreview(url) { + clearOverlayCanvas(); + if (!previewVideo || !previewPlaceholder) { + return; + } + if (!url) { + state.currentVideoUrl = null; + previewVideo.removeAttribute("src"); + previewVideo.load(); + previewVideo.hidden = true; + previewPlaceholder.hidden = false; + previewPlaceholder.textContent = "Waiting for the selected preview stream."; + return; + } + previewPlaceholder.hidden = true; + previewVideo.hidden = false; + ensurePreviewVideo(url); + } + + function renderPreview(payload) { + if (!previewCaption) { + return; + } + + previewCaption.textContent = previewCaptionFor(payload); + if (state.mode === "overlay") { + renderOverlayPreview(payload); + return; + } + renderStandardPreview(previewUrlFor(payload)); + } + + function renderPayload(payload) { + state.payload = payload; + if (statusNode) { + statusNode.textContent = payload.status_label || payload.status; + } + if (queueNode) { + queueNode.textContent = payload.queue_position + ? `Queue position: ${payload.queue_position}` + : ""; + } + if (messageNode) { + messageNode.textContent = payload.warning_text || payload.error_text || ""; + messageNode.dataset.state = payload.error_text ? "error" : "info"; + } + renderReviewSummary(payload.job_summary, payload); + renderTargetReview(payload.job_summary); + renderTimeline(payload.timeline); + renderWarningPanel(payload); + tabs.forEach((tab) => { + tab.toggleAttribute("data-active", tab.dataset.mode === state.mode); + }); + renderArtifacts(payload.artifact_details); + renderPreview(payload); + } + + async function refreshStatus() { + const payload = await parseJson(await fetch(root.dataset.statusEndpoint)); + renderPayload(payload); + return TERMINAL_STATES.has(payload.status); + } + + tabs.forEach((tab) => { + tab.addEventListener("click", () => { + state.mode = tab.dataset.mode; + if (state.payload) { + renderPayload(state.payload); + } + }); + }); + + refreshStatus() + .then((isTerminal) => { + if (isTerminal) { + return; + } + const intervalMs = Number(root.dataset.pollIntervalMs || "2000"); + const timer = window.setInterval(async () => { + try { + const shouldStop = await refreshStatus(); + if (shouldStop) { + window.clearInterval(timer); + } + } catch (error) { + window.clearInterval(timer); + setStatus(messageNode, error.message, true); + } + }, intervalMs); + }) + .catch((error) => { + setStatus(messageNode, error.message, true); + }); +} + +document.addEventListener("DOMContentLoaded", bindResultsPage); diff --git a/matanyone2/webapp/static/shared.js b/matanyone2/webapp/static/shared.js new file mode 100644 index 0000000..0248c29 --- /dev/null +++ b/matanyone2/webapp/static/shared.js @@ -0,0 +1,76 @@ +export function setStatus(target, message, isError = false) { + if (!target) { + return; + } + target.textContent = message; + target.dataset.state = isError ? "error" : "info"; +} + +export async function parseJson(response) { + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(payload.detail || "Request failed"); + } + return payload; +} + +export function withCacheBust(url) { + if (!url) { + return url; + } + const separator = url.includes("?") ? "&" : "?"; + return `${url}${separator}t=${Date.now()}`; +} + +export function formatBytes(bytes) { + if (!Number.isFinite(bytes) || bytes <= 0) { + return "-"; + } + const units = ["B", "KB", "MB", "GB"]; + let value = bytes; + let unitIndex = 0; + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024; + unitIndex += 1; + } + return `${value.toFixed(value >= 10 || unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`; +} + +export function formatDuration(seconds) { + if (!Number.isFinite(seconds) || seconds <= 0) { + return "-"; + } + const wholeSeconds = Math.round(seconds); + const minutes = Math.floor(wholeSeconds / 60); + const remainder = wholeSeconds % 60; + return `${minutes}:${String(remainder).padStart(2, "0")}`; +} + +export async function probeVideoFile(file) { + return new Promise((resolve, reject) => { + const objectUrl = URL.createObjectURL(file); + const video = document.createElement("video"); + const cleanup = () => { + URL.revokeObjectURL(objectUrl); + video.removeAttribute("src"); + video.load(); + }; + + video.preload = "metadata"; + video.muted = true; + video.onloadedmetadata = () => { + const result = { + width: video.videoWidth, + height: video.videoHeight, + duration: video.duration, + }; + cleanup(); + resolve(result); + }; + video.onerror = () => { + cleanup(); + reject(new Error("Unable to read local video metadata.")); + }; + video.src = objectUrl; + }); +} diff --git a/matanyone2/webapp/static/styles.css b/matanyone2/webapp/static/styles.css new file mode 100644 index 0000000..a8dbbbd --- /dev/null +++ b/matanyone2/webapp/static/styles.css @@ -0,0 +1,1622 @@ +:root { + color-scheme: dark; + --bg: #0a0e14; + --bg-elevated: rgba(14, 20, 30, 0.9); + --bg-muted: rgba(16, 22, 34, 0.76); + --bg-highlight: linear-gradient(180deg, rgba(18, 31, 48, 0.92), rgba(11, 18, 29, 0.96)); + --border: rgba(118, 143, 173, 0.18); + --border-strong: rgba(117, 199, 255, 0.34); + --text: #edf3fb; + --text-muted: #9cadc3; + --text-soft: #6e7f95; + --accent: #76c7ff; + --accent-strong: #2f9be0; + --success: #58c59b; + --warning: #f2b366; + --danger: #f06c60; + --shadow-soft: 0 24px 80px rgba(0, 0, 0, 0.32); + --shadow-panel: 0 10px 30px rgba(2, 7, 14, 0.32); + --radius-xl: 28px; + --radius-lg: 20px; + --radius-md: 14px; + --radius-sm: 10px; + --space-2: 0.5rem; + --space-3: 0.75rem; + --space-4: 1rem; + --space-5: 1.25rem; + --space-6: 1.5rem; + --space-8: 2rem; + --space-10: 2.5rem; + --space-12: 3rem; + --font-sans: "IBM Plex Sans", "Segoe UI", "Helvetica Neue", sans-serif; + --font-mono: "IBM Plex Mono", "SFMono-Regular", Consolas, monospace; +} + +* { + box-sizing: border-box; +} + +html, +body { + min-height: 100%; +} + +body { + margin: 0; + font-family: var(--font-sans); + color: var(--text); + background: + radial-gradient(circle at top left, rgba(73, 117, 161, 0.18), transparent 32%), + radial-gradient(circle at top right, rgba(28, 83, 132, 0.18), transparent 24%), + linear-gradient(180deg, #0b1017 0%, #080c12 100%); +} + +body::before { + content: ""; + position: fixed; + inset: 0; + pointer-events: none; + background-image: + linear-gradient(rgba(255, 255, 255, 0.015) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.015) 1px, transparent 1px); + background-size: 48px 48px; + mask-image: linear-gradient(180deg, rgba(0, 0, 0, 0.75), transparent 96%); +} + +a { + color: var(--accent); +} + +img { + max-width: 100%; + display: block; +} + +button, +input, +textarea, +select { + font: inherit; +} + +button { + cursor: pointer; +} + +.app-shell { + min-height: 100vh; + padding: var(--space-8); + display: flex; + flex-direction: column; + gap: var(--space-8); +} + +.topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-6); + padding: var(--space-5) var(--space-6); + border: 1px solid var(--border); + border-radius: var(--radius-xl); + background: rgba(9, 14, 22, 0.78); + backdrop-filter: blur(22px); + box-shadow: var(--shadow-panel); +} + +.topbar__brand, +.topbar__status { + display: flex; + align-items: center; + gap: var(--space-4); +} + +.topbar__status { + flex-wrap: wrap; + justify-content: flex-end; +} + +.brand-mark { + width: 3rem; + height: 3rem; + border-radius: 1rem; + display: grid; + place-items: center; + color: #06111b; + font-weight: 700; + letter-spacing: 0.08em; + background: linear-gradient(135deg, #8fd5ff, #52aaf0); + box-shadow: 0 14px 32px rgba(39, 132, 194, 0.34); +} + +.brand-title, +.page-title, +.panel__title { + margin: 0; +} + +.brand-title { + font-size: 1.125rem; + font-weight: 600; +} + +.eyebrow, +.panel__eyebrow, +.dropzone__eyebrow { + margin: 0 0 0.35rem; + text-transform: uppercase; + letter-spacing: 0.14em; + font-size: 0.72rem; + color: var(--text-soft); +} + +.status-chip { + display: inline-flex; + align-items: center; + min-height: 2.25rem; + padding: 0 0.9rem; + border-radius: 999px; + border: 1px solid rgba(118, 199, 255, 0.24); + background: rgba(16, 31, 44, 0.72); + color: var(--text); + font-size: 0.9rem; +} + +.status-chip--muted { + border-color: rgba(118, 143, 173, 0.18); + color: var(--text-muted); +} + +.workspace { + flex: 1; + display: flex; + flex-direction: column; + gap: var(--space-8); +} + +.page { + display: flex; + flex-direction: column; + gap: var(--space-8); +} + +.page-intro { + max-width: 62rem; +} + +.page-title { + font-size: clamp(2rem, 4vw, 3.4rem); + line-height: 1.05; + letter-spacing: -0.04em; +} + +.page-copy { + margin: var(--space-4) 0 0; + max-width: 46rem; + font-size: 1.02rem; + line-height: 1.7; + color: var(--text-muted); +} + +.upload-grid { + display: grid; + grid-template-columns: minmax(18rem, 1fr) minmax(28rem, 1.35fr) minmax(20rem, 1fr); + gap: var(--space-6); + align-items: stretch; +} + +.panel { + position: relative; + min-height: 24rem; + padding: var(--space-6); + border-radius: var(--radius-xl); + border: 1px solid var(--border); + background: var(--bg-elevated); + box-shadow: var(--shadow-soft); + overflow: hidden; +} + +.panel::after { + content: ""; + position: absolute; + inset: 0; + pointer-events: none; + border-radius: inherit; + background: linear-gradient(180deg, rgba(255, 255, 255, 0.04), transparent 30%); +} + +.panel--muted { + background: var(--bg-muted); +} + +.panel--highlight { + background: var(--bg-highlight); + border-color: var(--border-strong); +} + +.panel--elevated { + background: + linear-gradient(180deg, rgba(17, 24, 35, 0.96), rgba(11, 17, 27, 0.92)), + rgba(14, 20, 30, 0.92); +} + +.panel__header { + display: flex; + flex-direction: column; + gap: var(--space-2); + margin-bottom: var(--space-6); +} + +.panel__title { + font-size: 1.35rem; + font-weight: 600; +} + +.panel-note, +.status-text { + color: var(--text-muted); + line-height: 1.6; +} + +.status-text[data-state="error"] { + color: #ffb3ac; +} + +.status-text[data-state="info"] { + color: var(--text-muted); +} + +.meta-list { + margin: 0; +} + +.meta-list--stacked { + display: grid; + gap: var(--space-4); +} + +.meta-list--stacked div { + display: grid; + gap: 0.2rem; + padding-bottom: var(--space-4); + border-bottom: 1px solid rgba(255, 255, 255, 0.06); +} + +.meta-list--stacked div:last-child { + padding-bottom: 0; + border-bottom: 0; +} + +.meta-list dt { + font-size: 0.78rem; + text-transform: uppercase; + letter-spacing: 0.12em; + color: var(--text-soft); +} + +.meta-list dd { + margin: 0; + font-size: 0.98rem; + line-height: 1.6; + color: var(--text); +} + +.dropzone { + min-height: 18rem; + display: flex; + flex-direction: column; + justify-content: center; + gap: var(--space-4); + padding: var(--space-8); + border-radius: calc(var(--radius-xl) - 8px); + border: 1px dashed rgba(118, 199, 255, 0.42); + background: + radial-gradient(circle at top, rgba(118, 199, 255, 0.1), transparent 42%), + rgba(8, 14, 22, 0.54); + transition: border-color 180ms ease, transform 180ms ease, background-color 180ms ease; + outline: none; +} + +.dropzone:hover, +.dropzone:focus-visible, +.dropzone[data-dragging="true"] { + border-color: rgba(143, 213, 255, 0.9); + background: + radial-gradient(circle at top, rgba(118, 199, 255, 0.16), transparent 48%), + rgba(10, 20, 31, 0.82); + transform: translateY(-2px); +} + +.dropzone__title { + font-size: 1.55rem; + line-height: 1.15; + letter-spacing: -0.03em; +} + +.dropzone__copy { + max-width: 30rem; + color: var(--text-muted); + line-height: 1.7; +} + +.form-actions { + display: flex; + align-items: center; + gap: var(--space-3); + margin-top: var(--space-6); +} + +.button { + min-height: 3rem; + padding: 0 1.15rem; + border-radius: 999px; + border: 1px solid transparent; + transition: transform 160ms ease, border-color 160ms ease, background-color 160ms ease; +} + +.button:hover { + transform: translateY(-1px); +} + +.button:disabled { + opacity: 0.42; + cursor: not-allowed; + transform: none; +} + +.button--primary { + color: #06111b; + font-weight: 600; + background: linear-gradient(135deg, #8fd5ff, #52aaf0); + box-shadow: 0 14px 28px rgba(55, 143, 203, 0.32); +} + +.button--ghost { + color: var(--text); + border-color: rgba(255, 255, 255, 0.1); + background: rgba(255, 255, 255, 0.03); +} + +.media-card__preview { + min-height: 8rem; + display: grid; + place-items: center; + margin-bottom: var(--space-6); + border-radius: calc(var(--radius-xl) - 12px); + border: 1px solid rgba(255, 255, 255, 0.08); + background: + linear-gradient(135deg, rgba(118, 199, 255, 0.12), rgba(255, 255, 255, 0.02)), + rgba(7, 12, 20, 0.72); + color: var(--text-soft); + font-family: var(--font-mono); + text-transform: uppercase; + letter-spacing: 0.08em; + text-align: center; +} + +.media-card__preview[data-ready="true"] { + color: var(--accent); +} + +.workbench-shell { + display: grid; + grid-template-columns: minmax(22rem, 25rem) minmax(0, 1fr); + gap: var(--space-6); + align-items: start; +} + +.workbench-main, +.workbench-sidebar { + display: flex; + flex-direction: column; + gap: var(--space-5); +} + +.workbench-workflow { + min-height: auto; + position: sticky; + top: var(--space-8); + display: flex; + flex-direction: column; + gap: var(--space-4); + max-height: calc(100dvh - (var(--space-8) * 2)); + overflow: auto; +} + +.workbench-sidebar { + min-width: 0; +} + +.sidebar-panel, +.workbench-header, +.stage-switcher, +.canvas-stage { + min-height: auto; +} + +.workflow-header { + margin-bottom: 0; +} + +.workflow-section { + display: grid; + gap: var(--space-4); + padding-top: var(--space-4); + border-top: 1px solid rgba(255, 255, 255, 0.08); +} + +.workflow-section:first-of-type { + padding-top: 0; + border-top: 0; +} + +.workflow-section__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-3); +} + +.workflow-section__title { + margin: 0; + font-size: 1rem; + font-weight: 600; +} + +.workflow-section--summary, +.workflow-section--commit { + gap: var(--space-3); +} + +.workflow-subpanel { + display: grid; + gap: var(--space-3); + padding: var(--space-4); + border-radius: calc(var(--radius-lg) - 4px); + border: 1px solid rgba(255, 255, 255, 0.05); + background: rgba(255, 255, 255, 0.02); +} + +.workflow-subpanel--nested { + padding: var(--space-3); +} + +.workflow-subpanel__header { + display: grid; + gap: var(--space-2); +} + +.workflow-action-bar { + display: grid; + grid-template-columns: 1fr; + gap: var(--space-3); +} + +.meta-list--compact { + gap: var(--space-3); +} + +.meta-list--compact div { + padding-bottom: var(--space-3); +} + +.guidance-card--compact { + margin-top: 0; + padding-top: 0; + border-top: 0; +} + +.workflow-inline-button { + min-height: 2.4rem; +} + +.workflow-inline-actions, +.tool-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-3); +} + +.preset-grid, +.slider-stack, +.tool-stack { + display: grid; + gap: var(--space-3); +} + +.keyframe-selection-summary { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: var(--space-2); + color: var(--text-soft); + font-size: 0.78rem; + font-family: var(--font-mono); +} + +.workflow-field { + display: grid; + gap: var(--space-2); +} + +.workflow-field__label { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-3); + color: var(--text-muted); + font-size: 0.9rem; +} + +.workflow-input, +.slider-stack input[type="range"] { + width: 100%; +} + +.workflow-input { + min-height: 2.85rem; + padding: 0 0.9rem; + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.1); + background: rgba(255, 255, 255, 0.03); + color: var(--text); +} + +.tool-button--compact { + min-height: 2.6rem; + padding: 0.75rem 0.9rem; +} + +.tool-button, +.stage-button, +.target-card__select { + width: 100%; + min-height: 3.1rem; + padding: 0.95rem 1rem; + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.03); + color: var(--text); + text-align: left; + transition: border-color 160ms ease, background-color 160ms ease, transform 160ms ease; +} + +.tool-button:hover, +.stage-button:hover, +.target-card__select:hover { + transform: translateY(-1px); + border-color: rgba(118, 199, 255, 0.34); +} + +.tool-button:disabled, +.button:disabled { + border-color: rgba(255, 255, 255, 0.06); + background: rgba(255, 255, 255, 0.02); + color: var(--text-soft); +} + +.tool-button[data-active], +.stage-button[data-active], +.preset-button[data-active], +.target-card[data-selected="true"] .target-card__select { + border-color: rgba(118, 199, 255, 0.48); + background: rgba(34, 75, 108, 0.28); +} + +.tool-button--ghost { + background: rgba(255, 255, 255, 0.02); +} + +.workbench-meta { + display: flex; + gap: var(--space-3); + flex-wrap: wrap; + color: var(--text-muted); + font-size: 0.92rem; +} + +.canvas-stage { + display: flex; + flex-direction: column; + gap: var(--space-5); +} + +.canvas-keyframe-panel { + display: grid; + gap: var(--space-3); + padding: var(--space-5); + border-radius: calc(var(--radius-xl) - 10px); + border: 1px solid rgba(255, 255, 255, 0.06); + background: rgba(255, 255, 255, 0.025); +} + +.canvas-stage--monitor { + gap: var(--space-4); +} + +.monitor-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-5); +} + +.monitor-header__copy { + display: grid; + gap: var(--space-3); +} + +.monitor-header__controls { + display: grid; + gap: var(--space-3); + justify-items: end; +} + +.monitor-pill-row { + display: flex; + align-items: center; + gap: var(--space-3); + flex-wrap: wrap; +} + +.stage-button--pill, +.canvas-view-tab { + min-height: 2.65rem; + padding: 0.72rem 1rem; + border-radius: 999px; + text-align: center; +} + +.canvas-meta--compact { + justify-content: flex-start; +} + +.timeline-control-group { + display: grid; + gap: var(--space-3); + padding: var(--space-4); + border-radius: calc(var(--radius-lg) - 4px); + border: 1px solid rgba(255, 255, 255, 0.05); + background: rgba(255, 255, 255, 0.02); +} + +.canvas-keyframe-panel__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-4); +} + +.canvas-keyframe-panel__header--monitor { + align-items: center; +} + +.canvas-meta, +.canvas-toolbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-4); +} + +.canvas-view-tabs { + display: flex; + align-items: center; + gap: var(--space-3); + flex-wrap: wrap; +} + +.canvas-view-tab { + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.03); + color: var(--text); +} + +.canvas-view-tab[data-active] { + border-color: rgba(118, 199, 255, 0.48); + background: rgba(34, 75, 108, 0.28); +} + +.canvas-frame { + display: grid; + place-items: center; + min-height: min(70vh, 54rem); + padding: var(--space-4); + border-radius: calc(var(--radius-xl) - 10px); + border: 1px solid rgba(255, 255, 255, 0.08); + background: + linear-gradient(45deg, rgba(255, 255, 255, 0.03) 25%, transparent 25%), + linear-gradient(-45deg, rgba(255, 255, 255, 0.03) 25%, transparent 25%), + linear-gradient(45deg, transparent 75%, rgba(255, 255, 255, 0.03) 75%), + linear-gradient(-45deg, transparent 75%, rgba(255, 255, 255, 0.03) 75%), + rgba(7, 12, 20, 0.72); + background-size: 24px 24px; + background-position: 0 0, 0 12px, 12px -12px, -12px 0; +} + +.canvas-media { + width: 100%; + max-height: 70vh; + object-fit: contain; + border-radius: calc(var(--radius-lg) - 6px); + background: #03060a; +} + +.canvas-media[hidden] { + display: none; +} + +.preview-compare-strip { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-4); +} + +.preview-compare-strip--compact { + gap: var(--space-3); +} + +.preview-compare-card { + display: grid; + gap: var(--space-3); + padding: var(--space-3); + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.06); + background: rgba(255, 255, 255, 0.03); +} + +.preview-compare-card img { + width: 100%; + max-height: 9rem; + object-fit: contain; + border-radius: calc(var(--radius-md) - 6px); + border: 1px solid rgba(255, 255, 255, 0.08); + background: + linear-gradient(45deg, rgba(255, 255, 255, 0.03) 25%, transparent 25%), + linear-gradient(-45deg, rgba(255, 255, 255, 0.03) 25%, transparent 25%), + linear-gradient(45deg, transparent 75%, rgba(255, 255, 255, 0.03) 75%), + linear-gradient(-45deg, transparent 75%, rgba(255, 255, 255, 0.03) 75%), + rgba(5, 10, 16, 0.88); + background-size: 20px 20px; + background-position: 0 0, 0 10px, 10px -10px, -10px 0; +} + +#annotation-image { + transition: opacity 160ms ease, box-shadow 160ms ease; +} + +#annotation-image[data-editable="true"] { + box-shadow: 0 0 0 1px rgba(118, 199, 255, 0.12); +} + +#annotation-image[data-editable="false"] { + opacity: 0.92; +} + +.target-list { + display: grid; + gap: var(--space-3); +} + +.target-card { + display: grid; + gap: var(--space-3); + padding: var(--space-3); + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.06); + background: rgba(255, 255, 255, 0.03); +} + +.target-card[data-hidden="true"] { + opacity: 0.72; +} + +.target-card[data-selected="true"] { + border-color: rgba(118, 199, 255, 0.26); + background: rgba(17, 33, 48, 0.72); +} + +.target-card[data-locked="true"] .target-card__select { + border-color: rgba(242, 179, 102, 0.22); +} + +.target-card__select { + display: grid; + gap: 0.35rem; +} + +.target-card__actions { + display: flex; + flex-wrap: wrap; + gap: var(--space-2); +} + +.target-card__name { + font-weight: 600; +} + +.target-card__meta { + color: var(--text-muted); + font-size: 0.88rem; +} + +.target-chip { + min-height: 2.1rem; + padding: 0 0.8rem; + border-radius: 999px; + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.03); + color: var(--text-muted); +} + +.target-chip:hover { + border-color: rgba(118, 199, 255, 0.22); + color: var(--text); +} + +.saved-mask-group { + margin-top: var(--space-6); +} + +.saved-mask-list { + display: grid; + gap: var(--space-2); +} + +.saved-mask-list label { + display: flex; + align-items: center; + gap: var(--space-2); + padding: 0.65rem 0.75rem; + border-radius: var(--radius-sm); + background: rgba(255, 255, 255, 0.03); +} + +.guidance-card { + margin-top: var(--space-6); + padding-top: var(--space-5); + border-top: 1px solid rgba(255, 255, 255, 0.08); +} + +.guidance-card__title { + margin: 0 0 var(--space-3); + font-size: 1rem; + font-weight: 600; +} + +.timeline-status-row, +.timeline-chip-row { + display: flex; + align-items: center; + gap: var(--space-3); + flex-wrap: wrap; +} + +.timeline-chip { + display: inline-flex; + align-items: center; + min-height: 2.3rem; + padding: 0 0.85rem; + border-radius: 999px; + border: 1px solid rgba(118, 143, 173, 0.18); + background: rgba(255, 255, 255, 0.03); + color: var(--text-muted); + font-size: 0.82rem; + font-family: var(--font-mono); +} + +.timeline-chip--success { + border-color: rgba(88, 197, 155, 0.35); + background: rgba(32, 81, 63, 0.4); + color: #dff8ee; +} + +.timeline-chip--outline { + color: var(--text); +} + +.timeline-chip[data-pending="true"], +.timeline-chip--muted { + color: var(--text-muted); +} + +.timeline-control-group--monitor { + gap: var(--space-4); +} + +.timeline-range-rail { + position: relative; + display: flex; + align-items: center; + min-height: 2.5rem; + --applied-range-start: 0%; + --applied-range-end: 100%; + --pending-range-start: 0%; + --pending-range-end: 100%; +} + +.timeline-range-rail::before, +.timeline-range-selection { + content: ""; + position: absolute; + left: 0; + right: 0; + height: 0.45rem; + border-radius: 999px; +} + +.timeline-range-rail::before { + background: rgba(255, 255, 255, 0.1); +} + +.timeline-range-selection { + left: var(--applied-range-start); + right: calc(100% - var(--applied-range-end)); + background: linear-gradient(90deg, rgba(88, 197, 155, 0.8), rgba(117, 242, 180, 0.98)); + box-shadow: 0 0 0 1px rgba(117, 242, 180, 0.18); +} + +.timeline-range-selection[data-range-state="pending"] { + background: linear-gradient(90deg, rgba(242, 179, 102, 0.75), rgba(245, 201, 129, 0.95)); +} + +.timeline-range-rail[data-range-state="pending"] .timeline-range-selection { + left: var(--pending-range-start); + right: calc(100% - var(--pending-range-end)); +} + +.source-playhead-slider { + position: relative; + z-index: 1; + width: 100%; + margin: 0; + background: transparent; + appearance: none; +} + +.source-playhead-slider::-webkit-slider-runnable-track { + height: 0.45rem; + background: transparent; +} + +.source-playhead-slider::-webkit-slider-thumb { + appearance: none; + width: 1rem; + height: 1rem; + margin-top: -0.275rem; + border: 0; + border-radius: 50%; + background: #8fd5ff; + box-shadow: 0 0 0 4px rgba(27, 56, 80, 0.85); +} + +.source-playhead-slider::-moz-range-track { + height: 0.45rem; + background: transparent; +} + +.source-playhead-slider::-moz-range-thumb { + width: 1rem; + height: 1rem; + border: 0; + border-radius: 50%; + background: #8fd5ff; + box-shadow: 0 0 0 4px rgba(27, 56, 80, 0.85); +} + +.timeline-action-row { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-4); +} + +.timeline-inline-actions { + display: flex; + align-items: center; + gap: var(--space-3); + flex-wrap: wrap; +} + +.timeline-inline-actions--compact { + flex-wrap: nowrap; +} + +.timeline-inline-actions--compact .tool-button { + width: auto; + min-width: 0; + white-space: nowrap; +} + +.anchor-control-group { + gap: var(--space-4); +} + +.anchor-control-group__header { + align-items: center; +} + +#annotator-app, +#job-app { + padding: var(--space-6); + border: 1px solid var(--border); + border-radius: var(--radius-xl); + background: rgba(9, 14, 22, 0.78); +} + +#annotation-image { + margin-top: var(--space-4); + border-radius: var(--radius-md); + border: 1px solid var(--border); +} + +.results-shell { + display: flex; + flex-direction: column; + gap: var(--space-6); +} + +.results-header, +.results-preview, +#artifact-panel { + min-height: auto; +} + +.results-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(18rem, 24rem); + gap: var(--space-6); + align-items: start; +} + +.results-sidebar { + display: grid; + gap: var(--space-6); +} + +.preview-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-4); +} + +.preview-mode-tabs { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: var(--space-3); +} + +.preview-tab { + min-height: 2.9rem; + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.03); + color: var(--text); +} + +.preview-tab[data-active] { + border-color: rgba(118, 199, 255, 0.48); + background: rgba(34, 75, 108, 0.28); +} + +.preview-viewport { + min-height: 26rem; + display: grid; + place-items: center; + padding: var(--space-4); + position: relative; + border-radius: calc(var(--radius-xl) - 10px); + border: 1px solid rgba(255, 255, 255, 0.08); + background: + radial-gradient(circle at top, rgba(118, 199, 255, 0.08), transparent 34%), + rgba(7, 12, 20, 0.82); +} + +#preview-video { + width: 100%; + max-height: 62vh; + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.08); + background: #04070b; +} + +.preview-overlay-canvas { + position: absolute; + inset: var(--space-4); + width: calc(100% - (var(--space-4) * 2)); + height: calc(100% - (var(--space-4) * 2)); + pointer-events: none; + border-radius: var(--radius-md); + object-fit: contain; +} + +.preview-placeholder { + color: var(--text-soft); + text-align: center; + line-height: 1.7; +} + +.timeline-section { + display: grid; + gap: var(--space-4); + margin-top: var(--space-6); + padding-top: var(--space-5); + border-top: 1px solid rgba(255, 255, 255, 0.08); +} + +.job-timeline { + list-style: none; + margin: 0; + padding: 0; + display: grid; + gap: var(--space-4); +} + +.timeline-step { + display: grid; + grid-template-columns: auto 1fr; + align-items: start; + gap: var(--space-3); +} + +.timeline-step__dot { + width: 0.85rem; + height: 0.85rem; + margin-top: 0.32rem; + border-radius: 999px; + border: 1px solid rgba(255, 255, 255, 0.18); + background: rgba(255, 255, 255, 0.08); + box-shadow: 0 0 0 6px rgba(255, 255, 255, 0.02); +} + +.timeline-step__label, +.timeline-step__state, +.artifact-card__label, +.artifact-card__name, +.artifact-card__meta, +.warning-panel__title { + margin: 0; +} + +.timeline-step__label { + font-weight: 600; +} + +.timeline-step__state { + margin-top: 0.18rem; + color: var(--text-soft); + text-transform: capitalize; + font-size: 0.86rem; +} + +.timeline-step[data-state="complete"] .timeline-step__dot { + border-color: rgba(88, 197, 155, 0.38); + background: rgba(88, 197, 155, 0.9); +} + +.timeline-step[data-state="current"] .timeline-step__dot { + border-color: rgba(118, 199, 255, 0.44); + background: rgba(118, 199, 255, 0.96); +} + +.warning-panel { + display: grid; + gap: var(--space-3); + margin-top: var(--space-6); + padding: var(--space-5); + border-radius: var(--radius-md); + border: 1px solid rgba(242, 179, 102, 0.22); + background: rgba(68, 40, 19, 0.2); +} + +.warning-panel[data-state="error"] { + border-color: rgba(240, 108, 96, 0.28); + background: rgba(72, 20, 16, 0.24); +} + +.warning-panel__title { + font-size: 0.9rem; + text-transform: uppercase; + letter-spacing: 0.1em; + color: var(--warning); +} + +.warning-panel[data-state="error"] .warning-panel__title { + color: #ffb3ac; +} + +.artifact-summary-list { + display: grid; + gap: var(--space-3); + margin: 0; + padding: 0; + list-style: none; +} + +.target-review-list { + display: grid; + gap: var(--space-3); + margin: 0; + padding: 0; + list-style: none; +} + +.target-review-card { + display: grid; + gap: var(--space-3); + padding: 0.95rem 1rem; + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.06); + background: rgba(255, 255, 255, 0.03); +} + +.target-review-card--empty { + color: var(--text-muted); +} + +.target-review-card__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-4); +} + +.target-review-card__title, +.target-review-card__subtitle { + margin: 0; +} + +.target-review-card__title { + font-weight: 600; +} + +.target-review-card__subtitle { + margin-top: 0.18rem; + color: var(--text-soft); + font-size: 0.84rem; + font-family: var(--font-mono); +} + +.target-review-meta { + display: grid; + gap: var(--space-2); + margin: 0; +} + +.target-review-meta div { + display: flex; + justify-content: space-between; + gap: var(--space-4); +} + +.target-review-meta dt { + color: var(--text-soft); + font-size: 0.82rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.target-review-meta dd { + margin: 0; + color: var(--text); + text-align: right; +} + +.artifact-card { + display: grid; + gap: var(--space-3); + padding: 0.95rem 1rem; + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.06); + background: rgba(255, 255, 255, 0.03); +} + +.artifact-card[data-available="true"] { + border-color: rgba(118, 199, 255, 0.16); + background: rgba(21, 33, 48, 0.68); +} + +.artifact-card__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-4); +} + +.artifact-card__label { + font-size: 0.98rem; + font-weight: 600; +} + +.artifact-card__name { + margin-top: 0.18rem; + color: var(--text-soft); + font-size: 0.84rem; + font-family: var(--font-mono); +} + +.artifact-card__meta { + color: var(--text-muted); + line-height: 1.5; +} + +.artifact-card__state { + display: inline-flex; + align-items: center; + min-height: 2rem; + padding: 0 0.75rem; + border-radius: 999px; + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.04); + color: var(--text-muted); + font-size: 0.8rem; + text-transform: uppercase; + letter-spacing: 0.08em; +} + +.artifact-card[data-available="true"] .artifact-card__state { + border-color: rgba(88, 197, 155, 0.22); + color: var(--success); +} + +.artifact-card__link { + display: inline-flex; + align-items: center; + justify-content: center; + width: fit-content; + min-height: 2.4rem; + padding: 0 0.95rem; + border-radius: 999px; + border: 1px solid rgba(118, 199, 255, 0.22); + background: rgba(118, 199, 255, 0.08); + color: var(--accent); + text-decoration: none; +} + +.workspace-shell { + display: flex; + flex-direction: column; + gap: var(--space-5); +} + +.workspace-header { + min-height: auto; + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-6); + padding: var(--space-5) var(--space-6); +} + +.workspace-header__copy, +.workspace-header__controls { + display: grid; + gap: var(--space-3); +} + +.workspace-header__controls { + justify-items: end; +} + +.workflow-stepper, +.workspace-nav, +.workspace-sidebar-tabs { + display: flex; + align-items: center; + gap: var(--space-2); + flex-wrap: wrap; +} + +.workflow-stepper__step, +.workspace-sidebar-tab { + min-height: 2.5rem; + padding: 0.6rem 0.95rem; + border-radius: 999px; + border: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.03); + color: var(--text-muted); +} + +.workflow-stepper__step[data-active], +.workspace-sidebar-tab[data-active], +#compare-toggle[data-active] { + border-color: rgba(118, 199, 255, 0.44); + background: rgba(29, 69, 100, 0.42); + color: var(--text); +} + +.workspace-layout { + display: grid; + grid-template-columns: minmax(18rem, 21rem) minmax(0, 1fr) minmax(18rem, 22rem); + gap: var(--space-5); + align-items: start; +} + +.workspace-sidebar, +.workspace-review-sidebar, +.workspace-monitor, +.workspace-timeline-dock, +.compare-drawer { + min-height: auto; +} + +.workspace-sidebar { + display: grid; + gap: var(--space-4); + position: sticky; + top: var(--space-8); + max-height: calc(100dvh - (var(--space-8) * 2)); + overflow: auto; +} + +.workspace-sidebar-panel { + display: grid; + gap: var(--space-4); +} + +.compact-summary { + display: flex; + align-items: center; + gap: var(--space-2); + flex-wrap: wrap; +} + +.compact-context { + gap: var(--space-2); +} + +.compact-context div { + padding-bottom: var(--space-2); +} + +.workspace-note-card { + padding: var(--space-3); + border-radius: var(--radius-md); + border: 1px solid rgba(255, 255, 255, 0.06); + background: rgba(255, 255, 255, 0.02); +} + +.workspace-main { + display: grid; + gap: var(--space-4); + min-width: 0; +} + +.workspace-monitor { + display: grid; + gap: var(--space-4); + min-width: 0; +} + +.workspace-monitor__header, +.workspace-monitor__toolbar, +.workspace-monitor__footer { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-4); + flex-wrap: wrap; +} + +.workspace-monitor-frame { + display: grid; + place-items: center; + min-height: min(64vh, 42rem); + padding: var(--space-3); + border-radius: calc(var(--radius-xl) - 10px); + border: 1px solid rgba(255, 255, 255, 0.08); + background: + linear-gradient(45deg, rgba(255, 255, 255, 0.03) 25%, transparent 25%), + linear-gradient(-45deg, rgba(255, 255, 255, 0.03) 25%, transparent 25%), + linear-gradient(45deg, transparent 75%, rgba(255, 255, 255, 0.03) 75%), + linear-gradient(-45deg, transparent 75%, rgba(255, 255, 255, 0.03) 75%), + rgba(7, 12, 20, 0.72); + background-size: 24px 24px; + background-position: 0 0, 0 12px, 12px -12px, -12px 0; +} + +.workspace-timeline-dock { + display: grid; + gap: var(--space-3); + padding: var(--space-4) var(--space-5); +} + +.timeline-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-4); + flex-wrap: wrap; +} + +.timeline-row--rail, +.timeline-row--anchor { + display: block; +} + +.timeline-row--anchor { + padding-top: var(--space-2); + border-top: 1px solid rgba(255, 255, 255, 0.06); +} + +#clip-primary-rail { + width: 100%; +} + +.compare-drawer { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-3); +} + +.workspace-review-sidebar { + display: grid; + gap: var(--space-4); + position: sticky; + top: var(--space-8); +} + +@media (max-width: 1280px) { + .upload-grid { + grid-template-columns: 1fr; + } + + .workbench-shell { + grid-template-columns: 1fr; + } + + .monitor-header, + .timeline-action-row { + flex-direction: column; + align-items: stretch; + } + + .monitor-header__controls { + justify-items: stretch; + } + + .results-grid { + grid-template-columns: 1fr; + } + + .workspace-layout { + grid-template-columns: 1fr; + } + + .panel { + min-height: auto; + } +} + +@media (max-width: 780px) { + .app-shell { + padding: var(--space-4); + } + + .topbar { + flex-direction: column; + align-items: flex-start; + } + + .topbar__status { + justify-content: flex-start; + } + + .form-actions { + flex-direction: column; + align-items: stretch; + } + + .preview-mode-tabs, + .preview-compare-strip, + .compare-drawer { + grid-template-columns: 1fr; + } + + .monitor-pill-row, + .timeline-inline-actions, + .timeline-status-row, + .timeline-chip-row { + width: 100%; + } +} diff --git a/matanyone2/webapp/static/upload.js b/matanyone2/webapp/static/upload.js new file mode 100644 index 0000000..408f63f --- /dev/null +++ b/matanyone2/webapp/static/upload.js @@ -0,0 +1,194 @@ +import { + formatBytes, + formatDuration, + parseJson, + probeVideoFile, + setStatus, +} from "/static/shared.js"; + +function bindUploadPage() { + const form = document.getElementById("upload-form"); + if (!form) { + return; + } + + const fileInput = document.getElementById("video-file"); + const dropzone = document.getElementById("dropzone-panel"); + const submitButton = document.getElementById("upload-submit"); + const resetButton = document.getElementById("upload-reset"); + const status = document.getElementById("upload-status"); + const mediaCard = document.getElementById("media-info-card"); + const preview = document.getElementById("media-preview"); + const mediaName = document.getElementById("media-name"); + const mediaType = document.getElementById("media-type"); + const mediaSize = document.getElementById("media-size"); + const mediaResolution = document.getElementById("media-resolution"); + const mediaDuration = document.getElementById("media-duration"); + + function resetMediaCard() { + if (mediaCard) { + mediaCard.dataset.empty = "true"; + } + if (preview) { + preview.dataset.ready = "false"; + preview.textContent = "No clip selected"; + } + if (mediaName) { + mediaName.textContent = "No file selected"; + } + if (mediaType) { + mediaType.textContent = "-"; + } + if (mediaSize) { + mediaSize.textContent = "-"; + } + if (mediaResolution) { + mediaResolution.textContent = "-"; + } + if (mediaDuration) { + mediaDuration.textContent = "-"; + } + if (submitButton) { + submitButton.disabled = true; + } + } + + async function updateMediaCard(file) { + if (!file) { + resetMediaCard(); + setStatus(status, "Select a source clip to prepare the draft.", false); + return; + } + + if (mediaCard) { + mediaCard.dataset.empty = "false"; + } + if (preview) { + preview.dataset.ready = "true"; + preview.textContent = (file.name.split(".").pop() || "video").toUpperCase(); + } + if (mediaName) { + mediaName.textContent = file.name; + } + if (mediaType) { + mediaType.textContent = file.type || "video"; + } + if (mediaSize) { + mediaSize.textContent = formatBytes(file.size); + } + if (mediaResolution) { + mediaResolution.textContent = "Reading metadata..."; + } + if (mediaDuration) { + mediaDuration.textContent = "Reading metadata..."; + } + + try { + const metadata = await probeVideoFile(file); + if (mediaResolution) { + mediaResolution.textContent = `${metadata.width} x ${metadata.height}`; + } + if (mediaDuration) { + mediaDuration.textContent = formatDuration(metadata.duration); + } + if (submitButton) { + submitButton.disabled = false; + } + setStatus(status, "Draft ready to create. Continue into annotation when ready.", false); + } catch (error) { + if (mediaResolution) { + mediaResolution.textContent = "-"; + } + if (mediaDuration) { + mediaDuration.textContent = "-"; + } + if (submitButton) { + submitButton.disabled = false; + } + setStatus(status, error.message, true); + } + } + + function assignDroppedFiles(files) { + if (!fileInput || !files?.length) { + return; + } + const transfer = new DataTransfer(); + transfer.items.add(files[0]); + fileInput.files = transfer.files; + updateMediaCard(files[0]); + } + + dropzone?.addEventListener("keydown", (event) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + fileInput?.click(); + } + }); + + ["dragenter", "dragover"].forEach((eventName) => { + dropzone?.addEventListener(eventName, (event) => { + event.preventDefault(); + dropzone.dataset.dragging = "true"; + }); + }); + + ["dragleave", "drop"].forEach((eventName) => { + dropzone?.addEventListener(eventName, () => { + if (dropzone) { + dropzone.dataset.dragging = "false"; + } + }); + }); + + dropzone?.addEventListener("drop", (event) => { + event.preventDefault(); + assignDroppedFiles(event.dataTransfer?.files); + }); + + fileInput?.addEventListener("change", () => { + updateMediaCard(fileInput.files?.[0] || null); + }); + + resetButton?.addEventListener("click", () => { + if (fileInput) { + fileInput.value = ""; + } + resetMediaCard(); + }); + + form.addEventListener("submit", async (event) => { + event.preventDefault(); + const file = fileInput?.files?.[0]; + if (!file) { + setStatus(status, "Select a video before entering the workbench.", true); + return; + } + + const body = new FormData(); + body.append("video", file); + if (submitButton) { + submitButton.disabled = true; + } + setStatus(status, "Uploading clip and preparing draft...", false); + + try { + const payload = await parseJson( + await fetch(form.dataset.uploadEndpoint, { + method: "POST", + body, + }) + ); + window.location.assign(`/drafts/${payload.draft_id}/workspace`); + } catch (error) { + if (submitButton) { + submitButton.disabled = false; + } + setStatus(status, error.message, true); + } + }); + + resetMediaCard(); +} + +document.addEventListener("DOMContentLoaded", bindUploadPage); diff --git a/matanyone2/webapp/static/workbench.js b/matanyone2/webapp/static/workbench.js new file mode 100644 index 0000000..3273f8b --- /dev/null +++ b/matanyone2/webapp/static/workbench.js @@ -0,0 +1,1523 @@ +import { + formatDuration, + parseJson, + setStatus, + withCacheBust, +} from "/static/shared.js"; + +const PRESET_META = { + balanced: { + label: "Balanced", + note: "Use this when the silhouette is already close and you want an even starting point.", + }, + hair: { + label: "Hair Priority", + note: "Bias your cleanup around flyaway strands and soft hairline gaps before committing the layer.", + }, + edge: { + label: "Edge Priority", + note: "Use this when the boundary should stay tight around shoulders, jaw lines, or wardrobe edges.", + }, + motion: { + label: "Motion Blur", + note: "Use this when motion softness matters more than a perfectly hard cut on the outer contour.", + }, +}; + +function bindWorkbench() { + const root = document.getElementById("annotator-app"); + if (!root) { + return; + } + + const status = document.getElementById("annotator-status"); + const image = document.getElementById("annotation-image"); + const canvasFrame = root.querySelector(".canvas-frame"); + const saveButton = document.getElementById("save-mask"); + const submitButton = document.getElementById("submit-job"); + const createTargetButton = document.getElementById("create-target"); + const undoButton = document.getElementById("undo-click"); + const resetButton = document.getElementById("reset-target"); + const savedMaskList = document.getElementById("saved-mask-list"); + const targetList = document.getElementById("target-list"); + const positiveButton = document.getElementById("positive-mode"); + const negativeButton = document.getElementById("negative-mode"); + const brushButtons = Array.from(root.querySelectorAll(".brush-button")); + const presetButtons = Array.from(root.querySelectorAll(".preset-button")); + const stageButtons = Array.from(root.querySelectorAll(".stage-button")); + const viewButtons = Array.from(root.querySelectorAll(".canvas-view-tab")); + const inspectorStage = document.getElementById("inspector-stage"); + const inspectorTarget = document.getElementById("inspector-target"); + const inspectorPreset = document.getElementById("inspector-preset"); + const inspectorPoints = document.getElementById("inspector-points"); + const inspectorMask = document.getElementById("inspector-mask"); + const canvasModeLabel = document.getElementById("canvas-mode-label"); + const canvasStageNote = document.getElementById("canvas-stage-note"); + const guidanceTitle = document.getElementById("stage-guidance-title"); + const guidanceCopy = document.getElementById("stage-guidance-copy"); + const selectionNote = document.getElementById("selection-note"); + const presetNote = document.getElementById("preset-note"); + const brushNote = document.getElementById("brush-note"); + const workflowStageChip = document.getElementById("workflow-stage-chip"); + const targetNameInput = document.getElementById("target-name-input"); + const applyTargetNameButton = document.getElementById("apply-target-name"); + const toggleTargetLockButton = document.getElementById("toggle-target-lock"); + const targetSummary = document.getElementById("target-summary"); + const brushRadiusInput = document.getElementById("brush-radius"); + const brushRadiusValue = document.getElementById("brush-radius-value"); + const overlayOpacityInput = document.getElementById("overlay-opacity"); + const overlayOpacityValue = document.getElementById("overlay-opacity-value"); + const sourcePlayheadSlider = document.getElementById("source-playhead-slider"); + const markRangeInButton = document.getElementById("mark-range-in"); + const markRangeOutButton = document.getElementById("mark-range-out"); + const clearRangeSelectionButton = document.getElementById("clear-range-selection"); + const toggleSourcePlaybackButton = document.getElementById("toggle-source-playback"); + const timelineCurrentLabel = document.getElementById("timeline-current-label"); + const timelineSelectedLabel = document.getElementById("timeline-selected-label"); + const timelineAppliedLabel = document.getElementById("timeline-applied-label"); + const timelineInChip = document.getElementById("timeline-in-chip"); + const timelineOutChip = document.getElementById("timeline-out-chip"); + const timelineDurationChip = document.getElementById("timeline-duration-chip"); + const timelineRangeRail = document.getElementById("timeline-range-rail"); + const timelineRangeSelection = document.getElementById("timeline-range-selection"); + const templateFrameSlider = document.getElementById("template-frame-slider"); + const templateFrameValue = document.getElementById("template-frame-value"); + const anchorFrameSummary = document.getElementById("anchor-frame-summary"); + const keyframeVideo = document.getElementById("keyframe-video"); + const keyframeSelectedLabel = document.getElementById("keyframe-selected-label"); + const keyframeAppliedLabel = document.getElementById("keyframe-applied-label"); + const keyframeTimeLabel = document.getElementById("keyframe-time-label"); + const presetStrengthInput = document.getElementById("preset-strength"); + const presetStrengthValue = document.getElementById("preset-strength-value"); + const motionStrengthInput = document.getElementById("motion-strength"); + const motionStrengthValue = document.getElementById("motion-strength-value"); + const temporalStabilityInput = document.getElementById("temporal-stability"); + const temporalStabilityValue = document.getElementById("temporal-stability-value"); + const previewBeforeImage = document.getElementById("preview-before-image"); + const previewLiveImage = document.getElementById("preview-live-image"); + + const state = { + activeTool: "point-positive", + canvasMode: root.dataset.defaultCanvasMode || "source", + workbench: null, + selectedMasks: new Set(), + brushRadius: Number(brushRadiusInput?.value || 28), + overlayOpacity: Number(overlayOpacityInput?.value || 72), + playheadFrame: Number(sourcePlayheadSlider?.value || 0), + rangeSelectionStart: 0, + rangeSelectionEnd: 0, + rangeSelectionTouchedStart: false, + rangeSelectionTouchedEnd: false, + rangeAppliedStart: 0, + rangeAppliedEnd: 0, + templateFrameSelection: Number(templateFrameSlider?.value || 0), + templateFrameApplied: templateFrameSlider?.value === "" ? null : Number(templateFrameSlider?.value || 0), + fps: Number(root.dataset.fps || 0), + durationSeconds: Number(root.dataset.durationSeconds || 0), + livePatchTimer: null, + livePatchRevision: 0, + lastAppliedLivePatchRevision: 0, + compareBeforeSrc: root.dataset.templateFrameUrl || "", + compareLiveSrc: root.dataset.templateFrameUrl || "", + }; + + const STAGE_ORDER = ["coarse", "refine", "preview"]; + + function isTypingContext(target) { + if (!(target instanceof HTMLElement)) { + return false; + } + const tagName = target.tagName; + return ( + tagName === "INPUT" || + tagName === "TEXTAREA" || + tagName === "SELECT" || + target.isContentEditable + ); + } + + function selectedMaskNames() { + return Array.from(state.selectedMasks).sort(); + } + + function activeTarget(payload = state.workbench) { + return payload?.targets?.find((target) => target.target_id === payload.active_target_id) || null; + } + + function activePreset(payload = state.workbench) { + return activeTarget(payload)?.refine_preset || "balanced"; + } + + function updateRangeOutput(outputElement, value, suffix = "%") { + if (!outputElement) { + return; + } + outputElement.value = `${value}${suffix}`; + outputElement.textContent = `${value}${suffix}`; + } + + function frameToSeconds(frameIndex, payload = state.workbench) { + const fps = Number(payload?.fps || state.fps || 0); + if (!Number.isFinite(fps) || fps <= 0) { + return 0; + } + return frameIndex / fps; + } + + function formatFrameTimestamp(frameIndex, payload = state.workbench) { + return formatDuration(frameToSeconds(frameIndex, payload)); + } + + function hasTemplateFrame(payload = state.workbench) { + return payload?.template_frame_index !== null && payload?.template_frame_index !== undefined; + } + + function clampFrame(frameIndex, minFrame, maxFrame) { + return Math.max(minFrame, Math.min(maxFrame, frameIndex)); + } + + function syncCompareStrip(payload = state.workbench) { + if (!previewBeforeImage || !previewLiveImage || !payload) { + return; + } + const liveSrc = image?.src || withCacheBust(resolveCanvasUrl(payload)); + if (!state.compareBeforeSrc) { + state.compareBeforeSrc = liveSrc; + } + state.compareLiveSrc = liveSrc; + previewBeforeImage.src = state.compareBeforeSrc; + previewLiveImage.src = state.compareLiveSrc; + } + + function syncTimelineRangeRail(payload) { + if (!timelineRangeRail || !timelineRangeSelection || !payload) { + return; + } + const maxFrame = Math.max(1, (payload.frame_count || 1) - 1); + const pendingStart = Math.min(state.rangeSelectionStart, state.rangeSelectionEnd); + const pendingEnd = Math.max(state.rangeSelectionStart, state.rangeSelectionEnd); + const appliedStartPercent = (state.rangeAppliedStart / maxFrame) * 100; + const appliedEndPercent = (state.rangeAppliedEnd / maxFrame) * 100; + const pendingStartPercent = (pendingStart / maxFrame) * 100; + const pendingEndPercent = (pendingEnd / maxFrame) * 100; + const rangeDirty = pendingStart !== state.rangeAppliedStart || pendingEnd !== state.rangeAppliedEnd; + + timelineRangeRail.style.setProperty("--applied-range-start", `${appliedStartPercent}%`); + timelineRangeRail.style.setProperty("--applied-range-end", `${appliedEndPercent}%`); + timelineRangeRail.style.setProperty("--pending-range-start", `${pendingStartPercent}%`); + timelineRangeRail.style.setProperty("--pending-range-end", `${pendingEndPercent}%`); + timelineRangeRail.dataset.rangeState = rangeDirty ? "pending" : "applied"; + timelineRangeSelection.dataset.rangeState = rangeDirty ? "pending" : "applied"; + } + + function syncKeyframeSummary(payload) { + if (!payload) { + return; + } + const maxFrame = Math.max(0, (payload.frame_count || 1) - 1); + state.fps = Number(payload.fps || state.fps || 0); + state.durationSeconds = Number(payload.duration_seconds || state.durationSeconds || 0); + state.rangeAppliedStart = Number(payload.process_start_frame_index || 0); + state.rangeAppliedEnd = Number( + payload.process_end_frame_index ?? maxFrame + ); + + if ( + state.rangeSelectionStart === undefined + || Number.isNaN(state.rangeSelectionStart) + || state.rangeSelectionStart < 0 + ) { + state.rangeSelectionStart = state.rangeAppliedStart; + } + if ( + state.rangeSelectionEnd === undefined + || Number.isNaN(state.rangeSelectionEnd) + || state.rangeSelectionEnd < 0 + ) { + state.rangeSelectionEnd = state.rangeAppliedEnd; + } + + state.rangeSelectionStart = clampFrame(state.rangeSelectionStart, 0, maxFrame); + state.rangeSelectionEnd = clampFrame(state.rangeSelectionEnd, 0, maxFrame); + if (state.rangeSelectionStart > state.rangeSelectionEnd) { + const nextStart = state.rangeSelectionEnd; + state.rangeSelectionEnd = state.rangeSelectionStart; + state.rangeSelectionStart = nextStart; + } + + state.templateFrameApplied = hasTemplateFrame(payload) + ? Number(payload.template_frame_index) + : null; + if ( + state.templateFrameSelection === undefined + || Number.isNaN(state.templateFrameSelection) + || state.templateFrameSelection < state.rangeAppliedStart + || state.templateFrameSelection > state.rangeAppliedEnd + ) { + state.templateFrameSelection = state.templateFrameApplied ?? state.rangeAppliedStart; + } + if ( + state.playheadFrame === undefined + || Number.isNaN(state.playheadFrame) + || state.playheadFrame < 0 + || state.playheadFrame > maxFrame + ) { + state.playheadFrame = state.templateFrameApplied ?? state.rangeAppliedStart; + } + + if (sourcePlayheadSlider) { + sourcePlayheadSlider.max = String(maxFrame); + sourcePlayheadSlider.value = String(clampFrame(state.playheadFrame, 0, maxFrame)); + } + + if (timelineCurrentLabel) { + timelineCurrentLabel.textContent = `Playhead ยท Frame ${state.playheadFrame} ยท ${formatFrameTimestamp(state.playheadFrame, payload)}`; + } + if (timelineSelectedLabel) { + timelineSelectedLabel.textContent = `Pending range ยท Frame ${state.rangeSelectionStart} - ${state.rangeSelectionEnd}`; + } + if (timelineAppliedLabel) { + timelineAppliedLabel.textContent = `Processing range ยท Frame ${state.rangeAppliedStart} - ${state.rangeAppliedEnd}`; + } + if (timelineInChip) { + timelineInChip.textContent = `In ยท ${formatFrameTimestamp(state.rangeSelectionStart, payload)} ยท F${state.rangeSelectionStart}`; + } + if (timelineOutChip) { + timelineOutChip.textContent = `Out ยท ${formatFrameTimestamp(state.rangeSelectionEnd, payload)} ยท F${state.rangeSelectionEnd}`; + } + if (timelineDurationChip) { + const durationFrames = Math.max(1, state.rangeSelectionEnd - state.rangeSelectionStart + 1); + const fps = Number(payload.fps || state.fps || 0); + const duration = fps > 0 ? durationFrames / fps : 0; + timelineDurationChip.textContent = `Duration ยท ${formatDuration(duration)} ยท ${durationFrames}f`; + } + + if (templateFrameSlider) { + templateFrameSlider.min = String(state.rangeAppliedStart); + templateFrameSlider.max = String(state.rangeAppliedEnd); + templateFrameSlider.value = String(state.templateFrameSelection); + } + updateRangeOutput(templateFrameValue, Number(state.templateFrameSelection || 0), ""); + + if (anchorFrameSummary) { + anchorFrameSummary.textContent = state.templateFrameApplied === null + ? "Anchor ยท Not set" + : `Anchor ยท Frame ${state.templateFrameApplied} ยท ${formatFrameTimestamp(state.templateFrameApplied, payload)}`; + } + if (keyframeSelectedLabel) { + keyframeSelectedLabel.textContent = `Selected frame ${state.templateFrameSelection}`; + } + if (keyframeAppliedLabel) { + keyframeAppliedLabel.textContent = state.templateFrameApplied === null + ? "Applied frame Not set" + : `Applied frame ${state.templateFrameApplied}`; + } + if (keyframeTimeLabel) { + keyframeTimeLabel.textContent = `${formatFrameTimestamp(state.templateFrameSelection, payload)} / ${formatFrameTimestamp(state.rangeAppliedEnd, payload)}`; + } + + syncTimelineRangeRail(payload); + + if (keyframeVideo) { + if (!keyframeVideo.src) { + keyframeVideo.src = root.dataset.sourceVideoUrl; + } + const desiredTime = frameToSeconds(state.playheadFrame, payload); + if (Number.isFinite(desiredTime) && Math.abs((keyframeVideo.currentTime || 0) - desiredTime) > 0.04) { + try { + keyframeVideo.currentTime = desiredTime; + } catch (_error) { + // Ignore transient seek failures before metadata is ready. + } + } + } + } + + function syncTargetControls(payload) { + const currentTarget = activeTarget(payload); + if (!currentTarget) { + return; + } + if (presetStrengthInput) { + presetStrengthInput.value = String(Math.round((currentTarget.preset_strength || 0) * 100)); + updateRangeOutput(presetStrengthValue, Number(presetStrengthInput.value)); + } + if (motionStrengthInput) { + motionStrengthInput.value = String(Math.round((currentTarget.motion_strength || 0) * 100)); + updateRangeOutput(motionStrengthValue, Number(motionStrengthInput.value)); + } + if (temporalStabilityInput) { + temporalStabilityInput.value = String(Math.round((currentTarget.temporal_stability || 0) * 100)); + updateRangeOutput(temporalStabilityValue, Number(temporalStabilityInput.value)); + } + } + + function applyImagePresentation() { + if (!image || !keyframeVideo) { + return; + } + const showVideo = state.canvasMode === "source"; + keyframeVideo.hidden = !showVideo; + image.hidden = showVideo; + canvasFrame?.setAttribute("data-canvas-mode", state.canvasMode); + + if (showVideo) { + image.style.opacity = "1"; + syncSourcePlaybackButton(); + return; + } + if (!keyframeVideo.paused) { + keyframeVideo.pause(); + } + image.style.opacity = String(state.overlayOpacity / 100); + syncSourcePlaybackButton(); + } + + function syncSelectedMasks(payload) { + const availableMaskNames = payload.mask_names || []; + const available = new Set(availableMaskNames); + const serverSelected = new Set(payload.selected_mask_names || []); + + if (state.selectedMasks.size === 0) { + state.selectedMasks = serverSelected.size > 0 + ? serverSelected + : new Set(availableMaskNames); + return; + } + + state.selectedMasks = new Set( + Array.from(state.selectedMasks).filter((maskName) => available.has(maskName)) + ); + if (state.selectedMasks.size === 0 && serverSelected.size > 0) { + state.selectedMasks = serverSelected; + } + } + + function syncToolButtons(payload) { + const canEdit = payload?.can_apply_clicks ?? false; + const pointPositive = state.activeTool === "point-positive"; + const pointNegative = state.activeTool === "point-negative"; + + positiveButton?.toggleAttribute("data-active", pointPositive); + negativeButton?.toggleAttribute("data-active", pointNegative); + positiveButton?.toggleAttribute("disabled", !canEdit); + negativeButton?.toggleAttribute("disabled", !canEdit); + + brushButtons.forEach((button) => { + const isActive = state.activeTool === `brush-${button.dataset.brushMode}`; + button.toggleAttribute("data-active", isActive); + button.toggleAttribute("disabled", !canEdit); + }); + + if (image) { + image.dataset.editable = canEdit ? "true" : "false"; + image.style.cursor = canEdit ? "crosshair" : "default"; + } + } + + function renderSavedMasks(maskNames) { + if (!savedMaskList) { + return; + } + savedMaskList.innerHTML = ""; + maskNames.forEach((maskName) => { + const label = document.createElement("label"); + const input = document.createElement("input"); + input.type = "checkbox"; + input.name = "mask_name"; + input.value = maskName; + input.checked = state.selectedMasks.has(maskName); + input.addEventListener("change", () => { + if (input.checked) { + state.selectedMasks.add(maskName); + } else { + state.selectedMasks.delete(maskName); + } + syncActionState(state.workbench); + }); + label.appendChild(input); + label.append(` ${maskName}`); + savedMaskList.appendChild(label); + }); + } + + async function updateTarget(targetId, patch, pendingMessage, successMessage) { + setStatus(status, pendingMessage, false); + try { + const payload = await requestTargetPatch(targetId, patch); + renderWorkbench(payload); + setStatus(status, successMessage(payload), false); + } catch (error) { + setStatus(status, error.message, true); + } + } + + async function requestTargetPatch(targetId, patch) { + return parseJson( + await fetch(`${root.dataset.targetsEndpoint}/${targetId}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(patch), + }) + ); + } + + function renderTargets(targets, activeTargetId) { + if (!targetList) { + return; + } + targetList.innerHTML = ""; + targets.forEach((target) => { + const card = document.createElement("article"); + card.className = "target-card"; + card.dataset.selected = target.target_id === activeTargetId ? "true" : "false"; + card.dataset.hidden = target.visible ? "false" : "true"; + card.dataset.locked = target.locked ? "true" : "false"; + + const selectButton = document.createElement("button"); + selectButton.type = "button"; + selectButton.className = "target-card__select"; + selectButton.innerHTML = ` + ${target.name} + ${target.point_count} point${target.point_count === 1 ? "" : "s"} | ${PRESET_META[target.refine_preset]?.label || "Balanced"} | ${target.saved_mask_name || "unsaved"} | ${target.visible ? "visible" : "hidden"} | ${target.locked ? "locked" : "editable"} + `; + selectButton.addEventListener("click", async () => { + if (target.target_id === activeTargetId) { + return; + } + setStatus(status, `Switching to ${target.name}...`, false); + try { + const payload = await parseJson( + await fetch(`${root.dataset.targetsEndpoint}/${target.target_id}/select`, { + method: "POST", + }) + ); + renderWorkbench(payload); + setStatus(status, `Active target: ${target.name}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + const actions = document.createElement("div"); + actions.className = "target-card__actions"; + + const visibilityButton = document.createElement("button"); + visibilityButton.type = "button"; + visibilityButton.className = "target-chip"; + visibilityButton.textContent = target.visible ? "Hide" : "Show"; + visibilityButton.addEventListener("click", () => { + const nextVisible = !target.visible; + updateTarget( + target.target_id, + { visible: nextVisible }, + `${nextVisible ? "Showing" : "Hiding"} ${target.name}...`, + () => `${target.name} is now ${nextVisible ? "visible" : "hidden"}.` + ); + }); + + const lockButton = document.createElement("button"); + lockButton.type = "button"; + lockButton.className = "target-chip"; + lockButton.textContent = target.locked ? "Unlock" : "Lock"; + lockButton.addEventListener("click", () => { + const nextLocked = !target.locked; + updateTarget( + target.target_id, + { locked: nextLocked }, + `${nextLocked ? "Locking" : "Unlocking"} ${target.name}...`, + () => `${target.name} is now ${nextLocked ? "locked" : "editable"}.` + ); + }); + + actions.append(visibilityButton, lockButton); + card.append(selectButton, actions); + targetList.appendChild(card); + }); + } + + function syncActionState(payload) { + if (!payload) { + return; + } + const canSubmit = payload.can_submit && state.selectedMasks.size > 0 && hasTemplateFrame(payload); + const rangeDirty = ( + state.rangeSelectionStart !== state.rangeAppliedStart + || state.rangeSelectionEnd !== state.rangeAppliedEnd + ); + const rangePendingCompletion = ( + state.rangeSelectionTouchedStart !== state.rangeSelectionTouchedEnd + ); + root.dataset.stage = payload.stage; + root.dataset.editable = payload.can_apply_clicks ? "true" : "false"; + + createTargetButton?.toggleAttribute("disabled", !payload.can_create_target); + undoButton?.toggleAttribute("disabled", !payload.can_undo_clicks); + resetButton?.toggleAttribute("disabled", !payload.can_reset_target); + saveButton?.toggleAttribute("disabled", !payload.can_save_current_target); + submitButton?.toggleAttribute("disabled", !canSubmit); + applyTargetNameButton?.toggleAttribute("disabled", !payload.active_target_id); + markRangeInButton?.toggleAttribute("disabled", !payload.can_apply_range); + markRangeOutButton?.toggleAttribute("disabled", !payload.can_apply_range); + clearRangeSelectionButton?.toggleAttribute("disabled", !payload.can_apply_range); + sourcePlayheadSlider?.toggleAttribute("disabled", !payload.can_apply_range); + templateFrameSlider?.toggleAttribute("disabled", !payload.can_change_template_frame || rangeDirty || rangePendingCompletion); + toggleSourcePlaybackButton?.toggleAttribute("disabled", state.canvasMode !== "source"); + + const currentTarget = activeTarget(payload); + toggleTargetLockButton?.toggleAttribute("disabled", !currentTarget); + if (toggleTargetLockButton && currentTarget) { + toggleTargetLockButton.textContent = currentTarget.locked ? "Unlock Target" : "Lock Target"; + } + + if (submitButton) { + submitButton.textContent = payload.stage === "preview" + ? "Queue Matting Job" + : "Submit Matting Job"; + } + if (timelineSelectedLabel) { + timelineSelectedLabel.dataset.pending = rangeDirty || rangePendingCompletion ? "true" : "false"; + } + + syncToolButtons(payload); + } + + function resolveCanvasUrl(payload) { + if (state.canvasMode === "mask") { + return payload.active_mask_url || payload.current_mask_url || payload.template_frame_url || root.dataset.templateFrameUrl; + } + if (state.canvasMode === "source") { + return payload.template_frame_url || root.dataset.templateFrameUrl; + } + return payload.current_preview_url || payload.template_frame_url || root.dataset.templateFrameUrl; + } + + function syncCanvasMode(payload) { + if (!hasTemplateFrame(payload) && state.canvasMode !== "source") { + state.canvasMode = "source"; + } + if (state.canvasMode === "mask" && !payload.active_mask_url && !payload.current_mask_url) { + state.canvasMode = payload.current_preview_url ? "overlay" : "source"; + } + + const modeLabel = { + source: "Source plate", + overlay: "Overlay preview", + mask: "Mask inspection", + }[state.canvasMode]; + + if (canvasModeLabel) { + canvasModeLabel.textContent = `${payload.canvas_mode_label || "Guided silhouette pass"} | ${modeLabel}`; + } + + if (image) { + image.src = withCacheBust(resolveCanvasUrl(payload)); + image.alt = { + source: `Template frame for ${payload.draft_id}`, + overlay: `Overlay preview for ${payload.draft_id}`, + mask: `Mask preview for ${payload.draft_id}`, + }[state.canvasMode]; + } + + viewButtons.forEach((button) => { + const isMask = button.dataset.canvasMode === "mask"; + const isOverlay = button.dataset.canvasMode === "overlay"; + button.toggleAttribute( + "disabled", + (isMask && (!hasTemplateFrame(payload) || (!payload.active_mask_url && !payload.current_mask_url))) + || (isOverlay && !hasTemplateFrame(payload)) + ); + button.toggleAttribute("data-active", button.dataset.canvasMode === state.canvasMode); + }); + + applyImagePresentation(); + } + + function syncPresetButtons(payload) { + const preset = activePreset(payload); + presetButtons.forEach((button) => { + button.toggleAttribute("data-active", button.dataset.preset === preset); + }); + if (inspectorPreset) { + inspectorPreset.textContent = PRESET_META[preset]?.label || "Balanced"; + } + if (presetNote) { + presetNote.textContent = PRESET_META[preset]?.note || PRESET_META.balanced.note; + } + } + + function renderWorkbench(payload) { + const rangeChangedOnServer = ( + state.rangeAppliedStart !== Number(payload.process_start_frame_index || 0) + || state.rangeAppliedEnd !== Number(payload.process_end_frame_index ?? Math.max(0, (payload.frame_count || 1) - 1)) + ); + const templateChangedOnServer = ( + state.templateFrameApplied !== (hasTemplateFrame(payload) ? Number(payload.template_frame_index) : null) + ); + state.workbench = payload; + syncSelectedMasks(payload); + if (rangeChangedOnServer || (!state.rangeSelectionTouchedStart && !state.rangeSelectionTouchedEnd)) { + state.rangeSelectionStart = Number(payload.process_start_frame_index || 0); + state.rangeSelectionEnd = Number( + payload.process_end_frame_index ?? Math.max(0, (payload.frame_count || 1) - 1) + ); + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + } + if ( + templateChangedOnServer + || state.templateFrameSelection === state.templateFrameApplied + || state.templateFrameApplied === null + ) { + state.templateFrameSelection = hasTemplateFrame(payload) + ? Number(payload.template_frame_index) + : Number(payload.process_start_frame_index || 0); + } + if (rangeChangedOnServer || templateChangedOnServer || state.playheadFrame === undefined || Number.isNaN(state.playheadFrame)) { + state.playheadFrame = hasTemplateFrame(payload) + ? Number(payload.template_frame_index) + : Number(payload.process_start_frame_index || 0); + } + + const currentTarget = activeTarget(payload); + syncTargetControls(payload); + + syncCanvasMode(payload); + syncCompareStrip(payload); + syncPresetButtons(payload); + + if (canvasStageNote) { + canvasStageNote.textContent = hasTemplateFrame(payload) + ? (payload.stage_note || "") + : "Range changed. Re-apply an anchor frame to continue annotation."; + } + if (guidanceTitle) { + guidanceTitle.textContent = payload.stage_label || "Coarse Selection"; + } + if (guidanceCopy) { + guidanceCopy.textContent = payload.stage_note || ""; + } + if (workflowStageChip) { + workflowStageChip.textContent = payload.stage_label || payload.stage; + } + if (selectionNote) { + selectionNote.textContent = !hasTemplateFrame(payload) + ? "Mark the processing segment, then choose an anchor frame before placing points." + : payload.stage === "preview" + ? "Preview is locked. Return to coarse or refine before placing more points." + : "Use points to establish the person first, then switch into presets or brush cleanup."; + } + if (brushNote) { + brushNote.textContent = !hasTemplateFrame(payload) + ? "Brush refinement stays locked until an anchor frame has been applied inside the green processing segment." + : payload.stage === "preview" + ? "Brush refinement is disabled in preview mode." + : "Brush actions edit the active mask directly, which is useful when SAM2 gets the rough silhouette but misses small edge corrections."; + } + + if (inspectorStage) { + inspectorStage.textContent = payload.stage_label || payload.stage; + } + if (inspectorTarget) { + inspectorTarget.textContent = currentTarget + ? `${currentTarget.name}${currentTarget.locked ? " | Locked" : ""}${currentTarget.visible ? "" : " | Hidden"}` + : "-"; + } + if (inspectorPoints) { + inspectorPoints.textContent = String(currentTarget?.point_count || 0); + } + if (inspectorMask) { + inspectorMask.textContent = currentTarget?.saved_mask_name || "Not saved yet"; + } + if (targetNameInput && currentTarget) { + targetNameInput.value = currentTarget.name; + targetNameInput.disabled = false; + } + syncKeyframeSummary(payload); + if (targetSummary) { + targetSummary.textContent = currentTarget + ? `${currentTarget.name} is ${currentTarget.visible ? "visible" : "hidden"}, ${currentTarget.locked ? "locked" : "editable"}, and uses the ${PRESET_META[currentTarget.refine_preset]?.label || "Balanced"} preset.` + : "No active target selected."; + } + + stageButtons.forEach((button) => { + button.toggleAttribute("data-active", button.dataset.stage === payload.stage); + }); + + renderTargets(payload.targets || [], payload.active_target_id); + renderSavedMasks(payload.mask_names || []); + syncActionState(payload); + } + + async function refreshWorkbench() { + const payload = await parseJson(await fetch(root.dataset.workbenchEndpoint)); + renderWorkbench(payload); + return payload; + } + + function setActiveTool(nextTool) { + state.activeTool = nextTool; + syncToolButtons(state.workbench); + } + + function setCanvasMode(nextCanvasMode) { + state.canvasMode = nextCanvasMode; + if (state.workbench) { + syncCanvasMode(state.workbench); + } + } + + positiveButton?.addEventListener("click", () => setActiveTool("point-positive")); + negativeButton?.addEventListener("click", () => setActiveTool("point-negative")); + + brushButtons.forEach((button) => { + button.addEventListener("click", () => { + setActiveTool(`brush-${button.dataset.brushMode}`); + setStatus(status, `${button.textContent} tool ready. Click the canvas to refine the mask.`, false); + }); + }); + + brushRadiusInput?.addEventListener("input", () => { + state.brushRadius = Number(brushRadiusInput.value); + if (brushRadiusValue) { + brushRadiusValue.value = `${state.brushRadius} px`; + brushRadiusValue.textContent = `${state.brushRadius} px`; + } + }); + + overlayOpacityInput?.addEventListener("input", () => { + state.overlayOpacity = Number(overlayOpacityInput.value); + updateRangeOutput(overlayOpacityValue, state.overlayOpacity); + applyImagePresentation(); + }); + + function seekVideoToFrame(frameIndex, payload = state.workbench) { + if (!keyframeVideo || !payload) { + return; + } + const desiredTime = frameToSeconds(frameIndex, payload); + if (!Number.isFinite(desiredTime)) { + return; + } + try { + keyframeVideo.currentTime = desiredTime; + } catch (_error) { + // Ignore transient seek failures before metadata is ready. + } + } + + function syncSourcePlaybackButton() { + if (!toggleSourcePlaybackButton || !keyframeVideo) { + return; + } + const canPlay = state.canvasMode === "source"; + toggleSourcePlaybackButton.disabled = !canPlay; + toggleSourcePlaybackButton.textContent = keyframeVideo.paused ? "Play" : "Pause"; + } + + function syncPlayheadFromVideo() { + const payload = state.workbench; + if (!payload) { + return; + } + const fps = Number(payload.fps || state.fps || 0); + if (!Number.isFinite(fps) || fps <= 0) { + return; + } + const nextFrame = clampFrame( + Math.round((keyframeVideo?.currentTime || 0) * fps), + 0, + Math.max(0, (payload.frame_count || 1) - 1) + ); + state.playheadFrame = nextFrame; + if (sourcePlayheadSlider) { + sourcePlayheadSlider.value = String(nextFrame); + } + syncKeyframeSummary(payload); + syncActionState(payload); + } + + function clearPendingRangeSelection() { + const payload = state.workbench; + if (!payload) { + return; + } + state.rangeSelectionStart = state.rangeAppliedStart; + state.rangeSelectionEnd = state.rangeAppliedEnd; + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + syncKeyframeSummary(payload); + syncActionState(payload); + } + + async function applyRangeSelection() { + const payload = state.workbench; + if (!payload) { + return; + } + const nextStart = Math.min(state.rangeSelectionStart, state.rangeSelectionEnd); + const nextEnd = Math.max(state.rangeSelectionStart, state.rangeSelectionEnd); + if ( + nextStart === state.rangeAppliedStart + && nextEnd === state.rangeAppliedEnd + ) { + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + syncKeyframeSummary(payload); + syncActionState(payload); + return; + } + + const hasExistingAnnotations = ( + (payload.mask_names || []).length > 0 + || hasTemplateFrame(payload) + || payload.current_mask_url + || payload.current_preview_url + ); + if (hasExistingAnnotations) { + const shouldContinue = window.confirm( + "Changing the processing segment will clear the current anchor, current preview, and every saved mask. Continue?" + ); + if (!shouldContinue) { + clearPendingRangeSelection(); + return; + } + } + + setStatus(status, `Applying segment ${nextStart} - ${nextEnd}...`, false); + try { + const nextPayload = await parseJson( + await fetch(root.dataset.processingRangeEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + start_frame_index: nextStart, + end_frame_index: nextEnd, + }), + }) + ); + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + state.canvasMode = "source"; + renderWorkbench(nextPayload); + setStatus( + status, + "Range changed. Re-apply an anchor frame to continue annotation.", + false + ); + } catch (error) { + setStatus(status, error.message, true); + } + } + + function markRangeBoundary(boundary) { + const payload = state.workbench; + if (!payload) { + return; + } + if (state.canvasMode !== "source") { + setCanvasMode("source"); + } + const frame = clampFrame( + state.playheadFrame, + 0, + Math.max(0, (payload.frame_count || 1) - 1) + ); + if (boundary === "start") { + state.rangeSelectionStart = frame; + state.rangeSelectionTouchedStart = true; + if (state.rangeSelectionStart > state.rangeSelectionEnd) { + state.rangeSelectionEnd = state.rangeSelectionStart; + } + } else { + state.rangeSelectionEnd = frame; + state.rangeSelectionTouchedEnd = true; + if (state.rangeSelectionEnd < state.rangeSelectionStart) { + state.rangeSelectionStart = state.rangeSelectionEnd; + } + } + + syncKeyframeSummary(payload); + syncActionState(payload); + if (state.rangeSelectionTouchedStart && state.rangeSelectionTouchedEnd) { + void applyRangeSelection(); + return; + } + + setStatus( + status, + boundary === "start" + ? `In point set to frame ${state.rangeSelectionStart}. Mark Out to confirm the segment.` + : `Out point set to frame ${state.rangeSelectionEnd}. Mark In to confirm the segment.`, + false + ); + } + + async function applyTemplateFrameSelection() { + const payload = state.workbench; + if (!payload || !templateFrameSlider || templateFrameSlider.disabled) { + return; + } + const nextFrame = clampFrame( + state.templateFrameSelection, + state.rangeAppliedStart, + state.rangeAppliedEnd + ); + if (hasTemplateFrame(payload) && nextFrame === Number(payload.template_frame_index || 0)) { + return; + } + setStatus(status, `Switching template frame to ${nextFrame}...`, false); + try { + const nextPayload = await parseJson( + await fetch(root.dataset.templateFrameEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ frame_index: nextFrame }), + }) + ); + state.playheadFrame = nextFrame; + renderWorkbench(nextPayload); + setStatus( + status, + `Template frame ${nextPayload.template_frame_index} is now active. Existing unsaved annotations were reset.`, + false + ); + } catch (error) { + setStatus(status, error.message, true); + } + } + + sourcePlayheadSlider?.addEventListener("input", () => { + const payload = state.workbench; + if (!payload) { + return; + } + if (state.canvasMode !== "source") { + setCanvasMode("source"); + } + state.playheadFrame = clampFrame( + Number(sourcePlayheadSlider.value), + 0, + Math.max(0, (payload.frame_count || 1) - 1) + ); + syncKeyframeSummary(payload); + syncActionState(payload); + seekVideoToFrame(state.playheadFrame, payload); + }); + + markRangeInButton?.addEventListener("click", () => { + if (markRangeInButton.disabled) { + return; + } + markRangeBoundary("start"); + }); + + markRangeOutButton?.addEventListener("click", () => { + if (markRangeOutButton.disabled) { + return; + } + markRangeBoundary("end"); + }); + + clearRangeSelectionButton?.addEventListener("click", () => { + if (clearRangeSelectionButton.disabled) { + return; + } + clearPendingRangeSelection(); + setStatus(status, "Pending range marks cleared.", false); + }); + + templateFrameSlider?.addEventListener("input", () => { + const payload = state.workbench; + if (!payload) { + return; + } + state.templateFrameSelection = clampFrame( + Number(templateFrameSlider.value), + state.rangeAppliedStart, + state.rangeAppliedEnd + ); + state.playheadFrame = state.templateFrameSelection; + syncKeyframeSummary(payload); + syncActionState(payload); + seekVideoToFrame(state.playheadFrame, payload); + }); + + templateFrameSlider?.addEventListener("change", () => { + void applyTemplateFrameSelection(); + }); + + toggleSourcePlaybackButton?.addEventListener("click", async () => { + if (!keyframeVideo || toggleSourcePlaybackButton.disabled) { + return; + } + try { + if (keyframeVideo.paused) { + await keyframeVideo.play(); + } else { + keyframeVideo.pause(); + } + } catch (_error) { + setStatus(status, "Video playback is temporarily unavailable.", true); + } + syncSourcePlaybackButton(); + }); + + keyframeVideo?.addEventListener("loadedmetadata", () => { + syncKeyframeSummary(state.workbench); + syncSourcePlaybackButton(); + }); + keyframeVideo?.addEventListener("seeked", syncPlayheadFromVideo); + keyframeVideo?.addEventListener("timeupdate", syncPlayheadFromVideo); + keyframeVideo?.addEventListener("play", syncSourcePlaybackButton); + keyframeVideo?.addEventListener("pause", syncSourcePlaybackButton); + + async function patchActiveTarget(patch, pendingMessage, successMessage) { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + await updateTarget( + currentTarget.target_id, + patch, + pendingMessage, + successMessage + ); + } + + function scheduleLiveTargetPatch(patch, pendingMessage, successMessage) { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + if (state.livePatchTimer) { + clearTimeout(state.livePatchTimer); + } + + const baselineSrc = image?.src || state.compareLiveSrc || withCacheBust(resolveCanvasUrl(state.workbench)); + state.livePatchTimer = window.setTimeout(async () => { + const revision = ++state.livePatchRevision; + setStatus(status, pendingMessage, false); + try { + const payload = await requestTargetPatch(currentTarget.target_id, patch); + if (revision < state.lastAppliedLivePatchRevision) { + return; + } + state.lastAppliedLivePatchRevision = revision; + state.compareBeforeSrc = baselineSrc; + renderWorkbench(payload); + setStatus(status, successMessage(payload), false); + } catch (error) { + setStatus(status, error.message, true); + } + }, 140); + } + + presetStrengthInput?.addEventListener("input", () => { + updateRangeOutput(presetStrengthValue, Number(presetStrengthInput.value)); + scheduleLiveTargetPatch( + { preset_strength: Number(presetStrengthInput.value) / 100 }, + "Refreshing live detail preview...", + () => "Preset strength updated." + ); + }); + + motionStrengthInput?.addEventListener("input", () => { + updateRangeOutput(motionStrengthValue, Number(motionStrengthInput.value)); + scheduleLiveTargetPatch( + { motion_strength: Number(motionStrengthInput.value) / 100 }, + "Refreshing live detail preview...", + () => "Motion softness updated." + ); + }); + + temporalStabilityInput?.addEventListener("input", () => { + updateRangeOutput(temporalStabilityValue, Number(temporalStabilityInput.value)); + scheduleLiveTargetPatch( + { temporal_stability: Number(temporalStabilityInput.value) / 100 }, + "Refreshing live detail preview...", + () => "Temporal stability updated." + ); + }); + + viewButtons.forEach((button) => { + button.addEventListener("click", () => { + if (button.disabled) { + return; + } + setCanvasMode(button.dataset.canvasMode); + }); + }); + + stageButtons.forEach((button) => { + button.addEventListener("click", async () => { + setStatus(status, `Switching to ${button.dataset.stage} stage...`, false); + try { + const payload = await parseJson( + await fetch(root.dataset.stageEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ stage: button.dataset.stage }), + }) + ); + renderWorkbench(payload); + setStatus(status, `${payload.stage_label} ready.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + }); + + function moveStage(offset) { + if (!state.workbench) { + return; + } + const currentIndex = STAGE_ORDER.indexOf(state.workbench.stage); + const nextIndex = Math.min(Math.max(currentIndex + offset, 0), STAGE_ORDER.length - 1); + const nextStage = STAGE_ORDER[nextIndex]; + if (nextStage === state.workbench.stage) { + return; + } + stageButtons.find((button) => button.dataset.stage === nextStage)?.click(); + } + + presetButtons.forEach((button) => { + button.addEventListener("click", async () => { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + await updateTarget( + currentTarget.target_id, + { refine_preset: button.dataset.preset }, + `Applying ${button.textContent} preset...`, + () => `${button.textContent} preset is now active for ${currentTarget.name}.` + ); + }); + }); + + createTargetButton?.addEventListener("click", async () => { + if (createTargetButton.disabled) { + return; + } + setStatus(status, "Creating a new target layer...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.targetsEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }) + ); + renderWorkbench(payload); + setStatus(status, `Created ${payload.name}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + applyTargetNameButton?.addEventListener("click", async () => { + const currentTarget = activeTarget(); + const nextName = targetNameInput?.value?.trim(); + if (!currentTarget || !nextName || nextName === currentTarget.name) { + return; + } + await updateTarget( + currentTarget.target_id, + { name: nextName }, + `Renaming ${currentTarget.name}...`, + () => `Renamed target to ${nextName}.` + ); + }); + + toggleTargetLockButton?.addEventListener("click", async () => { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + const nextLocked = !currentTarget.locked; + await updateTarget( + currentTarget.target_id, + { locked: nextLocked }, + `${nextLocked ? "Locking" : "Unlocking"} ${currentTarget.name}...`, + () => `${currentTarget.name} is now ${nextLocked ? "locked" : "editable"}.` + ); + }); + + undoButton?.addEventListener("click", async () => { + if (undoButton.disabled) { + return; + } + setStatus(status, "Removing the last click...", false); + try { + const payload = await parseJson( + await fetch(`${root.dataset.workbenchEndpoint}/undo`, { + method: "POST", + }) + ); + renderWorkbench(payload); + setStatus(status, "Removed the last click from the active target.", false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + resetButton?.addEventListener("click", async () => { + if (resetButton.disabled) { + return; + } + setStatus(status, "Resetting the active target...", false); + try { + const payload = await parseJson( + await fetch(`${root.dataset.workbenchEndpoint}/reset-target`, { + method: "POST", + }) + ); + renderWorkbench(payload); + setStatus(status, "Cleared the active target back to an empty click state.", false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + image?.addEventListener("click", async (event) => { + if (!state.workbench?.can_apply_clicks) { + setStatus( + status, + hasTemplateFrame(state.workbench) + ? "Preview mode is read-only. Switch back to coarse or refine to edit the target." + : "Range changed. Re-apply an anchor frame before editing the target.", + false + ); + return; + } + + const bounds = image.getBoundingClientRect(); + const scaleX = image.naturalWidth / bounds.width; + const scaleY = image.naturalHeight / bounds.height; + const x = Math.round((event.clientX - bounds.left) * scaleX); + const y = Math.round((event.clientY - bounds.top) * scaleY); + + try { + let payload; + if (state.activeTool.startsWith("brush-")) { + const brushMode = state.activeTool.replace("brush-", ""); + setStatus(status, `Applying ${brushMode} brush...`, false); + payload = await parseJson( + await fetch(root.dataset.brushEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + mode: brushMode, + radius: state.brushRadius, + points: [[x, y]], + }), + }) + ); + renderWorkbench(payload); + setStatus(status, `${brushMode} brush updated ${activeTarget(payload)?.name || "the active target"}.`, false); + return; + } + + setStatus(status, "Updating target preview...", false); + payload = await parseJson( + await fetch(root.dataset.clickEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ x, y, positive: state.activeTool === "point-positive" }), + }) + ); + renderWorkbench(payload); + setStatus( + status, + `${state.activeTool === "point-positive" ? "Positive" : "Negative"} point applied to ${activeTarget(payload)?.name || "target"}.`, + false + ); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + saveButton?.addEventListener("click", async () => { + if (saveButton.disabled) { + return; + } + setStatus(status, "Saving current target mask...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.saveEndpoint, { + method: "POST", + }) + ); + if (payload.mask_name) { + state.selectedMasks.add(payload.mask_name); + } + renderWorkbench(payload); + setStatus(status, `Saved ${payload.mask_name}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + submitButton?.addEventListener("click", async () => { + const selectedMasks = selectedMaskNames(); + if (submitButton.disabled || selectedMasks.length === 0) { + setStatus(status, "Select at least one saved mask before queueing the job.", true); + return; + } + setStatus(status, "Submitting queued job...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.submitEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + process_start_frame_index: state.workbench?.process_start_frame_index ?? 0, + process_end_frame_index: state.workbench?.process_end_frame_index ?? 0, + template_frame_index: state.workbench?.template_frame_index ?? 0, + selected_masks: selectedMasks, + }), + }) + ); + window.location.assign(`${root.dataset.jobPagePrefix}${payload.job_id}`); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + refreshWorkbench().catch((error) => { + setStatus(status, error.message, true); + }); + + document.addEventListener("keydown", (event) => { + if (event.defaultPrevented) { + return; + } + + const key = event.key; + const lowerKey = key.toLowerCase(); + const hasPrimaryModifier = event.ctrlKey || event.metaKey; + const typingContext = isTypingContext(event.target); + + if (hasPrimaryModifier && lowerKey === "s") { + event.preventDefault(); + saveButton?.click(); + return; + } + + if (hasPrimaryModifier && key === "Enter") { + event.preventDefault(); + submitButton?.click(); + return; + } + + if (event.altKey || hasPrimaryModifier || typingContext) { + return; + } + + switch (key) { + case "1": + case "2": + case "3": { + event.preventDefault(); + const stageButton = stageButtons.find((button) => button.dataset.stage === { + "1": "coarse", + "2": "refine", + "3": "preview", + }[key]); + stageButton?.click(); + return; + } + case "Backspace": + event.preventDefault(); + resetButton?.click(); + return; + case "ArrowLeft": + if (!typingContext) { + event.preventDefault(); + moveStage(-1); + return; + } + break; + case "ArrowRight": + if (!typingContext) { + event.preventDefault(); + moveStage(1); + return; + } + break; + default: + break; + } + + switch (lowerKey) { + case "i": + event.preventDefault(); + markRangeInButton?.click(); + return; + case "o": + event.preventDefault(); + markRangeOutButton?.click(); + return; + case "p": + event.preventDefault(); + setActiveTool("point-positive"); + setStatus(status, "Shortcut: Positive point tool.", false); + return; + case "n": + event.preventDefault(); + setActiveTool("point-negative"); + setStatus(status, "Shortcut: Negative point tool.", false); + return; + case "b": + event.preventDefault(); + setActiveTool("brush-add"); + setStatus(status, "Shortcut: Add brush tool.", false); + return; + case "e": + event.preventDefault(); + setActiveTool("brush-remove"); + setStatus(status, "Shortcut: Remove brush tool.", false); + return; + case "g": + event.preventDefault(); + setActiveTool("brush-feather"); + setStatus(status, "Shortcut: Feather brush tool.", false); + return; + case "t": + event.preventDefault(); + createTargetButton?.click(); + return; + case "u": + event.preventDefault(); + undoButton?.click(); + return; + case "r": + event.preventDefault(); + resetButton?.click(); + return; + case "f": + event.preventDefault(); + setCanvasMode("source"); + setStatus(status, "Shortcut: Source view.", false); + return; + case "v": + event.preventDefault(); + setCanvasMode("overlay"); + setStatus(status, "Shortcut: Overlay view.", false); + return; + case "m": + if (viewButtons.find((button) => button.dataset.canvasMode === "mask")?.disabled) { + return; + } + event.preventDefault(); + setCanvasMode("mask"); + setStatus(status, "Shortcut: Mask view.", false); + return; + default: + break; + } + }); +} + +document.addEventListener("DOMContentLoaded", bindWorkbench); diff --git a/matanyone2/webapp/static/workspace.js b/matanyone2/webapp/static/workspace.js new file mode 100644 index 0000000..c0449b8 --- /dev/null +++ b/matanyone2/webapp/static/workspace.js @@ -0,0 +1,2059 @@ +import { + formatDuration, + parseJson, + setStatus, + withCacheBust, +} from "/static/shared.js"; + +const PRESET_META = { + balanced: { + label: "Balanced", + note: "Use this when the silhouette is already close and you want an even starting point.", + }, + hair: { + label: "Hair Priority", + note: "Bias your cleanup around flyaway strands and soft hairline gaps before committing the layer.", + }, + edge: { + label: "Edge Priority", + note: "Use this when the boundary should stay tight around shoulders, jaw lines, or wardrobe edges.", + }, + motion: { + label: "Motion Blur", + note: "Use this when motion softness matters more than a perfectly hard cut on the outer contour.", + }, +}; + +const TERMINAL_JOB_STATES = new Set([ + "completed", + "completed_with_warning", + "failed", + "interrupted", +]); + +function bindWorkbench() { + const root = document.getElementById("workspace-app"); + if (!root) { + return; + } + + const status = document.getElementById("workspace-status"); + const image = document.getElementById("workspace-monitor-image"); + const canvasFrame = document.getElementById("workspace-monitor-frame"); + const saveButton = document.getElementById("save-mask"); + const submitButton = document.getElementById("submit-job"); + const createTargetButton = document.getElementById("create-target"); + const undoButton = document.getElementById("undo-click"); + const resetButton = document.getElementById("reset-target"); + const savedMaskList = document.getElementById("saved-mask-list"); + const targetList = document.getElementById("target-list"); + const positiveButton = document.getElementById("positive-mode"); + const negativeButton = document.getElementById("negative-mode"); + const brushButtons = Array.from(root.querySelectorAll(".brush-button")); + const presetButtons = Array.from(root.querySelectorAll(".preset-button")); + const workflowButtons = Array.from(root.querySelectorAll(".workflow-stepper__step")); + const sidebarTabButtons = Array.from(root.querySelectorAll(".workspace-sidebar-tab")); + const stageButtons = []; + const viewButtons = Array.from(root.querySelectorAll(".canvas-view-tab")); + const inspectorStage = document.getElementById("inspector-stage"); + const inspectorTarget = document.getElementById("inspector-target"); + const inspectorPreset = document.getElementById("inspector-preset"); + const inspectorPoints = document.getElementById("inspector-points"); + const inspectorMask = document.getElementById("inspector-mask"); + const canvasModeLabel = document.getElementById("canvas-mode-label"); + const canvasStageNote = document.getElementById("canvas-stage-note"); + const guidanceTitle = document.getElementById("stage-guidance-title"); + const guidanceCopy = document.getElementById("stage-guidance-copy"); + const selectionNote = document.getElementById("selection-note"); + const presetNote = document.getElementById("preset-note"); + const brushNote = document.getElementById("brush-note"); + const workflowStageChip = document.getElementById("workflow-stage-chip"); + const targetNameInput = document.getElementById("target-name-input"); + const applyTargetNameButton = document.getElementById("apply-target-name"); + const toggleTargetLockButton = document.getElementById("toggle-target-lock"); + const targetSummary = document.getElementById("target-summary"); + const brushRadiusInput = document.getElementById("brush-radius"); + const brushRadiusValue = document.getElementById("brush-radius-value"); + const overlayOpacityInput = document.getElementById("overlay-opacity"); + const overlayOpacityValue = document.getElementById("overlay-opacity-value"); + const sourcePlayheadSlider = document.getElementById("source-playhead-slider"); + const markRangeInButton = document.getElementById("mark-range-in"); + const markRangeOutButton = document.getElementById("mark-range-out"); + const clearRangeSelectionButton = document.getElementById("clear-range-selection"); + const toggleSourcePlaybackButton = document.getElementById("toggle-source-playback"); + const timelineCurrentLabel = document.getElementById("timeline-current-label"); + const timelineSelectedLabel = document.getElementById("timeline-selected-label"); + const timelineAppliedLabel = document.getElementById("timeline-applied-label"); + const timelineInChip = document.getElementById("timeline-in-chip"); + const timelineOutChip = document.getElementById("timeline-out-chip"); + const timelineDurationChip = document.getElementById("timeline-duration-chip"); + const timelineRangeRail = document.getElementById("clip-primary-rail"); + const timelineRangeSelection = document.getElementById("timeline-range-selection"); + const templateFrameSlider = document.getElementById("anchor-frame-slider"); + const templateFrameValue = document.getElementById("anchor-frame-value"); + const anchorRail = document.getElementById("anchor-rail"); + const anchorFrameSummary = null; + const keyframeVideo = document.getElementById("workspace-monitor-video"); + const keyframeSelectedLabel = document.getElementById("keyframe-selected-label"); + const keyframeAppliedLabel = document.getElementById("keyframe-applied-label"); + const keyframeTimeLabel = document.getElementById("keyframe-time-label"); + const presetStrengthInput = document.getElementById("preset-strength"); + const presetStrengthValue = document.getElementById("preset-strength-value"); + const motionStrengthInput = document.getElementById("motion-strength"); + const motionStrengthValue = document.getElementById("motion-strength-value"); + const temporalStabilityInput = document.getElementById("temporal-stability"); + const temporalStabilityValue = document.getElementById("temporal-stability-value"); + const edgeFeatherRadiusInput = document.getElementById("edge-feather-radius"); + const edgeFeatherRadiusValue = document.getElementById("edge-feather-radius-value"); + const reviewSidebar = document.getElementById("workspace-review-sidebar"); + const reviewSummaryList = document.getElementById("review-summary-list"); + const reviewSummaryListSide = document.getElementById("review-summary-list-side"); + const targetReviewList = document.getElementById("target-review-list"); + const artifactSummaryList = document.getElementById("artifact-summary-list"); + const jobTimeline = document.getElementById("job-timeline"); + const warningPanel = document.getElementById("warning-panel"); + const warningTitle = document.getElementById("warning-title"); + const warningCopy = document.getElementById("warning-copy"); + const overlayCanvas = document.getElementById("workspace-overlay-canvas"); + const overlayForegroundVideo = document.getElementById("workspace-overlay-foreground-video"); + const overlayAlphaVideo = document.getElementById("workspace-overlay-alpha-video"); + const workspaceNavBack = document.getElementById("workspace-nav-back"); + const workspaceNavNext = document.getElementById("workspace-nav-next"); + const workspaceReturnToClip = document.getElementById("workspace-return-to-clip"); + const workspaceReturnToRefine = document.getElementById("workspace-return-to-refine"); + + const state = { + activeTool: "point-positive", + canvasMode: root.dataset.defaultCanvasMode || "source", + workbench: null, + selectedMasks: new Set(), + brushRadius: Number(brushRadiusInput?.value || 28), + overlayOpacity: Number(overlayOpacityInput?.value || 72), + playheadFrame: Number(sourcePlayheadSlider?.value || 0), + rangeSelectionStart: 0, + rangeSelectionEnd: 0, + rangeSelectionTouchedStart: false, + rangeSelectionTouchedEnd: false, + rangeAppliedStart: 0, + rangeAppliedEnd: 0, + templateFrameSelection: Number(templateFrameSlider?.value || 0), + templateFrameApplied: templateFrameSlider?.value === "" ? null : Number(templateFrameSlider?.value || 0), + fps: Number(root.dataset.fps || 0), + durationSeconds: Number(root.dataset.durationSeconds || 0), + livePatchTimer: null, + livePatchRevision: 0, + lastAppliedLivePatchRevision: 0, + jobPollTimer: null, + reviewPayload: null, + reviewMode: "source", + overlayFrameHandle: null, + overlayVideosBound: false, + foregroundCanvas: document.createElement("canvas"), + alphaCanvas: document.createElement("canvas"), + }; + + const WORKFLOW_STEPS = ["clip", "mask", "refine", "review"]; + const SIDEBAR_TABS = ["targets", "refine", "export"]; + const foregroundContext = state.foregroundCanvas.getContext("2d", { willReadFrequently: true }); + const alphaContext = state.alphaCanvas.getContext("2d", { willReadFrequently: true }); + const overlayContext = overlayCanvas?.getContext("2d", { willReadFrequently: true }) || null; + + function isTypingContext(target) { + if (!(target instanceof HTMLElement)) { + return false; + } + const tagName = target.tagName; + return ( + tagName === "INPUT" || + tagName === "TEXTAREA" || + tagName === "SELECT" || + target.isContentEditable + ); + } + + function selectedMaskNames() { + return Array.from(state.selectedMasks).sort(); + } + + function activeTarget(payload = state.workbench) { + return payload?.targets?.find((target) => target.target_id === payload.active_target_id) || null; + } + + function activePreset(payload = state.workbench) { + return activeTarget(payload)?.refine_preset || "balanced"; + } + + function reviewStatusEndpoint(jobId = state.workbench?.latest_job_id) { + return jobId ? `/api/jobs/${jobId}` : null; + } + + function sourceVideoEndpointFor(jobId = state.workbench?.latest_job_id) { + return jobId ? `/api/jobs/${jobId}/source-video` : root.dataset.sourceVideoUrl; + } + + function reviewPreviewEndpoint(jobId, kind) { + return jobId ? `/api/jobs/${jobId}/artifacts/${kind}` : null; + } + + function workflowStepIndex(step = state.workbench?.workflow_step) { + return WORKFLOW_STEPS.indexOf(step || "clip"); + } + + function activeSidebarTab(payload = state.workbench) { + return payload?.active_sidebar_tab || "targets"; + } + + function updateRangeOutput(outputElement, value, suffix = "%") { + if (!outputElement) { + return; + } + outputElement.value = `${value}${suffix}`; + outputElement.textContent = `${value}${suffix}`; + } + + function syncSidebarPanels(payload = state.workbench) { + const activeTab = activeSidebarTab(payload); + sidebarTabButtons.forEach((button) => { + button.toggleAttribute("data-active", button.dataset.sidebarTab === activeTab); + }); + ["targets", "refine", "export"].forEach((tabName) => { + const panel = document.getElementById(`sidebar-panel-${tabName}`); + if (!panel) { + return; + } + panel.hidden = tabName !== activeTab; + }); + } + + function syncWorkflowStepper(payload = state.workbench) { + const step = payload?.workflow_step || "clip"; + workflowButtons.forEach((button) => { + const disableReview = button.dataset.workflowStep === "review" && !payload?.latest_job_id; + button.toggleAttribute("disabled", disableReview); + button.toggleAttribute("data-active", button.dataset.workflowStep === step); + }); + workspaceNavBack?.toggleAttribute("disabled", !payload?.can_go_back); + workspaceNavNext?.toggleAttribute("disabled", !payload?.can_go_next); + if (workspaceReturnToClip) { + workspaceReturnToClip.hidden = step !== "review"; + } + if (workspaceReturnToRefine) { + workspaceReturnToRefine.hidden = step !== "review"; + } + if (reviewSidebar) { + reviewSidebar.hidden = step !== "review"; + } + } + + function frameToSeconds(frameIndex, payload = state.workbench) { + const fps = Number(payload?.fps || state.fps || 0); + if (!Number.isFinite(fps) || fps <= 0) { + return 0; + } + return frameIndex / fps; + } + + function formatFrameTimestamp(frameIndex, payload = state.workbench) { + return formatDuration(frameToSeconds(frameIndex, payload)); + } + + function hasTemplateFrame(payload = state.workbench) { + return payload?.template_frame_index !== null && payload?.template_frame_index !== undefined; + } + + function clampFrame(frameIndex, minFrame, maxFrame) { + return Math.max(minFrame, Math.min(maxFrame, frameIndex)); + } + + function cancelOverlayLoop() { + if (state.overlayFrameHandle !== null) { + window.cancelAnimationFrame(state.overlayFrameHandle); + state.overlayFrameHandle = null; + } + } + + function clearOverlayCanvas() { + cancelOverlayLoop(); + if (overlayContext && overlayCanvas) { + overlayContext.clearRect(0, 0, overlayCanvas.width, overlayCanvas.height); + } + if (overlayCanvas) { + overlayCanvas.hidden = true; + } + overlayForegroundVideo?.pause(); + overlayAlphaVideo?.pause(); + } + + function ensureMediaSource(videoNode, url) { + if (!videoNode || !url) { + return false; + } + if (videoNode.dataset.assetUrl === url) { + return false; + } + videoNode.dataset.assetUrl = url; + videoNode.src = withCacheBust(url); + videoNode.load(); + return true; + } + + function previewCaption(mode, payload = state.reviewPayload || state.workbench) { + if (mode === "overlay") { + return payload?.preview_artifacts?.foreground && payload?.preview_artifacts?.alpha + ? "Overlay preview is compositing browser-safe foreground and alpha streams." + : "Overlay preview becomes available after foreground and alpha preview streams are ready."; + } + if (mode === "alpha") { + return payload?.preview_artifacts?.alpha + ? "Alpha preview is using the browser-safe matte stream." + : "Alpha preview becomes available after export preview generation."; + } + if (mode === "foreground") { + return payload?.preview_artifacts?.foreground + ? "Foreground preview is using the browser-safe rendered foreground stream." + : "Foreground preview becomes available after export preview generation."; + } + return "Source preview uses the browser-safe clip preview for clip selection and result comparison."; + } + + function renderReviewSummaryList(targetNode, summary, payload) { + if (!targetNode) { + return; + } + const rows = []; + if (summary) { + rows.push(["Source", summary.source_name || "Unknown source"]); + if (Number.isInteger(summary.process_start_frame_index) && Number.isInteger(summary.process_end_frame_index)) { + const fps = Number(summary.source_fps || 0); + const label = fps > 0 + ? `Frame ${summary.process_start_frame_index}-${summary.process_end_frame_index} | ${formatDuration(summary.process_start_frame_index / fps)} - ${formatDuration(summary.process_end_frame_index / fps)}` + : `Frame ${summary.process_start_frame_index}-${summary.process_end_frame_index}`; + rows.push(["Process range", label]); + } + rows.push(["Anchor", `Frame ${summary.template_frame_index ?? "-"}`]); + rows.push(["Selected masks", Array.isArray(summary.selected_masks) && summary.selected_masks.length ? summary.selected_masks.join(", ") : "None"]); + const presetMap = summary.selected_mask_presets || {}; + if (Object.keys(presetMap).length > 0) { + rows.push(["Presets", Object.entries(presetMap).map(([mask, preset]) => `${mask}: ${preset}`).join(" | ")]); + } + if (summary.process_range_duration_seconds) { + rows.push(["Duration", formatDuration(summary.process_range_duration_seconds)]); + } + } + if (payload?.status_label) { + rows.splice(1, 0, ["Status", payload.status_label]); + } + targetNode.innerHTML = ""; + rows.forEach(([labelText, valueText]) => { + const row = document.createElement("div"); + const dt = document.createElement("dt"); + const dd = document.createElement("dd"); + dt.textContent = labelText; + dd.textContent = valueText; + row.append(dt, dd); + targetNode.appendChild(row); + }); + } + + function renderReviewTargets(summary) { + if (!targetReviewList) { + return; + } + const selectedMasks = Array.isArray(summary?.selected_masks) ? summary.selected_masks : []; + const presetMap = summary?.selected_mask_presets || {}; + targetReviewList.innerHTML = ""; + if (selectedMasks.length === 0) { + const empty = document.createElement("li"); + empty.className = "target-review-card target-review-card--empty"; + empty.textContent = "No saved masks selected for export."; + targetReviewList.appendChild(empty); + return; + } + selectedMasks.forEach((maskName, index) => { + const item = document.createElement("li"); + item.className = "target-review-card"; + item.innerHTML = ` +
+
+

Target ${index + 1}

+

${maskName}

+
+ Included +
+ `; + const meta = document.createElement("dl"); + meta.className = "target-review-meta"; + [["Mask", maskName], ["Preset", presetMap[maskName] || "balanced"], ["Export", "Merged in current job"]].forEach(([labelText, valueText]) => { + const row = document.createElement("div"); + const dt = document.createElement("dt"); + const dd = document.createElement("dd"); + dt.textContent = labelText; + dd.textContent = valueText; + row.append(dt, dd); + meta.appendChild(row); + }); + item.appendChild(meta); + targetReviewList.appendChild(item); + }); + } + + function renderReviewArtifacts(artifactDetails) { + if (!artifactSummaryList) { + return; + } + artifactSummaryList.innerHTML = ""; + Object.values(artifactDetails || {}).forEach((artifact) => { + const item = document.createElement("li"); + item.className = "artifact-card"; + item.dataset.available = artifact.available ? "true" : "false"; + item.innerHTML = ` +
+
+

${artifact.label}

+

${artifact.name}

+
+ ${artifact.available ? "Ready" : "Pending"} +
+

${artifact.available ? `${artifact.kind.replace("_", " ")} | ${artifact.size_label || "Available"}` : `${artifact.kind.replace("_", " ")} | Waiting for export`}

+ `; + if (artifact.available && artifact.url) { + const link = document.createElement("a"); + link.className = "artifact-card__link"; + link.href = artifact.url; + link.textContent = "Download"; + item.appendChild(link); + } + artifactSummaryList.appendChild(item); + }); + } + + function renderReviewTimeline(timeline) { + if (!jobTimeline) { + return; + } + jobTimeline.innerHTML = ""; + (timeline || []).forEach((step) => { + const item = document.createElement("li"); + item.className = "timeline-step"; + item.dataset.state = step.state; + item.innerHTML = ` + +
+

${step.label}

+

${step.state}

+
+ `; + jobTimeline.appendChild(item); + }); + } + + function renderReviewWarning(payload) { + if (!warningPanel || !warningTitle || !warningCopy) { + return; + } + const copy = payload?.error_text || payload?.warning_text; + if (!copy) { + warningPanel.hidden = true; + warningTitle.textContent = ""; + warningCopy.textContent = ""; + return; + } + warningPanel.hidden = false; + warningPanel.dataset.state = payload.error_text ? "error" : "warning"; + warningTitle.textContent = payload.error_text ? "Failure reported" : "Warning"; + warningCopy.textContent = copy; + } + + function drawOverlayFrame() { + if ( + state.canvasMode !== "overlay" || + !overlayCanvas || + !overlayContext || + !keyframeVideo || + !overlayForegroundVideo || + !overlayAlphaVideo + ) { + cancelOverlayLoop(); + return; + } + + if ( + overlayForegroundVideo.readyState < 2 || + overlayAlphaVideo.readyState < 2 || + keyframeVideo.readyState < 2 + ) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + return; + } + + const width = overlayForegroundVideo.videoWidth || keyframeVideo.videoWidth; + const height = overlayForegroundVideo.videoHeight || keyframeVideo.videoHeight; + if (!width || !height) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + return; + } + + if (overlayCanvas.width !== width || overlayCanvas.height !== height) { + overlayCanvas.width = width; + overlayCanvas.height = height; + state.foregroundCanvas.width = width; + state.foregroundCanvas.height = height; + state.alphaCanvas.width = width; + state.alphaCanvas.height = height; + } + + foregroundContext.clearRect(0, 0, width, height); + alphaContext.clearRect(0, 0, width, height); + foregroundContext.drawImage(overlayForegroundVideo, 0, 0, width, height); + alphaContext.drawImage(overlayAlphaVideo, 0, 0, width, height); + + const foregroundFrame = foregroundContext.getImageData(0, 0, width, height); + const alphaFrame = alphaContext.getImageData(0, 0, width, height); + const composed = foregroundFrame.data; + const matte = alphaFrame.data; + + for (let index = 0; index < composed.length; index += 4) { + composed[index + 3] = matte[index]; + } + + overlayContext.clearRect(0, 0, width, height); + overlayContext.putImageData(foregroundFrame, 0, 0); + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + } + + function syncOverlayPlayback() { + if (!keyframeVideo || !overlayForegroundVideo || !overlayAlphaVideo || state.canvasMode !== "overlay") { + return; + } + const targetTime = keyframeVideo.currentTime || 0; + const tolerance = 0.08; + [overlayForegroundVideo, overlayAlphaVideo].forEach((videoNode) => { + try { + if (Math.abs((videoNode.currentTime || 0) - targetTime) > tolerance) { + videoNode.currentTime = targetTime; + } + } catch (_error) { + // Ignore sync jitter while metadata is loading. + } + videoNode.playbackRate = keyframeVideo.playbackRate || 1; + if (keyframeVideo.paused) { + videoNode.pause(); + } else { + videoNode.play().catch(() => {}); + } + }); + } + + function bindOverlayVideoSync() { + if (!keyframeVideo || state.overlayVideosBound) { + return; + } + const syncAndMaybeDraw = () => { + syncOverlayPlayback(); + if (state.canvasMode === "overlay" && state.overlayFrameHandle === null) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + } + }; + ["play", "pause", "seeking", "seeked", "timeupdate", "ratechange", "loadeddata"].forEach((eventName) => { + keyframeVideo.addEventListener(eventName, syncAndMaybeDraw); + }); + state.overlayVideosBound = true; + } + + function renderReviewPayload(payload) { + state.reviewPayload = payload; + renderReviewSummaryList(reviewSummaryList, payload.job_summary, payload); + renderReviewSummaryList(reviewSummaryListSide, payload.job_summary, payload); + renderReviewTargets(payload.job_summary); + renderReviewArtifacts(payload.artifact_details); + renderReviewTimeline(payload.timeline); + renderReviewWarning(payload); + } + + async function refreshReview(jobId = state.workbench?.latest_job_id) { + const endpoint = reviewStatusEndpoint(jobId); + if (!endpoint) { + return null; + } + const payload = await parseJson(await fetch(endpoint)); + renderReviewPayload(payload); + return payload; + } + + function stopReviewPolling() { + if (state.jobPollTimer !== null) { + window.clearInterval(state.jobPollTimer); + state.jobPollTimer = null; + } + } + + function ensureReviewPolling() { + if (state.jobPollTimer !== null || !state.workbench?.latest_job_id) { + return; + } + state.jobPollTimer = window.setInterval(async () => { + try { + const payload = await refreshReview(); + if (payload && TERMINAL_JOB_STATES.has(payload.status)) { + stopReviewPolling(); + } + if (state.workbench?.workflow_step === "review") { + syncCanvasMode(state.workbench); + } + } catch (error) { + stopReviewPolling(); + setStatus(status, error.message, true); + } + }, 2000); + } + + function syncTimelineRangeRail(payload) { + if (!timelineRangeRail || !timelineRangeSelection || !payload) { + return; + } + const maxFrame = Math.max(1, (payload.frame_count || 1) - 1); + const pendingStart = Math.min(state.rangeSelectionStart, state.rangeSelectionEnd); + const pendingEnd = Math.max(state.rangeSelectionStart, state.rangeSelectionEnd); + const appliedStartPercent = (state.rangeAppliedStart / maxFrame) * 100; + const appliedEndPercent = (state.rangeAppliedEnd / maxFrame) * 100; + const pendingStartPercent = (pendingStart / maxFrame) * 100; + const pendingEndPercent = (pendingEnd / maxFrame) * 100; + const rangeDirty = pendingStart !== state.rangeAppliedStart || pendingEnd !== state.rangeAppliedEnd; + + timelineRangeRail.style.setProperty("--applied-range-start", `${appliedStartPercent}%`); + timelineRangeRail.style.setProperty("--applied-range-end", `${appliedEndPercent}%`); + timelineRangeRail.style.setProperty("--pending-range-start", `${pendingStartPercent}%`); + timelineRangeRail.style.setProperty("--pending-range-end", `${pendingEndPercent}%`); + timelineRangeRail.dataset.rangeState = rangeDirty ? "pending" : "applied"; + timelineRangeSelection.dataset.rangeState = rangeDirty ? "pending" : "applied"; + } + + function syncKeyframeSummary(payload) { + if (!payload) { + return; + } + const maxFrame = Math.max(0, (payload.frame_count || 1) - 1); + state.fps = Number(payload.fps || state.fps || 0); + state.durationSeconds = Number(payload.duration_seconds || state.durationSeconds || 0); + state.rangeAppliedStart = Number(payload.process_start_frame_index || 0); + state.rangeAppliedEnd = Number( + payload.process_end_frame_index ?? maxFrame + ); + + if ( + state.rangeSelectionStart === undefined + || Number.isNaN(state.rangeSelectionStart) + || state.rangeSelectionStart < 0 + ) { + state.rangeSelectionStart = state.rangeAppliedStart; + } + if ( + state.rangeSelectionEnd === undefined + || Number.isNaN(state.rangeSelectionEnd) + || state.rangeSelectionEnd < 0 + ) { + state.rangeSelectionEnd = state.rangeAppliedEnd; + } + + state.rangeSelectionStart = clampFrame(state.rangeSelectionStart, 0, maxFrame); + state.rangeSelectionEnd = clampFrame(state.rangeSelectionEnd, 0, maxFrame); + if (state.rangeSelectionStart > state.rangeSelectionEnd) { + const nextStart = state.rangeSelectionEnd; + state.rangeSelectionEnd = state.rangeSelectionStart; + state.rangeSelectionStart = nextStart; + } + + state.templateFrameApplied = hasTemplateFrame(payload) + ? Number(payload.template_frame_index) + : null; + if ( + state.templateFrameSelection === undefined + || Number.isNaN(state.templateFrameSelection) + || state.templateFrameSelection < state.rangeAppliedStart + || state.templateFrameSelection > state.rangeAppliedEnd + ) { + state.templateFrameSelection = state.templateFrameApplied ?? state.rangeAppliedStart; + } + if ( + state.playheadFrame === undefined + || Number.isNaN(state.playheadFrame) + || state.playheadFrame < 0 + || state.playheadFrame > maxFrame + ) { + state.playheadFrame = state.templateFrameApplied ?? state.rangeAppliedStart; + } + + if (sourcePlayheadSlider) { + sourcePlayheadSlider.max = String(maxFrame); + sourcePlayheadSlider.value = String(clampFrame(state.playheadFrame, 0, maxFrame)); + } + + if (timelineCurrentLabel) { + timelineCurrentLabel.textContent = `Playhead ยท Frame ${state.playheadFrame} ยท ${formatFrameTimestamp(state.playheadFrame, payload)}`; + } + if (timelineSelectedLabel) { + timelineSelectedLabel.textContent = `Pending range ยท Frame ${state.rangeSelectionStart} - ${state.rangeSelectionEnd}`; + } + if (timelineAppliedLabel) { + timelineAppliedLabel.textContent = `Processing range ยท Frame ${state.rangeAppliedStart} - ${state.rangeAppliedEnd}`; + } + if (timelineInChip) { + timelineInChip.textContent = `In ยท ${formatFrameTimestamp(state.rangeSelectionStart, payload)} ยท F${state.rangeSelectionStart}`; + } + if (timelineOutChip) { + timelineOutChip.textContent = `Out ยท ${formatFrameTimestamp(state.rangeSelectionEnd, payload)} ยท F${state.rangeSelectionEnd}`; + } + if (timelineDurationChip) { + const durationFrames = Math.max(1, state.rangeSelectionEnd - state.rangeSelectionStart + 1); + const fps = Number(payload.fps || state.fps || 0); + const duration = fps > 0 ? durationFrames / fps : 0; + timelineDurationChip.textContent = `Duration ยท ${formatDuration(duration)} ยท ${durationFrames}f`; + } + + if (templateFrameSlider) { + templateFrameSlider.min = String(state.rangeAppliedStart); + templateFrameSlider.max = String(state.rangeAppliedEnd); + templateFrameSlider.value = String(state.templateFrameSelection); + } + updateRangeOutput(templateFrameValue, Number(state.templateFrameSelection || 0), ""); + + if (anchorFrameSummary) { + anchorFrameSummary.textContent = state.templateFrameApplied === null + ? "Anchor ยท Not set" + : `Anchor ยท Frame ${state.templateFrameApplied} ยท ${formatFrameTimestamp(state.templateFrameApplied, payload)}`; + } + if (keyframeSelectedLabel) { + keyframeSelectedLabel.textContent = `Selected frame ${state.templateFrameSelection}`; + } + if (keyframeAppliedLabel) { + keyframeAppliedLabel.textContent = state.templateFrameApplied === null + ? "Applied frame Not set" + : `Applied frame ${state.templateFrameApplied}`; + } + if (keyframeTimeLabel) { + keyframeTimeLabel.textContent = `${formatFrameTimestamp(state.templateFrameSelection, payload)} / ${formatFrameTimestamp(state.rangeAppliedEnd, payload)}`; + } + + syncTimelineRangeRail(payload); + + if (keyframeVideo) { + if (!keyframeVideo.src) { + keyframeVideo.src = root.dataset.sourceVideoUrl; + } + const desiredTime = frameToSeconds(state.playheadFrame, payload); + if (Number.isFinite(desiredTime) && Math.abs((keyframeVideo.currentTime || 0) - desiredTime) > 0.04) { + try { + keyframeVideo.currentTime = desiredTime; + } catch (_error) { + // Ignore transient seek failures before metadata is ready. + } + } + } + } + + function syncTargetControls(payload) { + const currentTarget = activeTarget(payload); + if (!currentTarget) { + return; + } + if (presetStrengthInput) { + presetStrengthInput.value = String(Math.round((currentTarget.preset_strength || 0) * 100)); + updateRangeOutput(presetStrengthValue, Number(presetStrengthInput.value)); + } + if (motionStrengthInput) { + motionStrengthInput.value = String(Math.round((currentTarget.motion_strength || 0) * 100)); + updateRangeOutput(motionStrengthValue, Number(motionStrengthInput.value)); + } + if (temporalStabilityInput) { + temporalStabilityInput.value = String(Math.round((currentTarget.temporal_stability || 0) * 100)); + updateRangeOutput(temporalStabilityValue, Number(temporalStabilityInput.value)); + } + if (edgeFeatherRadiusInput) { + edgeFeatherRadiusInput.value = String(Math.round(currentTarget.edge_feather_radius || 0)); + updateRangeOutput(edgeFeatherRadiusValue, Number(edgeFeatherRadiusInput.value), " px"); + } + } + + function applyImagePresentation() { + if (!image || !keyframeVideo) { + return; + } + const reviewMode = state.workbench?.workflow_step === "review"; + const showVideo = reviewMode || state.canvasMode === "source" || state.canvasMode === "alpha" || state.canvasMode === "foreground"; + keyframeVideo.hidden = !showVideo; + image.hidden = showVideo || reviewMode; + canvasFrame?.setAttribute("data-canvas-mode", state.canvasMode); + + if (showVideo) { + image.style.opacity = "1"; + syncSourcePlaybackButton(); + return; + } + if (!keyframeVideo.paused) { + keyframeVideo.pause(); + } + image.style.opacity = String(state.overlayOpacity / 100); + syncSourcePlaybackButton(); + } + + function syncSelectedMasks(payload) { + const availableMaskNames = payload.mask_names || []; + const available = new Set(availableMaskNames); + const serverSelected = new Set(payload.selected_mask_names || []); + + if (state.selectedMasks.size === 0) { + state.selectedMasks = serverSelected.size > 0 + ? serverSelected + : new Set(availableMaskNames); + return; + } + + state.selectedMasks = new Set( + Array.from(state.selectedMasks).filter((maskName) => available.has(maskName)) + ); + if (state.selectedMasks.size === 0 && serverSelected.size > 0) { + state.selectedMasks = serverSelected; + } + } + + function syncToolButtons(payload) { + const canEdit = payload?.can_apply_clicks ?? false; + const pointPositive = state.activeTool === "point-positive"; + const pointNegative = state.activeTool === "point-negative"; + + positiveButton?.toggleAttribute("data-active", pointPositive); + negativeButton?.toggleAttribute("data-active", pointNegative); + positiveButton?.toggleAttribute("disabled", !canEdit); + negativeButton?.toggleAttribute("disabled", !canEdit); + + brushButtons.forEach((button) => { + const isActive = state.activeTool === `brush-${button.dataset.brushMode}`; + button.toggleAttribute("data-active", isActive); + button.toggleAttribute("disabled", !canEdit); + }); + + if (image) { + image.dataset.editable = canEdit ? "true" : "false"; + image.style.cursor = canEdit ? "crosshair" : "default"; + } + } + + function renderSavedMasks(maskNames) { + if (!savedMaskList) { + return; + } + savedMaskList.innerHTML = ""; + maskNames.forEach((maskName) => { + const label = document.createElement("label"); + const input = document.createElement("input"); + input.type = "checkbox"; + input.name = "mask_name"; + input.value = maskName; + input.checked = state.selectedMasks.has(maskName); + input.addEventListener("change", () => { + if (input.checked) { + state.selectedMasks.add(maskName); + } else { + state.selectedMasks.delete(maskName); + } + syncActionState(state.workbench); + }); + label.appendChild(input); + label.append(` ${maskName}`); + savedMaskList.appendChild(label); + }); + } + + async function updateTarget(targetId, patch, pendingMessage, successMessage) { + setStatus(status, pendingMessage, false); + try { + const payload = await requestTargetPatch(targetId, patch); + renderWorkbench(payload); + setStatus(status, successMessage(payload), false); + } catch (error) { + setStatus(status, error.message, true); + } + } + + async function requestTargetPatch(targetId, patch) { + return parseJson( + await fetch(`${root.dataset.targetsEndpoint}/${targetId}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(patch), + }) + ); + } + + function renderTargets(targets, activeTargetId) { + if (!targetList) { + return; + } + targetList.innerHTML = ""; + targets.forEach((target) => { + const card = document.createElement("article"); + card.className = "target-card"; + card.dataset.selected = target.target_id === activeTargetId ? "true" : "false"; + card.dataset.hidden = target.visible ? "false" : "true"; + card.dataset.locked = target.locked ? "true" : "false"; + + const selectButton = document.createElement("button"); + selectButton.type = "button"; + selectButton.className = "target-card__select"; + selectButton.innerHTML = ` + ${target.name} + ${target.point_count} point${target.point_count === 1 ? "" : "s"} | ${PRESET_META[target.refine_preset]?.label || "Balanced"} | ${target.saved_mask_name || "unsaved"} | ${target.visible ? "visible" : "hidden"} | ${target.locked ? "locked" : "editable"} + `; + selectButton.addEventListener("click", async () => { + if (target.target_id === activeTargetId) { + return; + } + setStatus(status, `Switching to ${target.name}...`, false); + try { + const payload = await parseJson( + await fetch(`${root.dataset.targetsEndpoint}/${target.target_id}/select`, { + method: "POST", + }) + ); + renderWorkbench(payload); + setStatus(status, `Active target: ${target.name}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + const actions = document.createElement("div"); + actions.className = "target-card__actions"; + + const visibilityButton = document.createElement("button"); + visibilityButton.type = "button"; + visibilityButton.className = "target-chip"; + visibilityButton.textContent = target.visible ? "Hide" : "Show"; + visibilityButton.addEventListener("click", () => { + const nextVisible = !target.visible; + updateTarget( + target.target_id, + { visible: nextVisible }, + `${nextVisible ? "Showing" : "Hiding"} ${target.name}...`, + () => `${target.name} is now ${nextVisible ? "visible" : "hidden"}.` + ); + }); + + const lockButton = document.createElement("button"); + lockButton.type = "button"; + lockButton.className = "target-chip"; + lockButton.textContent = target.locked ? "Unlock" : "Lock"; + lockButton.addEventListener("click", () => { + const nextLocked = !target.locked; + updateTarget( + target.target_id, + { locked: nextLocked }, + `${nextLocked ? "Locking" : "Unlocking"} ${target.name}...`, + () => `${target.name} is now ${nextLocked ? "locked" : "editable"}.` + ); + }); + + actions.append(visibilityButton, lockButton); + card.append(selectButton, actions); + targetList.appendChild(card); + }); + } + + function syncActionState(payload) { + if (!payload) { + return; + } + const canSubmit = payload.workflow_step !== "review" + && payload.can_submit + && state.selectedMasks.size > 0 + && hasTemplateFrame(payload); + const rangeDirty = ( + state.rangeSelectionStart !== state.rangeAppliedStart + || state.rangeSelectionEnd !== state.rangeAppliedEnd + ); + const rangePendingCompletion = ( + state.rangeSelectionTouchedStart !== state.rangeSelectionTouchedEnd + ); + root.dataset.stage = payload.stage; + root.dataset.editable = payload.can_apply_clicks ? "true" : "false"; + root.dataset.workflowStep = payload.workflow_step || "clip"; + + createTargetButton?.toggleAttribute("disabled", !payload.can_create_target); + undoButton?.toggleAttribute("disabled", !payload.can_undo_clicks); + resetButton?.toggleAttribute("disabled", !payload.can_reset_target); + saveButton?.toggleAttribute("disabled", !payload.can_save_current_target); + submitButton?.toggleAttribute("disabled", !canSubmit); + applyTargetNameButton?.toggleAttribute("disabled", !payload.active_target_id); + markRangeInButton?.toggleAttribute("disabled", !payload.can_apply_range); + markRangeOutButton?.toggleAttribute("disabled", !payload.can_apply_range); + clearRangeSelectionButton?.toggleAttribute("disabled", !payload.can_apply_range); + sourcePlayheadSlider?.toggleAttribute("disabled", !payload.can_apply_range); + templateFrameSlider?.toggleAttribute("disabled", !payload.can_change_template_frame || rangeDirty || rangePendingCompletion); + toggleSourcePlaybackButton?.toggleAttribute("disabled", state.canvasMode !== "source"); + const currentTarget = activeTarget(payload); + toggleTargetLockButton?.toggleAttribute("disabled", !currentTarget); + if (toggleTargetLockButton && currentTarget) { + toggleTargetLockButton.textContent = currentTarget.locked ? "Unlock Target" : "Lock Target"; + } + + if (submitButton) { + submitButton.textContent = payload.workflow_step === "review" + ? "Job Queued" + : "Submit Matting Job"; + } + if (timelineSelectedLabel) { + timelineSelectedLabel.dataset.pending = rangeDirty || rangePendingCompletion ? "true" : "false"; + } + if (anchorRail) { + anchorRail.hidden = !hasTemplateFrame(payload) && payload.workflow_step === "clip"; + } + + syncToolButtons(payload); + syncWorkflowStepper(payload); + syncSidebarPanels(payload); + } + + function resolveCanvasUrl(payload) { + if (state.canvasMode === "mask") { + return payload.active_mask_url || payload.current_mask_url || payload.template_frame_url || root.dataset.templateFrameUrl; + } + if (state.canvasMode === "source") { + return payload.template_frame_url || root.dataset.templateFrameUrl; + } + return payload.current_preview_url || payload.template_frame_url || root.dataset.templateFrameUrl; + } + + function syncCanvasMode(payload) { + const reviewMode = payload.workflow_step === "review"; + if (!reviewMode && !hasTemplateFrame(payload) && state.canvasMode !== "source") { + state.canvasMode = "source"; + } + if (!reviewMode && state.canvasMode === "mask" && !payload.active_mask_url && !payload.current_mask_url) { + state.canvasMode = payload.current_preview_url ? "overlay" : "source"; + } + + const modeLabelMap = { + source: "Source plate", + overlay: reviewMode ? "Overlay review" : "Overlay preview", + mask: "Mask inspection", + alpha: "Alpha review", + foreground: "Foreground review", + }; + if (canvasModeLabel) { + canvasModeLabel.textContent = `${payload.canvas_mode_label || "Guided silhouette pass"} | ${modeLabelMap[state.canvasMode] || "Source plate"}`; + } + + viewButtons.forEach((button) => { + const mode = button.dataset.canvasMode; + const isReviewOnly = mode === "alpha" || mode === "foreground"; + const disabled = reviewMode + ? (mode === "overlay" && !(state.reviewPayload?.preview_artifacts?.foreground && state.reviewPayload?.preview_artifacts?.alpha)) + || (mode === "alpha" && !state.reviewPayload?.preview_artifacts?.alpha) + || (mode === "foreground" && !state.reviewPayload?.preview_artifacts?.foreground) + || false + : isReviewOnly + || (mode === "mask" && (!hasTemplateFrame(payload) || (!payload.active_mask_url && !payload.current_mask_url))) + || (mode === "overlay" && !hasTemplateFrame(payload)); + button.toggleAttribute("disabled", disabled); + button.toggleAttribute("data-active", mode === state.canvasMode); + }); + + if (reviewMode) { + const sourceUrl = sourceVideoEndpointFor(payload.latest_job_id); + const foregroundUrl = reviewPreviewEndpoint(payload.latest_job_id, "preview_foreground.mp4"); + const alphaUrl = reviewPreviewEndpoint(payload.latest_job_id, "preview_alpha.mp4"); + clearOverlayCanvas(); + image.hidden = true; + if (state.canvasMode === "overlay" && sourceUrl && foregroundUrl && alphaUrl) { + bindOverlayVideoSync(); + ensureMediaSource(keyframeVideo, sourceUrl); + ensureMediaSource(overlayForegroundVideo, foregroundUrl); + ensureMediaSource(overlayAlphaVideo, alphaUrl); + keyframeVideo.hidden = false; + overlayCanvas.hidden = false; + syncOverlayPlayback(); + if (state.overlayFrameHandle === null) { + state.overlayFrameHandle = window.requestAnimationFrame(drawOverlayFrame); + } + } else { + const previewUrl = state.canvasMode === "alpha" + ? alphaUrl + : state.canvasMode === "foreground" + ? foregroundUrl + : sourceUrl; + ensureMediaSource(keyframeVideo, previewUrl || sourceUrl); + keyframeVideo.hidden = false; + } + return; + } + + clearOverlayCanvas(); + ensureMediaSource(keyframeVideo, root.dataset.sourceVideoUrl); + if (image) { + image.src = withCacheBust(resolveCanvasUrl(payload)); + image.alt = { + source: `Template frame for ${payload.draft_id}`, + overlay: `Overlay preview for ${payload.draft_id}`, + mask: `Mask preview for ${payload.draft_id}`, + }[state.canvasMode]; + } + applyImagePresentation(); + } + + function syncPresetButtons(payload) { + const preset = activePreset(payload); + presetButtons.forEach((button) => { + button.toggleAttribute("data-active", button.dataset.preset === preset); + }); + if (inspectorPreset) { + inspectorPreset.textContent = PRESET_META[preset]?.label || "Balanced"; + } + if (presetNote) { + presetNote.textContent = PRESET_META[preset]?.note || PRESET_META.balanced.note; + } + } + + function renderWorkbench(payload) { + const rangeChangedOnServer = ( + state.rangeAppliedStart !== Number(payload.process_start_frame_index || 0) + || state.rangeAppliedEnd !== Number(payload.process_end_frame_index ?? Math.max(0, (payload.frame_count || 1) - 1)) + ); + const templateChangedOnServer = ( + state.templateFrameApplied !== (hasTemplateFrame(payload) ? Number(payload.template_frame_index) : null) + ); + state.workbench = payload; + syncSelectedMasks(payload); + if (rangeChangedOnServer || (!state.rangeSelectionTouchedStart && !state.rangeSelectionTouchedEnd)) { + state.rangeSelectionStart = Number(payload.process_start_frame_index || 0); + state.rangeSelectionEnd = Number( + payload.process_end_frame_index ?? Math.max(0, (payload.frame_count || 1) - 1) + ); + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + } + if ( + templateChangedOnServer + || state.templateFrameSelection === state.templateFrameApplied + || state.templateFrameApplied === null + ) { + state.templateFrameSelection = hasTemplateFrame(payload) + ? Number(payload.template_frame_index) + : Number(payload.process_start_frame_index || 0); + } + if (rangeChangedOnServer || templateChangedOnServer || state.playheadFrame === undefined || Number.isNaN(state.playheadFrame)) { + state.playheadFrame = hasTemplateFrame(payload) + ? Number(payload.template_frame_index) + : Number(payload.process_start_frame_index || 0); + } + + const currentTarget = activeTarget(payload); + syncTargetControls(payload); + + syncCanvasMode(payload); + syncPresetButtons(payload); + + if (canvasStageNote) { + canvasStageNote.textContent = hasTemplateFrame(payload) + ? (payload.stage_note || "") + : "Range changed. Re-apply an anchor frame to continue annotation."; + } + if (guidanceTitle) { + guidanceTitle.textContent = payload.stage_label || "Coarse Selection"; + } + if (guidanceCopy) { + guidanceCopy.textContent = payload.stage_note || ""; + } + if (workflowStageChip) { + workflowStageChip.textContent = (payload.workflow_step || payload.stage || "clip").toUpperCase(); + } + if (selectionNote) { + selectionNote.textContent = !hasTemplateFrame(payload) + ? "Mark the processing segment, then choose an anchor frame before placing points." + : payload.stage === "preview" + ? "Preview is locked. Return to coarse or refine before placing more points." + : "Use points to establish the person first, then switch into presets or brush cleanup."; + } + if (brushNote) { + brushNote.textContent = !hasTemplateFrame(payload) + ? "Brush refinement stays locked until an anchor frame has been applied inside the green processing segment." + : payload.stage === "preview" + ? "Brush refinement is disabled in preview mode." + : "Brush actions edit the active mask directly, which is useful when SAM3 gets the rough silhouette but misses small edge corrections."; + } + if (presetNote && payload.active_sidebar_tab === "refine") { + presetNote.textContent = `${PRESET_META[activePreset(payload)]?.note || PRESET_META.balanced.note} Every refine control updates the main monitor directly.`; + } + + if (inspectorStage) { + inspectorStage.textContent = payload.workflow_step || payload.stage; + } + if (inspectorTarget) { + inspectorTarget.textContent = currentTarget + ? `${currentTarget.name}${currentTarget.locked ? " | Locked" : ""}${currentTarget.visible ? "" : " | Hidden"}` + : "-"; + } + if (inspectorPoints) { + inspectorPoints.textContent = String(currentTarget?.point_count || 0); + } + if (inspectorMask) { + inspectorMask.textContent = currentTarget?.saved_mask_name || "Not saved yet"; + } + if (targetNameInput && currentTarget) { + targetNameInput.value = currentTarget.name; + targetNameInput.disabled = false; + } + syncKeyframeSummary(payload); + if (targetSummary) { + targetSummary.textContent = currentTarget + ? `${currentTarget.name} is ${currentTarget.visible ? "visible" : "hidden"}, ${currentTarget.locked ? "locked" : "editable"}, and uses the ${PRESET_META[currentTarget.refine_preset]?.label || "Balanced"} preset.` + : "No active target selected."; + } + + renderTargets(payload.targets || [], payload.active_target_id); + renderSavedMasks(payload.mask_names || []); + syncActionState(payload); + if (payload.latest_job_id) { + void refreshReview(payload.latest_job_id).then((reviewPayload) => { + if (reviewPayload && payload.workflow_step === "review") { + syncCanvasMode(payload); + } + }).catch((error) => { + setStatus(status, error.message, true); + }); + ensureReviewPolling(); + } else { + stopReviewPolling(); + state.reviewPayload = null; + renderReviewSummaryList(reviewSummaryList, null, null); + renderReviewSummaryList(reviewSummaryListSide, null, null); + renderReviewTargets(null); + renderReviewArtifacts({}); + renderReviewTimeline([]); + renderReviewWarning({}); + } + } + + async function refreshWorkbench() { + const payload = await parseJson(await fetch(root.dataset.workbenchEndpoint)); + renderWorkbench(payload); + return payload; + } + + function setActiveTool(nextTool) { + state.activeTool = nextTool; + syncToolButtons(state.workbench); + } + + function setCanvasMode(nextCanvasMode) { + state.canvasMode = nextCanvasMode; + if (state.workbench) { + syncCanvasMode(state.workbench); + } + } + + positiveButton?.addEventListener("click", () => setActiveTool("point-positive")); + negativeButton?.addEventListener("click", () => setActiveTool("point-negative")); + + brushButtons.forEach((button) => { + button.addEventListener("click", () => { + setActiveTool(`brush-${button.dataset.brushMode}`); + setStatus(status, `${button.textContent} tool ready. Click the canvas to refine the mask.`, false); + }); + }); + + brushRadiusInput?.addEventListener("input", () => { + state.brushRadius = Number(brushRadiusInput.value); + if (brushRadiusValue) { + brushRadiusValue.value = `${state.brushRadius} px`; + brushRadiusValue.textContent = `${state.brushRadius} px`; + } + }); + + overlayOpacityInput?.addEventListener("input", () => { + state.overlayOpacity = Number(overlayOpacityInput.value); + updateRangeOutput(overlayOpacityValue, state.overlayOpacity); + applyImagePresentation(); + }); + + function seekVideoToFrame(frameIndex, payload = state.workbench) { + if (!keyframeVideo || !payload) { + return; + } + const desiredTime = frameToSeconds(frameIndex, payload); + if (!Number.isFinite(desiredTime)) { + return; + } + try { + keyframeVideo.currentTime = desiredTime; + } catch (_error) { + // Ignore transient seek failures before metadata is ready. + } + } + + function syncSourcePlaybackButton() { + if (!toggleSourcePlaybackButton || !keyframeVideo) { + return; + } + const canPlay = state.canvasMode === "source"; + toggleSourcePlaybackButton.disabled = !canPlay; + toggleSourcePlaybackButton.textContent = keyframeVideo.paused ? "Play" : "Pause"; + } + + function syncPlayheadFromVideo() { + const payload = state.workbench; + if (!payload) { + return; + } + const fps = Number(payload.fps || state.fps || 0); + if (!Number.isFinite(fps) || fps <= 0) { + return; + } + const nextFrame = clampFrame( + Math.round((keyframeVideo?.currentTime || 0) * fps), + 0, + Math.max(0, (payload.frame_count || 1) - 1) + ); + state.playheadFrame = nextFrame; + if (sourcePlayheadSlider) { + sourcePlayheadSlider.value = String(nextFrame); + } + syncKeyframeSummary(payload); + syncActionState(payload); + } + + function clearPendingRangeSelection() { + const payload = state.workbench; + if (!payload) { + return; + } + state.rangeSelectionStart = state.rangeAppliedStart; + state.rangeSelectionEnd = state.rangeAppliedEnd; + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + syncKeyframeSummary(payload); + syncActionState(payload); + } + + async function applyRangeSelection() { + const payload = state.workbench; + if (!payload) { + return; + } + const nextStart = Math.min(state.rangeSelectionStart, state.rangeSelectionEnd); + const nextEnd = Math.max(state.rangeSelectionStart, state.rangeSelectionEnd); + if ( + nextStart === state.rangeAppliedStart + && nextEnd === state.rangeAppliedEnd + ) { + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + syncKeyframeSummary(payload); + syncActionState(payload); + return; + } + + const hasExistingAnnotations = ( + (payload.mask_names || []).length > 0 + || hasTemplateFrame(payload) + || payload.current_mask_url + || payload.current_preview_url + ); + if (hasExistingAnnotations) { + const shouldContinue = window.confirm( + "Changing the processing segment will clear the current anchor, current preview, and every saved mask. Continue?" + ); + if (!shouldContinue) { + clearPendingRangeSelection(); + return; + } + } + + setStatus(status, `Applying segment ${nextStart} - ${nextEnd}...`, false); + try { + const nextPayload = await parseJson( + await fetch(root.dataset.processingRangeEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + start_frame_index: nextStart, + end_frame_index: nextEnd, + }), + }) + ); + state.rangeSelectionTouchedStart = false; + state.rangeSelectionTouchedEnd = false; + state.canvasMode = "source"; + renderWorkbench(nextPayload); + setStatus( + status, + "Range changed. Re-apply an anchor frame to continue annotation.", + false + ); + } catch (error) { + setStatus(status, error.message, true); + } + } + + function markRangeBoundary(boundary) { + const payload = state.workbench; + if (!payload) { + return; + } + if (state.canvasMode !== "source") { + setCanvasMode("source"); + } + const frame = clampFrame( + state.playheadFrame, + 0, + Math.max(0, (payload.frame_count || 1) - 1) + ); + if (boundary === "start") { + state.rangeSelectionStart = frame; + state.rangeSelectionTouchedStart = true; + if (state.rangeSelectionStart > state.rangeSelectionEnd) { + state.rangeSelectionEnd = state.rangeSelectionStart; + } + } else { + state.rangeSelectionEnd = frame; + state.rangeSelectionTouchedEnd = true; + if (state.rangeSelectionEnd < state.rangeSelectionStart) { + state.rangeSelectionStart = state.rangeSelectionEnd; + } + } + + syncKeyframeSummary(payload); + syncActionState(payload); + if (state.rangeSelectionTouchedStart && state.rangeSelectionTouchedEnd) { + void applyRangeSelection(); + return; + } + + setStatus( + status, + boundary === "start" + ? `In point set to frame ${state.rangeSelectionStart}. Mark Out to confirm the segment.` + : `Out point set to frame ${state.rangeSelectionEnd}. Mark In to confirm the segment.`, + false + ); + } + + async function applyTemplateFrameSelection() { + const payload = state.workbench; + if (!payload || !templateFrameSlider || templateFrameSlider.disabled) { + return; + } + const nextFrame = clampFrame( + state.templateFrameSelection, + state.rangeAppliedStart, + state.rangeAppliedEnd + ); + if (hasTemplateFrame(payload) && nextFrame === Number(payload.template_frame_index || 0)) { + return; + } + setStatus(status, `Switching template frame to ${nextFrame}...`, false); + try { + const nextPayload = await parseJson( + await fetch(root.dataset.templateFrameEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ frame_index: nextFrame }), + }) + ); + state.playheadFrame = nextFrame; + renderWorkbench(nextPayload); + setStatus( + status, + `Template frame ${nextPayload.template_frame_index} is now active. Existing unsaved annotations were reset.`, + false + ); + } catch (error) { + setStatus(status, error.message, true); + } + } + + sourcePlayheadSlider?.addEventListener("input", () => { + const payload = state.workbench; + if (!payload) { + return; + } + if (state.canvasMode !== "source") { + setCanvasMode("source"); + } + state.playheadFrame = clampFrame( + Number(sourcePlayheadSlider.value), + 0, + Math.max(0, (payload.frame_count || 1) - 1) + ); + syncKeyframeSummary(payload); + syncActionState(payload); + seekVideoToFrame(state.playheadFrame, payload); + }); + + markRangeInButton?.addEventListener("click", () => { + if (markRangeInButton.disabled) { + return; + } + markRangeBoundary("start"); + }); + + markRangeOutButton?.addEventListener("click", () => { + if (markRangeOutButton.disabled) { + return; + } + markRangeBoundary("end"); + }); + + clearRangeSelectionButton?.addEventListener("click", () => { + if (clearRangeSelectionButton.disabled) { + return; + } + clearPendingRangeSelection(); + setStatus(status, "Pending range marks cleared.", false); + }); + + templateFrameSlider?.addEventListener("input", () => { + const payload = state.workbench; + if (!payload) { + return; + } + state.templateFrameSelection = clampFrame( + Number(templateFrameSlider.value), + state.rangeAppliedStart, + state.rangeAppliedEnd + ); + state.playheadFrame = state.templateFrameSelection; + syncKeyframeSummary(payload); + syncActionState(payload); + seekVideoToFrame(state.playheadFrame, payload); + }); + + templateFrameSlider?.addEventListener("change", () => { + void applyTemplateFrameSelection(); + }); + + toggleSourcePlaybackButton?.addEventListener("click", async () => { + if (!keyframeVideo || toggleSourcePlaybackButton.disabled) { + return; + } + try { + if (keyframeVideo.paused) { + await keyframeVideo.play(); + } else { + keyframeVideo.pause(); + } + } catch (_error) { + setStatus(status, "Video playback is temporarily unavailable.", true); + } + syncSourcePlaybackButton(); + }); + + keyframeVideo?.addEventListener("loadedmetadata", () => { + syncKeyframeSummary(state.workbench); + syncSourcePlaybackButton(); + }); + keyframeVideo?.addEventListener("seeked", syncPlayheadFromVideo); + keyframeVideo?.addEventListener("timeupdate", syncPlayheadFromVideo); + keyframeVideo?.addEventListener("play", syncSourcePlaybackButton); + keyframeVideo?.addEventListener("pause", syncSourcePlaybackButton); + + async function patchActiveTarget(patch, pendingMessage, successMessage) { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + await updateTarget( + currentTarget.target_id, + patch, + pendingMessage, + successMessage + ); + } + + function scheduleLiveTargetPatch(patch, pendingMessage, successMessage) { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + if (state.livePatchTimer) { + clearTimeout(state.livePatchTimer); + } + + state.livePatchTimer = window.setTimeout(async () => { + const revision = ++state.livePatchRevision; + setStatus(status, pendingMessage, false); + try { + const payload = await requestTargetPatch(currentTarget.target_id, patch); + if (revision < state.lastAppliedLivePatchRevision) { + return; + } + state.lastAppliedLivePatchRevision = revision; + renderWorkbench(payload); + setStatus(status, successMessage(payload), false); + } catch (error) { + setStatus(status, error.message, true); + } + }, 140); + } + + presetStrengthInput?.addEventListener("input", () => { + updateRangeOutput(presetStrengthValue, Number(presetStrengthInput.value)); + scheduleLiveTargetPatch( + { preset_strength: Number(presetStrengthInput.value) / 100 }, + "Refreshing live detail preview...", + () => "Preset strength updated." + ); + }); + + motionStrengthInput?.addEventListener("input", () => { + updateRangeOutput(motionStrengthValue, Number(motionStrengthInput.value)); + scheduleLiveTargetPatch( + { motion_strength: Number(motionStrengthInput.value) / 100 }, + "Refreshing live detail preview...", + () => "Motion softness updated." + ); + }); + + temporalStabilityInput?.addEventListener("input", () => { + updateRangeOutput(temporalStabilityValue, Number(temporalStabilityInput.value)); + scheduleLiveTargetPatch( + { temporal_stability: Number(temporalStabilityInput.value) / 100 }, + "Refreshing live detail preview...", + () => "Temporal stability updated." + ); + }); + + edgeFeatherRadiusInput?.addEventListener("input", () => { + updateRangeOutput(edgeFeatherRadiusValue, Number(edgeFeatherRadiusInput.value), " px"); + scheduleLiveTargetPatch( + { edge_feather_radius: Number(edgeFeatherRadiusInput.value) }, + "Refreshing feathered edge preview...", + () => "Edge feather updated." + ); + }); + + viewButtons.forEach((button) => { + button.addEventListener("click", () => { + if (button.disabled) { + return; + } + setCanvasMode(button.dataset.canvasMode); + }); + }); + + async function moveWorkflowStep(nextStep) { + if (!state.workbench || !nextStep || nextStep === state.workbench.workflow_step) { + return; + } + if (nextStep === "review" && !state.workbench.latest_job_id) { + setStatus(status, "Submit a matting job before entering review.", true); + return; + } + setStatus(status, `Switching to ${nextStep}...`, false); + try { + const payload = await parseJson( + await fetch(root.dataset.workflowStepEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ workflow_step: nextStep }), + }) + ); + renderWorkbench(payload); + setStatus(status, `${nextStep} step ready.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + } + + function moveWorkflowBy(offset) { + if (!state.workbench) { + return; + } + const currentIndex = workflowStepIndex(); + const nextIndex = Math.min(Math.max(currentIndex + offset, 0), WORKFLOW_STEPS.length - 1); + const nextStep = WORKFLOW_STEPS[nextIndex]; + if (nextStep === state.workbench.workflow_step) { + return; + } + void moveWorkflowStep(nextStep); + } + + workflowButtons.forEach((button) => { + button.addEventListener("click", () => { + if (button.disabled) { + return; + } + void moveWorkflowStep(button.dataset.workflowStep); + }); + }); + + sidebarTabButtons.forEach((button) => { + button.addEventListener("click", () => { + if (!state.workbench) { + return; + } + state.workbench.active_sidebar_tab = button.dataset.sidebarTab; + syncSidebarPanels(state.workbench); + }); + }); + + workspaceNavBack?.addEventListener("click", () => { + moveWorkflowBy(-1); + }); + + workspaceNavNext?.addEventListener("click", () => { + moveWorkflowBy(1); + }); + + workspaceReturnToClip?.addEventListener("click", () => { + void moveWorkflowStep("clip"); + }); + + workspaceReturnToRefine?.addEventListener("click", () => { + void moveWorkflowStep("refine"); + }); + + presetButtons.forEach((button) => { + button.addEventListener("click", async () => { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + await updateTarget( + currentTarget.target_id, + { refine_preset: button.dataset.preset }, + `Applying ${button.textContent} preset...`, + () => `${button.textContent} preset is now active for ${currentTarget.name}.` + ); + }); + }); + + createTargetButton?.addEventListener("click", async () => { + if (createTargetButton.disabled) { + return; + } + setStatus(status, "Creating a new target layer...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.targetsEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({}), + }) + ); + renderWorkbench(payload); + setStatus(status, `Created ${payload.name}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + applyTargetNameButton?.addEventListener("click", async () => { + const currentTarget = activeTarget(); + const nextName = targetNameInput?.value?.trim(); + if (!currentTarget || !nextName || nextName === currentTarget.name) { + return; + } + await updateTarget( + currentTarget.target_id, + { name: nextName }, + `Renaming ${currentTarget.name}...`, + () => `Renamed target to ${nextName}.` + ); + }); + + toggleTargetLockButton?.addEventListener("click", async () => { + const currentTarget = activeTarget(); + if (!currentTarget) { + return; + } + const nextLocked = !currentTarget.locked; + await updateTarget( + currentTarget.target_id, + { locked: nextLocked }, + `${nextLocked ? "Locking" : "Unlocking"} ${currentTarget.name}...`, + () => `${currentTarget.name} is now ${nextLocked ? "locked" : "editable"}.` + ); + }); + + undoButton?.addEventListener("click", async () => { + if (undoButton.disabled) { + return; + } + setStatus(status, "Removing the last click...", false); + try { + const payload = await parseJson( + await fetch(`${root.dataset.workbenchEndpoint}/undo`, { + method: "POST", + }) + ); + renderWorkbench(payload); + setStatus(status, "Removed the last click from the active target.", false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + resetButton?.addEventListener("click", async () => { + if (resetButton.disabled) { + return; + } + setStatus(status, "Resetting the active target...", false); + try { + const payload = await parseJson( + await fetch(`${root.dataset.workbenchEndpoint}/reset-target`, { + method: "POST", + }) + ); + renderWorkbench(payload); + setStatus(status, "Cleared the active target back to an empty click state.", false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + image?.addEventListener("click", async (event) => { + if (!state.workbench?.can_apply_clicks) { + setStatus( + status, + hasTemplateFrame(state.workbench) + ? "Preview mode is read-only. Switch back to coarse or refine to edit the target." + : "Range changed. Re-apply an anchor frame before editing the target.", + false + ); + return; + } + + const bounds = image.getBoundingClientRect(); + const scaleX = image.naturalWidth / bounds.width; + const scaleY = image.naturalHeight / bounds.height; + const x = Math.round((event.clientX - bounds.left) * scaleX); + const y = Math.round((event.clientY - bounds.top) * scaleY); + + try { + let payload; + if (state.activeTool.startsWith("brush-")) { + const brushMode = state.activeTool.replace("brush-", ""); + setStatus(status, `Applying ${brushMode} brush...`, false); + payload = await parseJson( + await fetch(root.dataset.brushEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + mode: brushMode, + radius: state.brushRadius, + points: [[x, y]], + }), + }) + ); + renderWorkbench(payload); + setStatus(status, `${brushMode} brush updated ${activeTarget(payload)?.name || "the active target"}.`, false); + return; + } + + setStatus(status, "Updating target preview...", false); + payload = await parseJson( + await fetch(root.dataset.clickEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ x, y, positive: state.activeTool === "point-positive" }), + }) + ); + renderWorkbench(payload); + setStatus( + status, + `${state.activeTool === "point-positive" ? "Positive" : "Negative"} point applied to ${activeTarget(payload)?.name || "target"}.`, + false + ); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + saveButton?.addEventListener("click", async () => { + if (saveButton.disabled) { + return; + } + setStatus(status, "Saving current target mask...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.saveEndpoint, { + method: "POST", + }) + ); + if (payload.mask_name) { + state.selectedMasks.add(payload.mask_name); + } + renderWorkbench(payload); + setStatus(status, `Saved ${payload.mask_name}.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + submitButton?.addEventListener("click", async () => { + const selectedMasks = selectedMaskNames(); + if (submitButton.disabled || selectedMasks.length === 0) { + setStatus(status, "Select at least one saved mask before queueing the job.", true); + return; + } + setStatus(status, "Submitting queued job...", false); + try { + const payload = await parseJson( + await fetch(root.dataset.submitEndpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + process_start_frame_index: state.workbench?.process_start_frame_index ?? 0, + process_end_frame_index: state.workbench?.process_end_frame_index ?? 0, + template_frame_index: state.workbench?.template_frame_index ?? 0, + selected_masks: selectedMasks, + }), + }) + ); + const workbenchPayload = await refreshWorkbench(); + renderWorkbench(workbenchPayload); + setCanvasMode("source"); + setStatus(status, `Queued job ${payload.job_id}. Review will stay in this workspace.`, false); + } catch (error) { + setStatus(status, error.message, true); + } + }); + + refreshWorkbench().catch((error) => { + setStatus(status, error.message, true); + }); + + document.addEventListener("keydown", (event) => { + if (event.defaultPrevented) { + return; + } + + const key = event.key; + const lowerKey = key.toLowerCase(); + const hasPrimaryModifier = event.ctrlKey || event.metaKey; + const typingContext = isTypingContext(event.target); + + if (hasPrimaryModifier && lowerKey === "s") { + event.preventDefault(); + saveButton?.click(); + return; + } + + if (hasPrimaryModifier && key === "Enter") { + event.preventDefault(); + submitButton?.click(); + return; + } + + if (event.altKey || hasPrimaryModifier || typingContext) { + return; + } + + switch (key) { + case "1": + case "2": + case "3": + case "4": { + event.preventDefault(); + const nextStep = { + "1": "clip", + "2": "mask", + "3": "refine", + "4": "review", + }[key]; + void moveWorkflowStep(nextStep); + return; + } + case "Backspace": + event.preventDefault(); + resetButton?.click(); + return; + case "ArrowLeft": + if (!typingContext) { + event.preventDefault(); + moveWorkflowBy(-1); + return; + } + break; + case "ArrowRight": + if (!typingContext) { + event.preventDefault(); + moveWorkflowBy(1); + return; + } + break; + default: + break; + } + + switch (lowerKey) { + case "i": + event.preventDefault(); + markRangeInButton?.click(); + return; + case "o": + event.preventDefault(); + markRangeOutButton?.click(); + return; + case "p": + event.preventDefault(); + setActiveTool("point-positive"); + setStatus(status, "Shortcut: Positive point tool.", false); + return; + case "n": + event.preventDefault(); + setActiveTool("point-negative"); + setStatus(status, "Shortcut: Negative point tool.", false); + return; + case "b": + event.preventDefault(); + setActiveTool("brush-add"); + setStatus(status, "Shortcut: Add brush tool.", false); + return; + case "e": + event.preventDefault(); + setActiveTool("brush-remove"); + setStatus(status, "Shortcut: Remove brush tool.", false); + return; + case "g": + event.preventDefault(); + setActiveTool("brush-feather"); + setStatus(status, "Shortcut: Feather brush tool.", false); + return; + case "t": + event.preventDefault(); + createTargetButton?.click(); + return; + case "u": + event.preventDefault(); + undoButton?.click(); + return; + case "r": + event.preventDefault(); + resetButton?.click(); + return; + case "f": + event.preventDefault(); + setCanvasMode("source"); + setStatus(status, "Shortcut: Source view.", false); + return; + case "v": + event.preventDefault(); + setCanvasMode("overlay"); + setStatus(status, "Shortcut: Overlay view.", false); + return; + case "m": + if (viewButtons.find((button) => button.dataset.canvasMode === "mask")?.disabled) { + return; + } + event.preventDefault(); + setCanvasMode("mask"); + setStatus(status, "Shortcut: Mask view.", false); + return; + default: + break; + } + }); +} + +document.addEventListener("DOMContentLoaded", bindWorkbench); diff --git a/matanyone2/webapp/templates/annotate.html b/matanyone2/webapp/templates/annotate.html new file mode 100644 index 0000000..62cd0d1 --- /dev/null +++ b/matanyone2/webapp/templates/annotate.html @@ -0,0 +1,425 @@ +{% extends "base.html" %} +{% block title %}Annotation Workbench - MatAnyone2{% endblock %} +{% block page_name %}annotate{% endblock %} +{% block content %} +
+ + +
+
+
+
+

Source Monitor

+

Clip range and matte preview

+
+ Source: {{ draft.video_path.name }} + {{ draft.width }} x {{ draft.height }} +
+
+
+
+ + + +
+
+ + + +
+
+
+ +
+ Guided silhouette pass +
+

+ Establish the subject with a few positive and negative clicks before saving the current target. +

+ +
+ + Template frame for {{ draft_id }} +
+ +
+
+
+

Clip Range

+

Mark the segment, then choose the anchor inside it

+
+
+ +
+
+ +
+ Playhead ยท 00:00.00 + + Pending range ยท Frame {{ draft.process_start_frame_index }} - {{ draft.process_end_frame_index }} + + + Processing range ยท Frame {{ draft.process_start_frame_index }} - {{ draft.process_end_frame_index }} + +
+ +
+
+
+ +
+ +
+
+

Segment Selection

+

Scrub once, then mark In and Out

+
+
+ + + +
+
+ +
+ + In ยท {{ draft.process_start_frame_index }} + + + Out ยท {{ draft.process_end_frame_index }} + + + Duration ยท {{ "%.2f"|format(draft.duration_seconds) }}s + +
+ +

+ The green band is the processing segment. Changing it clears old masks and asks you to choose a new anchor. +

+
+ +
+
+
+

Inference Anchor

+

Choose the frame that seeds segmentation

+
+ + Anchor ยท {{ draft.template_frame_index if draft.template_frame_index is not none else "Not set" }} + +
+ + + +
+ Selected frame 0 + + Applied frame {{ draft.template_frame_index if draft.template_frame_index is not none else "Not set" }} + + 00:00.00 / 00:00.00 +
+ +

+ Drag the slider inside the selected segment. Releasing it applies the new anchor and refreshes the editable preview. +

+
+
+ +
+
+

Before

+ Previous preview state +
+
+

Live

+ Current preview state +
+
+ +

+
+
+
+{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/matanyone2/webapp/templates/base.html b/matanyone2/webapp/templates/base.html new file mode 100644 index 0000000..f7a3d07 --- /dev/null +++ b/matanyone2/webapp/templates/base.html @@ -0,0 +1,30 @@ + + + + + + {% block title %}MatAnyone2 Internal Web App{% endblock %} + + + +
+
+
+
M2
+
+

Internal Matting Workstation

+

MatAnyone2

+
+
+
+ Single GPU Worker + Alpha / Foreground / PNG / ProRes +
+
+
+ {% block content %}{% endblock %} +
+
+ {% block scripts %}{% endblock %} + + diff --git a/matanyone2/webapp/templates/job.html b/matanyone2/webapp/templates/job.html new file mode 100644 index 0000000..7567307 --- /dev/null +++ b/matanyone2/webapp/templates/job.html @@ -0,0 +1,100 @@ +{% extends "base.html" %} +{% block title %}Result Review - MatAnyone2{% endblock %} +{% block page_name %}results{% endblock %} +{% block content %} +
+
+
+

Result Review

+

Job {{ job.job_id }}

+
+
+ {{ job.status.value }} + +
+
+ +
+
+
+
+

Viewport

+

Output inspection

+
+
+ +
+ + + + +
+ +
+ + + + +
+ Waiting for the first renderable output. +
+
+ +

+ Source preview is always available. Alpha and foreground appear as the job reaches export output. +

+

+
+ +
+ + + + + +
+
+
+{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/matanyone2/webapp/templates/upload.html b/matanyone2/webapp/templates/upload.html new file mode 100644 index 0000000..9c5056b --- /dev/null +++ b/matanyone2/webapp/templates/upload.html @@ -0,0 +1,100 @@ +{% extends "base.html" %} +{% block title %}New Session - MatAnyone2{% endblock %} +{% block page_name %}upload{% endblock %} +{% block content %} +
+
+

New Session

+

Upload a short clip and move into the annotation workbench.

+

+ This internal workstation is tuned for high-quality people matting on TVC-length footage. + Start with the source clip, confirm the media profile, then enter the first-frame selection flow. +

+
+ +
+ + +
+ + + +
+ + +
+ +

+ Select a source clip to prepare the draft. +

+
+ + +
+
+{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/matanyone2/webapp/templates/workspace.html b/matanyone2/webapp/templates/workspace.html new file mode 100644 index 0000000..42f8fdb --- /dev/null +++ b/matanyone2/webapp/templates/workspace.html @@ -0,0 +1,359 @@ +{% extends "base.html" %} +{% block title %}Workspace - MatAnyone2{% endblock %} +{% block page_name %}workspace{% endblock %} +{% block content %} +
+
+
+

Session Workspace

+

Single-pass clip, matte, refine, review

+
+ {{ draft.video_path.name }} + {{ draft.width }} x {{ draft.height }} + {{ "%.2f"|format(draft.duration_seconds) }}s +
+
+ +
+ + +
+ + + + +
+
+
+ +
+ + +
+
+
+
+

Main Monitor

+

Single viewer for clip, mask, refine, and review

+
+
+
+ + + + + +
+
+
+ +
+ + + + + +
+ +
+ Source monitor +

+
+
+ +
+
+
+
+ +
+
+ +
+
+ + + + +
+
+ Playhead + In + Out + Duration +
+
+ +
+ +
+
+
+ + +
+
+{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/matanyone2/webapp/worker.py b/matanyone2/webapp/worker.py new file mode 100644 index 0000000..6e74ecb --- /dev/null +++ b/matanyone2/webapp/worker.py @@ -0,0 +1,107 @@ +from pathlib import Path +import time +import json + +from matanyone2.webapp.models import JobStatus +from matanyone2.webapp.queue import QueueCoordinator + + +class WorkerLoop: + def __init__( + self, + coordinator: QueueCoordinator, + *, + repository, + inference_service, + export_service, + runtime_root: Path, + ): + self.coordinator = coordinator + self.repository = repository + self.inference_service = inference_service + self.export_service = export_service + self.runtime_root = Path(runtime_root) + + def recover(self) -> None: + self.coordinator.recover_interrupted_jobs() + + def run_forever(self, poll_interval_seconds: float = 1.0) -> None: + while True: + processed_job_id = self.process_next_job() + if processed_job_id is None: + time.sleep(poll_interval_seconds) + + def process_next_job(self) -> str | None: + job_id = self.coordinator.next_job_id() + if job_id is None: + return None + + job = self.repository.get_job(job_id) + job_dir = self.runtime_root / "jobs" / job.job_id + job_dir.mkdir(parents=True, exist_ok=True) + + try: + self.repository.update_status(job.job_id, JobStatus.PREPARING) + self.repository.update_status(job.job_id, JobStatus.RUNNING) + job_params = json.loads(job.params_json or "{}") + selected_mask_controls = job_params.get("selected_mask_controls", {}) + selected_mask_presets = job_params.get("selected_mask_presets", {}) + motion_strength = max( + ( + float(control.get("motion_strength", 0.0)) + for control in selected_mask_controls.values() + ), + default=0.0, + ) + temporal_stability = max( + ( + float(control.get("temporal_stability", 0.0)) + for control in selected_mask_controls.values() + ), + default=0.0, + ) + edge_feather_radius = max( + ( + float(control.get("edge_feather_radius", 0.0)) + for control in selected_mask_controls.values() + ), + default=0.0, + ) + inference_result = self.inference_service.run_job( + source_video_path=Path(job.source_video_path), + mask_path=Path(job.mask_path), + job_dir=job_dir, + template_frame_index=job.template_frame_index, + process_start_frame_index=int(job_params.get("process_start_frame_index", 0)), + process_end_frame_index=job_params.get("process_end_frame_index"), + selected_mask_controls=selected_mask_controls, + selected_mask_presets=selected_mask_presets, + ) + self.repository.update_status(job.job_id, JobStatus.EXPORTING) + export_result = self.export_service.export_assets( + inference_result.foreground_video_path, + inference_result.alpha_video_path, + job_dir, + motion_strength=motion_strength, + temporal_stability=temporal_stability, + edge_feather_radius=edge_feather_radius, + ) + except Exception as exc: + self.repository.update_status( + job.job_id, + JobStatus.FAILED, + error_text=str(exc), + ) + return job.job_id + + final_status = ( + JobStatus.COMPLETED_WITH_WARNING + if export_result.warning_text + else JobStatus.COMPLETED + ) + self.repository.update_status( + job.job_id, + final_status, + warning_text=export_result.warning_text, + ) + return job.job_id diff --git a/pyproject.toml b/pyproject.toml index 226223e..d203047 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,26 +25,30 @@ dependencies = [ 'cython', 'gitpython >= 3.1', 'thinplate@git+https://github.com/cheind/py-thin-plate-spline', + 'SAM-2 @ git+https://github.com/facebookresearch/sam2.git', + 'segment-anything @ git+https://github.com/facebookresearch/segment-anything.git', 'hickle >= 5.0', 'tensorboard >= 2.11', 'numpy >= 1.21', + 'matplotlib >= 3.8', 'Pillow >= 9.5', 'opencv-python >= 4.8', 'scipy >= 1.7', 'pycocotools >= 2.0.7', 'tqdm >= 4.66.1', - 'gradio >= 3.34', + 'fastapi >= 0.111, < 1.0', + 'uvicorn >= 0.30, < 1.0', + 'jinja2 >= 3.1, < 4.0', + 'python-multipart >= 0.0.9, < 1.0', + 'httpx >= 0.27, < 1.0', + 'decord >= 0.6', + 'pytest >= 8.0, < 9.0', 'gdown >= 4.7.1', 'einops >= 0.6', 'hydra-core >= 1.3.2', - 'PySide6 >= 6.2.0', - 'charset-normalizer >= 3.1.0', - 'netifaces >= 0.11.0', - 'cchardet >= 2.1.7', 'easydict', 'av >= 0.5.2', 'requests', - 'pyqtdarktheme', 'imageio == 2.25.0', 'imageio[ffmpeg]', 'huggingface_hub == 0.36.2', diff --git a/scripts/_path_bootstrap.py b/scripts/_path_bootstrap.py new file mode 100644 index 0000000..20b59e3 --- /dev/null +++ b/scripts/_path_bootstrap.py @@ -0,0 +1,10 @@ +from pathlib import Path +import sys + + +def ensure_project_root_on_path(script_path: str | Path) -> Path: + project_root = Path(script_path).resolve().parents[1] + project_root_str = str(project_root) + if project_root_str not in sys.path: + sys.path.insert(0, project_root_str) + return project_root diff --git a/scripts/check_internal_webapp.ps1 b/scripts/check_internal_webapp.ps1 new file mode 100644 index 0000000..d4a5369 --- /dev/null +++ b/scripts/check_internal_webapp.ps1 @@ -0,0 +1,61 @@ +param( + [string]$ServiceRoot = "", + [string]$RuntimeRoot = "", + [int]$Port = 8010 +) + +. (Join-Path $PSScriptRoot "internal_webapp_common.ps1") + +$config = Get-InternalWebAppServiceConfig ` + -ScriptRoot $PSScriptRoot ` + -ServiceRoot $ServiceRoot ` + -RuntimeRoot $RuntimeRoot ` + -Port $Port + +$state = Read-InternalWebAppState -StateFile $config.state_file +if ($null -eq $state) { + Write-InternalWebAppJson @{ + status = "not_running" + service_root = $config.service_root + state_file = $config.state_file + } + exit 1 +} + +$webappAlive = Test-InternalWebAppProcess -ProcessId ([int]$state.webapp_pid) +$workerAlive = Test-InternalWebAppProcess -ProcessId ([int]$state.worker_pid) +$httpOk = $false +$httpStatus = $null +try { + $response = Invoke-WebRequest -UseBasicParsing -Uri $state.base_url -TimeoutSec 10 + $httpStatus = [int]$response.StatusCode + $httpOk = ($httpStatus -eq 200) +} catch { + $httpStatus = $null +} + +if ($webappAlive -and $workerAlive -and $httpOk) { + Write-InternalWebAppJson @{ + status = "running" + base_url = $state.base_url + service_root = $state.service_root + state_file = $config.state_file + webapp_pid = [int]$state.webapp_pid + worker_pid = [int]$state.worker_pid + http_status = $httpStatus + } + exit 0 +} + +Write-InternalWebAppJson @{ + status = "degraded" + base_url = $state.base_url + service_root = $state.service_root + state_file = $config.state_file + webapp_pid = [int]$state.webapp_pid + worker_pid = [int]$state.worker_pid + webapp_alive = $webappAlive + worker_alive = $workerAlive + http_status = $httpStatus +} +exit 1 diff --git a/scripts/internal_webapp_common.ps1 b/scripts/internal_webapp_common.ps1 new file mode 100644 index 0000000..d9325f6 --- /dev/null +++ b/scripts/internal_webapp_common.ps1 @@ -0,0 +1,191 @@ +function Get-InternalWebAppRepoRoot { + param( + [string]$ScriptRoot + ) + + return [System.IO.Path]::GetFullPath((Join-Path $ScriptRoot "..")) +} + +function Get-InternalWebAppPythonPath { + param( + [string]$RepoRoot + ) + + return [System.IO.Path]::GetFullPath((Join-Path $RepoRoot ".venv\Scripts\python.exe")) +} + +function Get-InternalWebAppServiceConfig { + param( + [string]$ScriptRoot, + [string]$ServiceRoot, + [string]$RuntimeRoot, + [int]$Port + ) + + $repoRoot = Get-InternalWebAppRepoRoot -ScriptRoot $ScriptRoot + $resolvedServiceRoot = if ($ServiceRoot) { + [System.IO.Path]::GetFullPath($ServiceRoot) + } else { + [System.IO.Path]::GetFullPath((Join-Path $repoRoot "runtime\webapp-service")) + } + $resolvedRuntimeRoot = if ($RuntimeRoot) { + [System.IO.Path]::GetFullPath($RuntimeRoot) + } else { + [System.IO.Path]::GetFullPath((Join-Path $resolvedServiceRoot "runtime")) + } + $logsDir = [System.IO.Path]::GetFullPath((Join-Path $resolvedServiceRoot "logs")) + + return @{ + repo_root = $repoRoot + service_root = $resolvedServiceRoot + runtime_root = $resolvedRuntimeRoot + logs_dir = $logsDir + state_file = [System.IO.Path]::GetFullPath((Join-Path $resolvedServiceRoot "service.json")) + database_path = [System.IO.Path]::GetFullPath((Join-Path $resolvedRuntimeRoot "jobs.db")) + base_url = "http://127.0.0.1:$Port" + port = $Port + python_path = Get-InternalWebAppPythonPath -RepoRoot $repoRoot + webapp_stdout = [System.IO.Path]::GetFullPath((Join-Path $logsDir "webapp.out.log")) + webapp_stderr = [System.IO.Path]::GetFullPath((Join-Path $logsDir "webapp.err.log")) + worker_stdout = [System.IO.Path]::GetFullPath((Join-Path $logsDir "worker.out.log")) + worker_stderr = [System.IO.Path]::GetFullPath((Join-Path $logsDir "worker.err.log")) + } +} + +function ConvertTo-CompactJson { + param( + [Parameter(ValueFromPipeline = $true)] + [object]$InputObject + ) + + process { + return $InputObject | ConvertTo-Json -Depth 8 -Compress + } +} + +function Write-InternalWebAppJson { + param( + [hashtable]$Payload + ) + + $Payload | ConvertTo-CompactJson | Write-Output +} + +function Ensure-InternalWebAppDirectories { + param( + [hashtable]$Config + ) + + foreach ($path in @($Config.service_root, $Config.runtime_root, $Config.logs_dir)) { + New-Item -ItemType Directory -Path $path -Force | Out-Null + } +} + +function Read-InternalWebAppState { + param( + [string]$StateFile + ) + + if (-not (Test-Path -LiteralPath $StateFile)) { + return $null + } + return Get-Content -LiteralPath $StateFile -Raw | ConvertFrom-Json +} + +function Write-InternalWebAppState { + param( + [string]$StateFile, + [hashtable]$Payload + ) + + $json = $Payload | ConvertTo-Json -Depth 8 + Set-Content -LiteralPath $StateFile -Value $json -Encoding UTF8 +} + +function Test-InternalWebAppProcess { + param( + [int]$ProcessId + ) + + if ($ProcessId -le 0) { + return $false + } + return $null -ne (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue) +} + +function Get-InternalWebAppManagedProcesses { + param( + [string]$RepoRoot, + [string]$RuntimeRoot = "", + [string]$ServiceRoot = "" + ) + + return Get-CimInstance Win32_Process -ErrorAction SilentlyContinue | + Where-Object { + $commandLine = $_.CommandLine + $matchesRepo = $commandLine -and $commandLine -like "*$RepoRoot*" + $matchesService = (-not $ServiceRoot) -or ($commandLine -like "*$ServiceRoot*") + $matchesRuntime = (-not $RuntimeRoot) -or ($commandLine -like "*$RuntimeRoot*") + $matchesRepo -and $matchesService -and $matchesRuntime -and ( + $commandLine -like "*$RepoRoot*" -and ( + $commandLine -like "*scripts.run_internal_webapp:app*" -or + $commandLine -like "*scripts/run_internal_worker.py*" -or + $commandLine -like "*scripts\\run_internal_worker.py*" + ) + ) + } +} + +function Stop-InternalWebAppProcessTree { + param( + [int]$ProcessId + ) + + if (-not (Test-InternalWebAppProcess -ProcessId $ProcessId)) { + return $false + } + + & taskkill /PID $ProcessId /T /F | Out-Null + for ($attempt = 0; $attempt -lt 20; $attempt++) { + if (-not (Test-InternalWebAppProcess -ProcessId $ProcessId)) { + return $true + } + Start-Sleep -Milliseconds 200 + } + return -not (Test-InternalWebAppProcess -ProcessId $ProcessId) +} + +function ConvertTo-PowerShellLiteral { + param( + [string]$Value + ) + + return "'" + $Value.Replace("'", "''") + "'" +} + +function New-InternalWebAppCommand { + param( + [string]$PythonPath, + [string[]]$PythonArguments, + [string]$RepoRoot, + [hashtable]$Environment, + [string]$StdoutPath, + [string]$StderrPath + ) + + $segments = @() + foreach ($entry in $Environment.GetEnumerator()) { + $segments += '$env:' + $entry.Key + ' = ' + (ConvertTo-PowerShellLiteral -Value $entry.Value) + } + $segments += 'Set-Location ' + (ConvertTo-PowerShellLiteral -Value $RepoRoot) + + $joinedArguments = ($PythonArguments | ForEach-Object { + ConvertTo-PowerShellLiteral -Value $_ + }) -join ' ' + $command = '& ' + (ConvertTo-PowerShellLiteral -Value $PythonPath) + ' ' + $joinedArguments + $command += ' 1>> ' + (ConvertTo-PowerShellLiteral -Value $StdoutPath) + $command += ' 2>> ' + (ConvertTo-PowerShellLiteral -Value $StderrPath) + $segments += $command + + return '& { ' + ($segments -join '; ') + ' }' +} diff --git a/scripts/run_internal_webapp.py b/scripts/run_internal_webapp.py new file mode 100644 index 0000000..e06a61b --- /dev/null +++ b/scripts/run_internal_webapp.py @@ -0,0 +1,12 @@ +try: + from scripts._path_bootstrap import ensure_project_root_on_path +except ModuleNotFoundError: # pragma: no cover - direct script execution path + from _path_bootstrap import ensure_project_root_on_path + +ensure_project_root_on_path(__file__) + +from matanyone2.webapp.api.app import create_app +from matanyone2.webapp.config import WebAppSettings + + +app = create_app(settings=WebAppSettings()) diff --git a/scripts/run_internal_worker.py b/scripts/run_internal_worker.py new file mode 100644 index 0000000..88d0494 --- /dev/null +++ b/scripts/run_internal_worker.py @@ -0,0 +1,31 @@ +try: + from scripts._path_bootstrap import ensure_project_root_on_path +except ModuleNotFoundError: # pragma: no cover - direct script execution path + from _path_bootstrap import ensure_project_root_on_path + +ensure_project_root_on_path(__file__) + +from matanyone2.webapp.config import WebAppSettings +from matanyone2.webapp.queue import QueueCoordinator +from matanyone2.webapp.repository import JobRepository +from matanyone2.webapp.services.export import ExportService +from matanyone2.webapp.services.inference import InferenceService +from matanyone2.webapp.worker import WorkerLoop + + +def main() -> None: + settings = WebAppSettings() + repository = JobRepository.from_path(settings.database_path) + worker = WorkerLoop( + QueueCoordinator(repository), + repository=repository, + inference_service=InferenceService(), + export_service=ExportService(enable_prores=settings.enable_prores_export), + runtime_root=settings.runtime_root, + ) + worker.recover() + worker.run_forever() + + +if __name__ == "__main__": + main() diff --git a/scripts/smoke_internal_webapp.ps1 b/scripts/smoke_internal_webapp.ps1 new file mode 100644 index 0000000..1ddbe9c --- /dev/null +++ b/scripts/smoke_internal_webapp.ps1 @@ -0,0 +1,14 @@ +param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$Args +) + +$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) +$pythonPath = [System.IO.Path]::GetFullPath((Join-Path $repoRoot ".venv\Scripts\python.exe")) + +if (-not (Test-Path -LiteralPath $pythonPath)) { + throw "Missing python executable: $pythonPath" +} + +& $pythonPath (Join-Path $repoRoot "scripts\smoke_internal_webapp.py") @Args +exit $LASTEXITCODE diff --git a/scripts/smoke_internal_webapp.py b/scripts/smoke_internal_webapp.py new file mode 100644 index 0000000..41525d2 --- /dev/null +++ b/scripts/smoke_internal_webapp.py @@ -0,0 +1,12 @@ +try: + from scripts._path_bootstrap import ensure_project_root_on_path +except ModuleNotFoundError: # pragma: no cover - direct script execution path + from _path_bootstrap import ensure_project_root_on_path + +ensure_project_root_on_path(__file__) + +from matanyone2.webapp.smoke import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start_internal_webapp.ps1 b/scripts/start_internal_webapp.ps1 new file mode 100644 index 0000000..8edb2c3 --- /dev/null +++ b/scripts/start_internal_webapp.ps1 @@ -0,0 +1,140 @@ +param( + [string]$ServiceRoot = "", + [string]$RuntimeRoot = "", + [int]$Port = 8010, + [switch]$DryRun +) + +. (Join-Path $PSScriptRoot "internal_webapp_common.ps1") + +$config = Get-InternalWebAppServiceConfig ` + -ScriptRoot $PSScriptRoot ` + -ServiceRoot $ServiceRoot ` + -RuntimeRoot $RuntimeRoot ` + -Port $Port + +$webappArgs = @( + "-m", + "uvicorn", + "scripts.run_internal_webapp:app", + "--host", + "127.0.0.1", + "--port", + "$Port" +) +$workerArgs = @("scripts/run_internal_worker.py") + +if ($DryRun) { + Write-InternalWebAppJson @{ + status = "dry_run" + service_root = $config.service_root + runtime_root = $config.runtime_root + state_file = $config.state_file + python_path = $config.python_path + base_url = $config.base_url + webapp_args = $webappArgs + worker_args = $workerArgs + } + exit 0 +} + +if (-not (Test-Path -LiteralPath $config.python_path)) { + throw "Missing python executable: $($config.python_path)" +} + +Ensure-InternalWebAppDirectories -Config $config +$existingState = Read-InternalWebAppState -StateFile $config.state_file +if ($null -ne $existingState) { + $webappAlive = Test-InternalWebAppProcess -ProcessId ([int]$existingState.webapp_pid) + $workerAlive = Test-InternalWebAppProcess -ProcessId ([int]$existingState.worker_pid) + if ($webappAlive -and $workerAlive) { + Write-InternalWebAppJson @{ + status = "already_running" + base_url = $existingState.base_url + service_root = $config.service_root + state_file = $config.state_file + webapp_pid = [int]$existingState.webapp_pid + worker_pid = [int]$existingState.worker_pid + } + exit 0 + } +} + +$environment = @{ + MATANYONE2_WEBAPP_RUNTIME_ROOT = $config.runtime_root + MATANYONE2_WEBAPP_DATABASE_PATH = $config.database_path + MATANYONE2_WEBAPP_ENABLE_PRORES = "1" + MATANYONE2_WEBAPP_SAM_BACKEND = "sam3" + PYTHONIOENCODING = "utf-8" +} +if ($env:MATANYONE2_WEBAPP_SAM_MODEL_TYPE) { + $environment["MATANYONE2_WEBAPP_SAM_MODEL_TYPE"] = $env:MATANYONE2_WEBAPP_SAM_MODEL_TYPE +} +if ($env:MATANYONE2_WEBAPP_SAM_BACKEND) { + $environment["MATANYONE2_WEBAPP_SAM_BACKEND"] = $env:MATANYONE2_WEBAPP_SAM_BACKEND +} +if ($env:MATANYONE2_WEBAPP_SAM2_VARIANT) { + $environment["MATANYONE2_WEBAPP_SAM2_VARIANT"] = $env:MATANYONE2_WEBAPP_SAM2_VARIANT +} +if ($env:MATANYONE2_WEBAPP_SAM2_CHECKPOINT_PATH) { + $environment["MATANYONE2_WEBAPP_SAM2_CHECKPOINT_PATH"] = $env:MATANYONE2_WEBAPP_SAM2_CHECKPOINT_PATH +} +if ($env:MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH) { + $environment["MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH"] = $env:MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH +} elseif (Test-Path -LiteralPath "D:\my_app\lens_hunter2\models\sam3\checkpoints\sam3.pt") { + $environment["MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH"] = "D:\my_app\lens_hunter2\models\sam3\checkpoints\sam3.pt" +} + +$webappCommand = New-InternalWebAppCommand ` + -PythonPath $config.python_path ` + -PythonArguments $webappArgs ` + -RepoRoot $config.repo_root ` + -Environment $environment ` + -StdoutPath $config.webapp_stdout ` + -StderrPath $config.webapp_stderr +$workerCommand = New-InternalWebAppCommand ` + -PythonPath $config.python_path ` + -PythonArguments $workerArgs ` + -RepoRoot $config.repo_root ` + -Environment $environment ` + -StdoutPath $config.worker_stdout ` + -StderrPath $config.worker_stderr + +$webappProcess = Start-Process ` + -FilePath "powershell.exe" ` + -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", $webappCommand) ` + -WorkingDirectory $config.repo_root ` + -PassThru ` + -WindowStyle Hidden +$workerProcess = Start-Process ` + -FilePath "powershell.exe" ` + -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", $workerCommand) ` + -WorkingDirectory $config.repo_root ` + -PassThru ` + -WindowStyle Hidden + +$state = @{ + repo_root = $config.repo_root + service_root = $config.service_root + runtime_root = $config.runtime_root + logs_dir = $config.logs_dir + state_file = $config.state_file + base_url = $config.base_url + port = $config.port + python_path = $config.python_path + webapp_pid = $webappProcess.Id + worker_pid = $workerProcess.Id + webapp_stdout = $config.webapp_stdout + webapp_stderr = $config.webapp_stderr + worker_stdout = $config.worker_stdout + worker_stderr = $config.worker_stderr +} +Write-InternalWebAppState -StateFile $config.state_file -Payload $state +Write-InternalWebAppJson @{ + status = "started" + base_url = $config.base_url + service_root = $config.service_root + state_file = $config.state_file + webapp_pid = $webappProcess.Id + worker_pid = $workerProcess.Id +} diff --git a/scripts/stop_internal_webapp.ps1 b/scripts/stop_internal_webapp.ps1 new file mode 100644 index 0000000..4301fae --- /dev/null +++ b/scripts/stop_internal_webapp.ps1 @@ -0,0 +1,75 @@ +param( + [string]$ServiceRoot = "", + [string]$RuntimeRoot = "", + [int]$Port = 8010, + [switch]$DryRun +) + +. (Join-Path $PSScriptRoot "internal_webapp_common.ps1") + +$config = Get-InternalWebAppServiceConfig ` + -ScriptRoot $PSScriptRoot ` + -ServiceRoot $ServiceRoot ` + -RuntimeRoot $RuntimeRoot ` + -Port $Port + +$state = Read-InternalWebAppState -StateFile $config.state_file +$stoppedPids = @() + +if ($null -eq $state) { + $fallbackPids = @( + Get-InternalWebAppManagedProcesses ` + -RepoRoot $config.repo_root ` + -RuntimeRoot $config.runtime_root ` + -ServiceRoot $config.service_root | + Select-Object -ExpandProperty ProcessId -Unique + ) + if ($DryRun) { + Write-InternalWebAppJson @{ + status = if ($fallbackPids.Count -gt 0) { "dry_run" } else { "not_running" } + service_root = $config.service_root + state_file = $config.state_file + candidate_pids = $fallbackPids + } + exit 0 + } + + foreach ($processId in $fallbackPids) { + if (Stop-InternalWebAppProcessTree -ProcessId $processId) { + $stoppedPids += $processId + } + } + + Write-InternalWebAppJson @{ + status = if ($stoppedPids.Count -gt 0) { "stopped" } else { "not_running" } + service_root = $config.service_root + state_file = $config.state_file + stopped_pids = $stoppedPids + } + exit 0 +} + +$candidatePids = @([int]$state.webapp_pid, [int]$state.worker_pid) | Where-Object { $_ -gt 0 } | Select-Object -Unique +if ($DryRun) { + Write-InternalWebAppJson @{ + status = if ($candidatePids.Count -gt 0) { "dry_run" } else { "not_running" } + service_root = $config.service_root + state_file = $config.state_file + candidate_pids = $candidatePids + } + exit 0 +} + +foreach ($processId in $candidatePids) { + if (Stop-InternalWebAppProcessTree -ProcessId $processId) { + $stoppedPids += $processId + } +} + +Remove-Item -LiteralPath $config.state_file -Force -ErrorAction SilentlyContinue +Write-InternalWebAppJson @{ + status = "stopped" + service_root = $config.service_root + state_file = $config.state_file + stopped_pids = $stoppedPids +} diff --git a/tests/test_inference_utils.py b/tests/test_inference_utils.py new file mode 100644 index 0000000..7f3f8be --- /dev/null +++ b/tests/test_inference_utils.py @@ -0,0 +1,36 @@ +from pathlib import Path +import shutil + +import cv2 +import numpy as np +import pytest + +from matanyone2.utils.inference_utils import read_frame_from_videos + + +@pytest.fixture +def sample_video_path(tmp_path) -> Path: + video_path = tmp_path / "sample.mp4" + writer = cv2.VideoWriter( + str(video_path), + cv2.VideoWriter_fourcc(*"mp4v"), + 3.0, + (16, 12), + ) + for idx in range(3): + frame = np.full((12, 16, 3), fill_value=idx * 40, dtype=np.uint8) + writer.write(frame) + writer.release() + return video_path + + +def test_read_frame_from_videos_accepts_m4v_extension(tmp_path, sample_video_path): + m4v_path = tmp_path / "sample.m4v" + shutil.copy2(sample_video_path, m4v_path) + + frames, fps, length, video_name = read_frame_from_videos(str(m4v_path)) + + assert length == 3 + assert tuple(frames.shape) == (3, 3, 12, 16) + assert fps == 3.0 + assert video_name == "sample" diff --git a/tests/webapp/conftest.py b/tests/webapp/conftest.py new file mode 100644 index 0000000..9c320b9 --- /dev/null +++ b/tests/webapp/conftest.py @@ -0,0 +1,80 @@ +from pathlib import Path + +import cv2 +import numpy as np +import pytest +from fastapi.testclient import TestClient +from PIL import Image + +from matanyone2.webapp.api.app import create_app +from matanyone2.webapp.config import WebAppSettings +from matanyone2.webapp.models import JobStatus +from matanyone2.webapp.services.masking import MaskingService + + +@pytest.fixture +def sample_video_path(tmp_path) -> Path: + video_path = tmp_path / "sample.mp4" + writer = cv2.VideoWriter( + str(video_path), + cv2.VideoWriter_fourcc(*"mp4v"), + 3.0, + (16, 12), + ) + for idx in range(3): + frame = np.full((12, 16, 3), fill_value=idx * 40, dtype=np.uint8) + writer.write(frame) + writer.release() + return video_path + + +@pytest.fixture +def sample_video_upload(sample_video_path): + return ("sample.mp4", sample_video_path.read_bytes(), "video/mp4") + + +@pytest.fixture +def app_client(tmp_path) -> TestClient: + runtime_root = tmp_path / "runtime" + settings = WebAppSettings( + runtime_root=runtime_root, + database_path=runtime_root / "jobs.db", + ) + app = create_app(settings=settings) + + class FakeController: + def first_frame_click(self, image, points, labels, multimask=True): + mask = np.zeros((image.shape[0], image.shape[1]), dtype=np.uint8) + for x, y in points.tolist(): + y0 = max(0, y - 1) + y1 = min(image.shape[0], y + 2) + x0 = max(0, x - 1) + x1 = min(image.shape[1], x + 2) + mask[y0:y1, x0:x1] = 1 + return mask, np.zeros_like(mask, dtype=np.float32), Image.fromarray(image) + + app.state.masking_service = MaskingService( + runtime_root=runtime_root, + controller_factory=lambda: FakeController(), + ) + with TestClient(app) as client: + yield client + + +@pytest.fixture +def seeded_jobs(app_client): + repository = app_client.app.state.repository + first = repository.create_job( + source_video_path="first.mp4", + template_frame_index=0, + mask_path="first.png", + params_json="{}", + ) + second = repository.create_job( + source_video_path="second.mp4", + template_frame_index=0, + mask_path="second.png", + params_json="{}", + ) + repository.update_status(first.job_id, JobStatus.RUNNING) + return first.job_id, second.job_id diff --git a/tests/webapp/test_api_flow.py b/tests/webapp/test_api_flow.py new file mode 100644 index 0000000..f27cfa8 --- /dev/null +++ b/tests/webapp/test_api_flow.py @@ -0,0 +1,918 @@ +import json + +from fastapi.testclient import TestClient +from matanyone2.webapp.models import JobStatus + + +def test_upload_page_exposes_browser_entrypoint(app_client: TestClient): + response = app_client.get("/") + + assert response.status_code == 200 + assert 'id="upload-form"' in response.text + assert 'data-upload-endpoint="/api/uploads"' in response.text + assert "/static/upload.js" in response.text + + +def test_submit_flow_keeps_workspace_in_review_state( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + assert upload_response.status_code == 200 + draft_id = upload_response.json()["draft_id"] + template_frame_url = upload_response.json()["template_frame_url"] + + template_response = app_client.get(template_frame_url) + workspace_page = app_client.get(f"/drafts/{draft_id}/workspace") + + assert template_response.status_code == 200 + assert template_response.headers["content-type"] == "image/png" + assert workspace_page.status_code == 200 + assert draft_id in workspace_page.text + + click_response = app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + assert click_response.status_code == 200 + current_preview_url = click_response.json()["current_preview_url"] + preview_response = app_client.get(current_preview_url) + + assert preview_response.status_code == 200 + assert preview_response.headers["content-type"] == "image/png" + assert 'id="workspace-app"' in workspace_page.text + assert f'data-click-endpoint="/api/drafts/{draft_id}/click"' in workspace_page.text + assert "/static/workspace.js" in workspace_page.text + + save_response = app_client.post(f"/api/drafts/{draft_id}/masks") + assert save_response.status_code == 200 + assert save_response.json()["mask_name"] == "mask_001" + + annotate_response = app_client.post( + f"/api/drafts/{draft_id}/submit", + json={ + "process_start_frame_index": 0, + "process_end_frame_index": 2, + "template_frame_index": 0, + "selected_masks": ["mask_001"], + }, + ) + + assert annotate_response.status_code == 200 + assert annotate_response.json()["status"] == "queued" + assert annotate_response.json()["workflow_step"] == "review" + state_response = app_client.get(f"/api/drafts/{draft_id}") + assert state_response.status_code == 200 + assert state_response.json()["workflow_step"] == "review" + assert state_response.json()["latest_job_id"] == annotate_response.json()["job_id"] + + +def test_submit_persists_selected_mask_presets_in_job_params( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.patch( + f"/api/drafts/{draft_id}/targets/target-001", + json={"refine_preset": "hair"}, + ) + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + app_client.post(f"/api/drafts/{draft_id}/masks") + submit_response = app_client.post( + f"/api/drafts/{draft_id}/submit", + json={ + "process_start_frame_index": 0, + "process_end_frame_index": 2, + "template_frame_index": 0, + "selected_masks": ["mask_001"], + }, + ) + + repository = app_client.app.state.repository + job = repository.get_job(submit_response.json()["job_id"]) + params = json.loads(job.params_json) + + assert submit_response.status_code == 200 + assert params["selected_mask_presets"] == {"mask_001": "hair"} + assert params["selected_mask_controls"]["mask_001"]["edge_feather_radius"] == 0.0 + + +def test_annotation_page_exposes_workbench_contract( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + response = app_client.get(f"/drafts/{draft_id}/workspace") + + assert response.status_code == 200 + assert f'data-workbench-endpoint="/api/drafts/{draft_id}"' in response.text + assert f'data-workflow-step-endpoint="/api/drafts/{draft_id}/workflow-step"' in response.text + assert f'data-targets-endpoint="/api/drafts/{draft_id}/targets"' in response.text + assert f'data-brush-endpoint="/api/drafts/{draft_id}/brush"' in response.text + assert 'id="workflow-stepper"' in response.text + assert 'id="workspace-sidebar-tabs"' in response.text + assert 'id="workspace-timeline-dock"' in response.text + + +def test_target_creation_and_selection_round_trip( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + state_response = app_client.get(f"/api/drafts/{draft_id}") + create_response = app_client.post( + f"/api/drafts/{draft_id}/targets", + json={"name": "Hero"}, + ) + created = create_response.json() + select_response = app_client.post( + f"/api/drafts/{draft_id}/targets/{created['target_id']}/select" + ) + selected = select_response.json() + + assert state_response.status_code == 200 + assert state_response.json()["stage"] == "coarse" + assert state_response.json()["active_target_id"] is not None + assert create_response.status_code == 200 + assert created["name"] == "Hero" + assert any(target["name"] == "Hero" for target in created["targets"]) + assert select_response.status_code == 200 + assert selected["active_target_id"] == created["target_id"] + assert any( + target["target_id"] == created["target_id"] and target["selected"] + for target in selected["targets"] + ) + + +def test_target_update_round_trip( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + create_response = app_client.post( + f"/api/drafts/{draft_id}/targets", + json={"name": "Hero"}, + ) + target_id = create_response.json()["target_id"] + + update_response = app_client.patch( + f"/api/drafts/{draft_id}/targets/{target_id}", + json={ + "name": "Lead Actor", + "visible": False, + "locked": True, + "refine_preset": "hair", + }, + ) + + assert update_response.status_code == 200 + payload = update_response.json() + assert payload["active_target_id"] == target_id + assert payload["can_apply_clicks"] is False + assert any( + target["target_id"] == target_id + and target["name"] == "Lead Actor" + and target["visible"] is False + and target["locked"] is True + and target["refine_preset"] == "hair" + for target in payload["targets"] + ) + + +def test_target_update_round_trip_includes_numeric_refine_controls( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + update_response = app_client.patch( + f"/api/drafts/{draft_id}/targets/target-001", + json={ + "refine_preset": "hair", + "preset_strength": 0.85, + "motion_strength": 0.55, + "temporal_stability": 0.7, + "edge_feather_radius": 9.0, + }, + ) + + assert update_response.status_code == 200 + payload = update_response.json() + active_target = next( + target for target in payload["targets"] if target["target_id"] == payload["active_target_id"] + ) + assert active_target["preset_strength"] == 0.85 + assert active_target["motion_strength"] == 0.55 + assert active_target["temporal_stability"] == 0.7 + assert active_target["edge_feather_radius"] == 9.0 + + +def test_template_frame_selection_round_trip( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + initial_state = app_client.get(f"/api/drafts/{draft_id}").json() + update_response = app_client.post( + f"/api/drafts/{draft_id}/template-frame", + json={"frame_index": 2}, + ) + + assert initial_state["template_frame_index"] == 0 + assert initial_state["frame_count"] == 3 + assert update_response.status_code == 200 + assert update_response.json()["template_frame_index"] == 2 + assert update_response.json()["template_frame_url"] == f"/api/drafts/{draft_id}/template-frame" + + +def test_workbench_exposes_source_video_scrubber_contract( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + state_response = app_client.get(f"/api/drafts/{draft_id}") + payload = state_response.json() + video_response = app_client.get(payload["source_video_url"]) + + assert state_response.status_code == 200 + assert payload["source_video_url"] == f"/api/drafts/{draft_id}/source-video" + assert payload["fps"] == 3.0 + assert payload["duration_seconds"] == 1.0 + assert payload["process_start_frame_index"] == 0 + assert payload["process_end_frame_index"] == 2 + assert payload["workflow_step"] == "clip" + assert payload["available_steps"] == ["clip", "mask", "refine", "review"] + assert payload["can_go_back"] is False + assert payload["can_go_next"] is True + assert payload["active_sidebar_tab"] == "targets" + assert payload["compare_enabled"] is False + assert payload["latest_job_id"] is None + assert payload["can_apply_range"] is True + assert payload["can_apply_template_frame"] is True + assert video_response.status_code == 200 + assert video_response.headers["content-type"].startswith("video/") + + +def test_workflow_step_navigation_round_trip( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + refine_response = app_client.post( + f"/api/drafts/{draft_id}/workflow-step", + json={"workflow_step": "refine"}, + ) + clip_response = app_client.post( + f"/api/drafts/{draft_id}/workflow-step", + json={"workflow_step": "clip"}, + ) + + assert refine_response.status_code == 200 + assert refine_response.json()["workflow_step"] == "refine" + assert refine_response.json()["can_go_back"] is True + assert refine_response.json()["can_go_next"] is True + assert refine_response.json()["active_sidebar_tab"] == "refine" + assert clip_response.status_code == 200 + assert clip_response.json()["workflow_step"] == "clip" + assert clip_response.json()["can_go_back"] is False + assert clip_response.json()["active_sidebar_tab"] == "targets" + + +def test_processing_range_update_clears_existing_anchor_and_masks( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + app_client.post(f"/api/drafts/{draft_id}/masks") + + response = app_client.post( + f"/api/drafts/{draft_id}/processing-range", + json={"start_frame_index": 1, "end_frame_index": 2}, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["process_start_frame_index"] == 1 + assert payload["process_end_frame_index"] == 2 + assert payload["template_frame_index"] is None + assert payload["mask_names"] == [] + assert payload["selected_mask_names"] == [] + assert payload["current_mask_url"] is None + assert payload["current_preview_url"] is None + assert payload["stage"] == "coarse" + assert payload["can_apply_clicks"] is False + assert payload["can_submit"] is False + assert payload["can_apply_template_frame"] is True + + +def test_template_frame_must_fall_inside_processing_range( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + app_client.post( + f"/api/drafts/{draft_id}/processing-range", + json={"start_frame_index": 1, "end_frame_index": 2}, + ) + + response = app_client.post( + f"/api/drafts/{draft_id}/template-frame", + json={"frame_index": 0}, + ) + + assert response.status_code == 400 + assert "processing range" in response.json()["detail"] + + +def test_submit_requires_anchor_inside_processing_range( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/processing-range", + json={"start_frame_index": 1, "end_frame_index": 2}, + ) + app_client.post( + f"/api/drafts/{draft_id}/template-frame", + json={"frame_index": 1}, + ) + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + app_client.post(f"/api/drafts/{draft_id}/masks") + + response = app_client.post( + f"/api/drafts/{draft_id}/submit", + json={ + "process_start_frame_index": 1, + "process_end_frame_index": 2, + "template_frame_index": 0, + "selected_masks": ["mask_001"], + }, + ) + + assert response.status_code == 400 + assert "processing range" in response.json()["detail"] + + +def test_draft_source_video_endpoint_prefers_browser_preview(app_client: TestClient, sample_video_upload): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + session = app_client.app.state.drafts[draft_id] + preview_path = session.draft.video_path.parent / "preview_source.mp4" + preview_path.write_bytes(b"preview-source") + session.draft.browser_preview_path = preview_path + + response = app_client.get(f"/api/drafts/{draft_id}/source-video") + + assert response.status_code == 200 + assert response.content == b"preview-source" + + +def test_job_status_exposes_browser_preview_artifacts( + app_client: TestClient, +): + repository = app_client.app.state.repository + runtime_root = app_client.app.state.settings.runtime_root + job = repository.create_job( + source_video_path="queued.mp4", + template_frame_index=0, + mask_path="queued.png", + params_json="{}", + ) + job_dir = runtime_root / "jobs" / job.job_id + job_dir.mkdir(parents=True, exist_ok=True) + (job_dir / "preview_foreground.mp4").write_bytes(b"preview-fg") + (job_dir / "preview_alpha.mp4").write_bytes(b"preview-alpha") + + response = app_client.get(f"/api/jobs/{job.job_id}") + + assert response.status_code == 200 + payload = response.json() + assert payload["preview_artifacts"]["foreground"] == f"/api/jobs/{job.job_id}/artifacts/preview_foreground.mp4" + assert payload["preview_artifacts"]["alpha"] == f"/api/jobs/{job.job_id}/artifacts/preview_alpha.mp4" + + +def test_target_preset_change_rebuilds_current_mask_preview( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + initial_mask = app_client.get(f"/api/drafts/{draft_id}/current-mask").content + + update_response = app_client.patch( + f"/api/drafts/{draft_id}/targets/target-001", + json={"refine_preset": "hair"}, + ) + updated_mask = app_client.get(f"/api/drafts/{draft_id}/current-mask").content + + assert update_response.status_code == 200 + assert update_response.json()["current_mask_url"] is not None + assert update_response.json()["current_preview_url"] is not None + assert updated_mask != initial_mask + + +def test_temporal_stability_change_rebuilds_current_mask_render( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + initial_mask = app_client.get(f"/api/drafts/{draft_id}/current-mask").content + + update_response = app_client.patch( + f"/api/drafts/{draft_id}/targets/target-001", + json={"temporal_stability": 1.0}, + ) + updated_mask = app_client.get(f"/api/drafts/{draft_id}/current-mask").content + + assert update_response.status_code == 200 + assert update_response.json()["current_mask_url"] is not None + assert update_response.json()["current_preview_url"] is not None + assert updated_mask != initial_mask + + +def test_stage_change_round_trip( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + response = app_client.post( + f"/api/drafts/{draft_id}/stage", + json={"stage": "refine"}, + ) + preview_response = app_client.post( + f"/api/drafts/{draft_id}/stage", + json={"stage": "preview"}, + ) + + assert response.status_code == 200 + assert response.json()["stage"] == "refine" + assert response.json()["stage_label"] == "Edge Refinement" + assert response.json()["can_apply_clicks"] is True + assert response.json()["can_create_target"] is True + assert preview_response.status_code == 200 + assert preview_response.json()["stage"] == "preview" + assert preview_response.json()["stage_label"] == "Preview" + assert preview_response.json()["can_apply_clicks"] is False + assert preview_response.json()["can_create_target"] is False + + +def test_preview_stage_locks_save_and_brush_actions( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + preview_response = app_client.post( + f"/api/drafts/{draft_id}/stage", + json={"stage": "preview"}, + ) + brush_response = app_client.post( + f"/api/drafts/{draft_id}/brush", + json={"mode": "add", "radius": 20, "points": [[2, 2]]}, + ) + + assert preview_response.status_code == 200 + assert preview_response.json()["can_save_current_target"] is False + assert brush_response.status_code == 400 + assert brush_response.json()["detail"] == "preview mode is read-only" + + +def test_brush_round_trip_updates_current_mask_preview( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + response = app_client.post( + f"/api/drafts/{draft_id}/brush", + json={"mode": "add", "radius": 20, "points": [[2, 2], [3, 3]]}, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["current_mask_url"] is not None + assert payload["current_preview_url"] is not None + assert payload["can_save_current_target"] is True + + +def test_saved_mask_is_selected_for_export_and_unlocks_submit( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + save_response = app_client.post(f"/api/drafts/{draft_id}/masks") + state_response = app_client.get(f"/api/drafts/{draft_id}") + + assert save_response.status_code == 200 + assert save_response.json()["mask_name"] == "mask_001" + assert save_response.json()["selected_mask_names"] == ["mask_001"] + assert save_response.json()["can_submit"] is True + assert save_response.json()["active_mask_url"].endswith( + f"/api/drafts/{draft_id}/masks/mask_001" + ) + assert state_response.status_code == 200 + assert state_response.json()["selected_mask_names"] == ["mask_001"] + assert state_response.json()["active_mask_url"].endswith( + f"/api/drafts/{draft_id}/masks/mask_001" + ) + + +def test_saved_mask_endpoint_serves_named_mask( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + app_client.post(f"/api/drafts/{draft_id}/masks") + + response = app_client.get(f"/api/drafts/{draft_id}/masks/mask_001") + + assert response.status_code == 200 + assert response.headers["content-type"] == "image/png" + + +def test_undo_click_and_reset_target_round_trip( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 1, "y": 1, "positive": True}, + ) + app_client.post( + f"/api/drafts/{draft_id}/click", + json={"x": 2, "y": 2, "positive": False}, + ) + + undo_response = app_client.post(f"/api/drafts/{draft_id}/undo") + reset_response = app_client.post(f"/api/drafts/{draft_id}/reset-target") + + assert undo_response.status_code == 200 + assert undo_response.json()["targets"][0]["point_count"] == 1 + assert undo_response.json()["current_preview_url"] is not None + assert reset_response.status_code == 200 + assert reset_response.json()["targets"][0]["point_count"] == 0 + assert reset_response.json()["current_preview_url"] is None + + +def test_upload_validation_errors_return_400(app_client: TestClient): + with TestClient(app_client.app, raise_server_exceptions=False) as client: + response = client.post( + "/api/uploads", + files={"video": ("broken.mp4", b"not-a-video", "video/mp4")}, + ) + + assert response.status_code == 400 + assert response.json()["detail"] == "unable to read video frames" + + +def test_second_job_waits_until_first_job_finishes(app_client, seeded_jobs): + first_job_id, second_job_id = seeded_jobs + + first_status = app_client.get(f"/api/jobs/{first_job_id}").json() + second_status = app_client.get(f"/api/jobs/{second_job_id}").json() + + assert first_status["status"] == "running" + assert second_status["status"] == "queued" + assert second_status["queue_position"] == 1 + + +def test_job_page_exposes_polling_entrypoint(app_client): + repository = app_client.app.state.repository + job = repository.create_job( + source_video_path="queued.mp4", + template_frame_index=0, + mask_path="queued.png", + params_json="{}", + ) + + response = app_client.get(f"/jobs/{job.job_id}") + + assert response.status_code == 200 + assert 'id="job-app"' in response.text + assert f'data-status-endpoint="/api/jobs/{job.job_id}"' in response.text + assert f'data-source-video-endpoint="/api/jobs/{job.job_id}/source-video"' in response.text + assert 'id="preview-viewport"' in response.text + assert 'id="preview-mode-tabs"' in response.text + assert 'id="artifact-panel"' in response.text + assert "/static/results.js" in response.text + + +def test_job_source_video_endpoint_serves_source_file(app_client): + runtime_root = app_client.app.state.settings.runtime_root + source_path = runtime_root / "jobs" / "source.mp4" + source_path.parent.mkdir(parents=True, exist_ok=True) + source_path.write_bytes(b"video-bytes") + + repository = app_client.app.state.repository + job = repository.create_job( + source_video_path=str(source_path), + template_frame_index=0, + mask_path="queued.png", + params_json="{}", + ) + + response = app_client.get(f"/api/jobs/{job.job_id}/source-video") + + assert response.status_code == 200 + assert response.content == b"video-bytes" + + +def test_job_source_video_endpoint_prefers_browser_preview_when_present(app_client): + runtime_root = app_client.app.state.settings.runtime_root + source_dir = runtime_root / "jobs" / "source-preview" + source_dir.mkdir(parents=True, exist_ok=True) + source_path = source_dir / "source.mp4" + preview_path = source_dir / "preview_source.mp4" + source_path.write_bytes(b"video-bytes") + preview_path.write_bytes(b"preview-video-bytes") + + repository = app_client.app.state.repository + job = repository.create_job( + source_video_path=str(source_path), + template_frame_index=0, + mask_path="queued.png", + params_json="{}", + ) + + response = app_client.get(f"/api/jobs/{job.job_id}/source-video") + + assert response.status_code == 200 + assert response.content == b"preview-video-bytes" + + +def test_job_source_video_endpoint_prefers_processing_range_preview_when_present(app_client): + runtime_root = app_client.app.state.settings.runtime_root + source_dir = runtime_root / "jobs" / "source-range-preview" + source_dir.mkdir(parents=True, exist_ok=True) + source_path = source_dir / "source.mp4" + source_preview_path = source_dir / "preview_source.mp4" + source_path.write_bytes(b"video-bytes") + source_preview_path.write_bytes(b"full-preview-bytes") + + repository = app_client.app.state.repository + job = repository.create_job( + source_video_path=str(source_path), + template_frame_index=1, + mask_path="queued.png", + params_json='{"process_start_frame_index": 1, "process_end_frame_index": 2}', + ) + + job_dir = runtime_root / "jobs" / job.job_id + job_dir.mkdir(parents=True, exist_ok=True) + (job_dir / "processing_range.mp4").write_bytes(b"clip-bytes") + (job_dir / "preview_source.mp4").write_bytes(b"range-preview-bytes") + + response = app_client.get(f"/api/jobs/{job.job_id}/source-video") + + assert response.status_code == 200 + assert response.content == b"range-preview-bytes" + + +def test_missing_job_page_returns_404(app_client: TestClient): + with TestClient(app_client.app, raise_server_exceptions=False) as client: + response = client.get("/jobs/missing-job") + + assert response.status_code == 404 + assert response.json()["detail"] == "job not found" + + +def test_completed_job_exposes_artifacts_and_downloads_zip(app_client): + repository = app_client.app.state.repository + runtime_root = app_client.app.state.settings.runtime_root + job = repository.create_job( + source_video_path="finished.mp4", + template_frame_index=0, + mask_path="finished.png", + params_json="{}", + ) + repository.update_status(job.job_id, JobStatus.COMPLETED) + + job_dir = runtime_root / "jobs" / job.job_id + job_dir.mkdir(parents=True, exist_ok=True) + artifact_path = job_dir / "rgba_png.zip" + artifact_path.write_bytes(b"zip") + + status_response = app_client.get(f"/api/jobs/{job.job_id}") + download_response = app_client.get( + f"/api/jobs/{job.job_id}/artifacts/rgba_png.zip" + ) + + assert status_response.status_code == 200 + assert status_response.json()["artifacts"]["rgba_png.zip"].endswith("rgba_png.zip") + assert download_response.status_code == 200 + assert download_response.content == b"zip" + + +def test_job_status_exposes_warning_and_error_text(app_client): + repository = app_client.app.state.repository + warning_job = repository.create_job( + source_video_path="warning.mp4", + template_frame_index=0, + mask_path="warning.png", + params_json="{}", + ) + failed_job = repository.create_job( + source_video_path="failed.mp4", + template_frame_index=0, + mask_path="failed.png", + params_json="{}", + ) + + repository.update_status( + warning_job.job_id, + JobStatus.COMPLETED_WITH_WARNING, + warning_text="prores export skipped", + ) + repository.update_status( + failed_job.job_id, + JobStatus.FAILED, + error_text="gpu worker crashed", + ) + + warning_response = app_client.get(f"/api/jobs/{warning_job.job_id}") + failed_response = app_client.get(f"/api/jobs/{failed_job.job_id}") + + assert warning_response.status_code == 200 + assert warning_response.json()["warning_text"] == "prores export skipped" + assert warning_response.json()["error_text"] is None + assert failed_response.status_code == 200 + assert failed_response.json()["warning_text"] is None + assert failed_response.json()["error_text"] == "gpu worker crashed" + + +def test_job_status_exposes_review_summary_and_artifact_metadata(app_client): + repository = app_client.app.state.repository + runtime_root = app_client.app.state.settings.runtime_root + source_path = runtime_root / "inputs" / "hero.mp4" + source_path.parent.mkdir(parents=True, exist_ok=True) + source_path.write_bytes(b"source") + + job = repository.create_job( + source_video_path=str(source_path), + template_frame_index=12, + mask_path="hero_mask.png", + params_json='{"process_start_frame_index": 10, "process_end_frame_index": 20, "process_range_duration_seconds": 0.46, "template_frame_index": 12, "selected_masks": ["mask_001", "mask_002"], "selected_mask_presets": {"mask_001": "hair"}}', + ) + repository.update_status( + job.job_id, + JobStatus.COMPLETED_WITH_WARNING, + warning_text="prores export skipped", + ) + + job_dir = runtime_root / "jobs" / job.job_id + job_dir.mkdir(parents=True, exist_ok=True) + (job_dir / "foreground.mp4").write_bytes(b"foreground-bytes") + (job_dir / "alpha.mp4").write_bytes(b"alpha-bytes") + (job_dir / "rgba_png.zip").write_bytes(b"zip-bytes") + + response = app_client.get(f"/api/jobs/{job.job_id}") + + assert response.status_code == 200 + payload = response.json() + assert payload["status_label"] == "Completed with warning" + assert payload["job_summary"]["source_name"] == "hero.mp4" + assert payload["job_summary"]["template_frame_index"] == 12 + assert payload["job_summary"]["process_start_frame_index"] == 10 + assert payload["job_summary"]["process_end_frame_index"] == 20 + assert payload["job_summary"]["process_range_duration_seconds"] == 0.46 + assert payload["job_summary"]["selected_mask_count"] == 2 + assert payload["job_summary"]["selected_masks"] == ["mask_001", "mask_002"] + assert payload["job_summary"]["selected_mask_presets"] == {"mask_001": "hair"} + assert [step["state"] for step in payload["timeline"]] == [ + "complete", + "complete", + "complete", + "current", + ] + assert payload["artifact_details"]["foreground.mp4"]["label"] == "Foreground pass" + assert payload["artifact_details"]["foreground.mp4"]["size_bytes"] == len(b"foreground-bytes") + assert payload["artifact_details"]["rgba_png.zip"]["kind"] == "png_sequence" + assert payload["artifact_details"]["output_prores4444.mov"]["available"] is False diff --git a/tests/webapp/test_app_factory.py b/tests/webapp/test_app_factory.py new file mode 100644 index 0000000..238005b --- /dev/null +++ b/tests/webapp/test_app_factory.py @@ -0,0 +1,65 @@ +import runpy +import sys +from importlib.util import find_spec +from pathlib import Path + +from fastapi.testclient import TestClient + +from matanyone2.webapp.api.app import create_app +from matanyone2.webapp.config import WebAppSettings +from scripts._path_bootstrap import ensure_project_root_on_path + + +def test_create_app_builds_health_route(tmp_path, monkeypatch): + monkeypatch.setenv("MATANYONE2_WEBAPP_RUNTIME_ROOT", str(tmp_path)) + settings = WebAppSettings() + app = create_app(settings=settings) + + with TestClient(app) as client: + response = client.get("/healthz") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +def test_create_app_uses_configured_sam_model_type(tmp_path): + settings = WebAppSettings( + runtime_root=tmp_path, + database_path=tmp_path / "jobs.db", + sam_model_type="vit_b", + ) + + app = create_app(settings=settings) + + assert app.state.masking_service.sam_model_type == "vit_b" + + +def test_ensure_project_root_on_path_prepends_repo_root(monkeypatch, tmp_path): + repo_root = tmp_path / "repo" + scripts_dir = repo_root / "scripts" + scripts_dir.mkdir(parents=True) + script_path = scripts_dir / "run_internal_worker.py" + script_path.write_text("", encoding="utf-8") + monkeypatch.setattr(sys, "path", [str(scripts_dir)]) + + resolved_root = ensure_project_root_on_path(script_path) + + assert resolved_root == repo_root + assert sys.path[0] == str(repo_root) + + +def test_run_internal_worker_script_bootstraps_from_scripts_dir(monkeypatch): + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "scripts" / "run_internal_worker.py" + + monkeypatch.setattr(sys, "path", [str(script_path.parent)]) + sys.modules.pop("_path_bootstrap", None) + sys.modules.pop("scripts._path_bootstrap", None) + + globals_after_load = runpy.run_path(str(script_path), run_name="__not_main__") + + assert "main" in globals_after_load + + +def test_internal_webapp_runtime_has_matplotlib_available(): + assert find_spec("matplotlib") is not None diff --git a/tests/webapp/test_config.py b/tests/webapp/test_config.py new file mode 100644 index 0000000..06ad60f --- /dev/null +++ b/tests/webapp/test_config.py @@ -0,0 +1,13 @@ +from matanyone2.webapp.config import WebAppSettings + + +def test_webapp_settings_default_to_sam3_backend(monkeypatch): + monkeypatch.delenv("MATANYONE2_WEBAPP_SAM_BACKEND", raising=False) + monkeypatch.delenv("MATANYONE2_WEBAPP_SAM2_VARIANT", raising=False) + monkeypatch.delenv("MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH", raising=False) + + settings = WebAppSettings() + + assert settings.sam_backend == "sam3" + assert settings.sam2_variant == "sam2.1_hiera_large" + assert settings.sam3_checkpoint_path is not None diff --git a/tests/webapp/test_export_service.py b/tests/webapp/test_export_service.py new file mode 100644 index 0000000..ce47213 --- /dev/null +++ b/tests/webapp/test_export_service.py @@ -0,0 +1,162 @@ +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +from matanyone2.webapp.services.export import ExportService + + +def test_export_assets_creates_png_zip_even_when_prores_fails(tmp_path, monkeypatch): + service = ExportService(enable_prores=True) + foreground = tmp_path / "foreground.mp4" + alpha = tmp_path / "alpha.mp4" + foreground.write_bytes(b"fg") + alpha.write_bytes(b"a") + + monkeypatch.setattr( + service, + "_extract_frames", + lambda *args, **kwargs: ( + [tmp_path / "fg-0001.png"], + [tmp_path / "a-0001.png"], + 24.0, + ), + ) + monkeypatch.setattr( + service, + "_process_alpha_frames", + lambda *args, **kwargs: [np.full((4, 4), 128, dtype=np.uint8)], + ) + monkeypatch.setattr( + service, + "_overwrite_alpha_frames", + lambda *args, **kwargs: None, + ) + monkeypatch.setattr( + service, + "_write_alpha_video", + lambda *args, **kwargs: tmp_path / "alpha.mp4", + ) + monkeypatch.setattr( + service, + "_write_rgba_pngs", + lambda *args, **kwargs: tmp_path / "rgba_png", + ) + monkeypatch.setattr( + service, + "_zip_directory", + lambda *args, **kwargs: tmp_path / "rgba_png.zip", + ) + monkeypatch.setattr( + service, + "_export_preview_videos", + lambda *args, **kwargs: (tmp_path / "preview_foreground.mp4", tmp_path / "preview_alpha.mp4"), + ) + monkeypatch.setattr( + service, + "_export_prores", + lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("ffmpeg failed")), + ) + + result = service.export_assets(foreground, alpha, tmp_path) + + assert result.png_zip_path.name == "rgba_png.zip" + assert result.preview_foreground_path.name == "preview_foreground.mp4" + assert result.preview_alpha_path.name == "preview_alpha.mp4" + assert result.warning_text == "ffmpeg failed" + + +def test_export_assets_writes_rgba_png_sequence_from_videos(tmp_path): + foreground = tmp_path / "foreground.mp4" + alpha = tmp_path / "alpha.mp4" + + fg_writer = cv2.VideoWriter( + str(foreground), + cv2.VideoWriter_fourcc(*"mp4v"), + 2.0, + (4, 4), + ) + alpha_writer = cv2.VideoWriter( + str(alpha), + cv2.VideoWriter_fourcc(*"mp4v"), + 2.0, + (4, 4), + ) + fg_writer.write(np.full((4, 4, 3), (10, 20, 30), dtype=np.uint8)) + alpha_writer.write(np.full((4, 4, 3), 128, dtype=np.uint8)) + fg_writer.release() + alpha_writer.release() + + service = ExportService(enable_prores=False) + result = service.export_assets(foreground, alpha, tmp_path) + + rgba_frame = Image.open(result.rgba_png_dir / "0000.png") + + assert result.png_zip_path.exists() + assert result.preview_foreground_path.exists() + assert result.preview_alpha_path.exists() + assert rgba_frame.mode == "RGBA" + assert rgba_frame.getextrema()[3][1] > 0 + + +def test_export_prores_uses_alpha_capable_ffmpeg_command(tmp_path, monkeypatch): + rgba_dir = tmp_path / "rgba_png" + rgba_dir.mkdir() + Image.new("RGBA", (4, 4), color=(10, 20, 30, 128)).save(rgba_dir / "0000.png") + + service = ExportService(enable_prores=True) + captured = {} + + def fake_run(command): + captured["command"] = command + Path(command[-1]).write_bytes(b"mov") + + monkeypatch.setattr(service, "_run_ffmpeg_command", fake_run) + + output_path = service._export_prores( + rgba_dir, + tmp_path / "output_prores4444.mov", + fps=23.976, + ) + + assert output_path.exists() + assert "prores_ks" in captured["command"] + assert "yuva444p10le" in captured["command"] + assert "23.976" in captured["command"] + + +def test_apply_motion_softness_blurs_alpha_frame_when_requested(): + service = ExportService(enable_prores=False) + alpha = np.zeros((7, 7), dtype=np.uint8) + alpha[3, 3] = 255 + + softened = service._apply_motion_softness(alpha, motion_strength=1.0) + + assert softened[3, 3] < 255 + assert int(softened.sum()) > 255 + + +def test_stabilize_alpha_frames_blends_adjacent_frames(): + service = ExportService(enable_prores=False) + alpha_frames = [ + np.zeros((4, 4), dtype=np.uint8), + np.full((4, 4), 255, dtype=np.uint8), + ] + + stabilized = service._stabilize_alpha_frames(alpha_frames, temporal_stability=0.5) + + assert stabilized[0].shape == (4, 4) + assert 0 < int(stabilized[1].mean()) < 255 + + +def test_apply_edge_feather_softens_mask_boundary(): + service = ExportService(enable_prores=False) + alpha = np.zeros((9, 9), dtype=np.uint8) + alpha[2:7, 2:7] = 255 + + feathered = service._apply_edge_feather(alpha, feather_radius=5.0) + + assert feathered[1, 2] > 0 + assert feathered[0, 0] == 0 + assert feathered[4, 4] == 255 diff --git a/tests/webapp/test_inference_service.py b/tests/webapp/test_inference_service.py new file mode 100644 index 0000000..4b03119 --- /dev/null +++ b/tests/webapp/test_inference_service.py @@ -0,0 +1,172 @@ +from pathlib import Path + +import torch + +from matanyone2.utils.get_default_model import get_matanyone2_model +from matanyone2.webapp.services.inference import InferenceService + + +def test_run_job_writes_foreground_and_alpha_outputs(tmp_path, monkeypatch): + service = InferenceService(model_name="MatAnyone 2") + job_dir = tmp_path / "job-1" + job_dir.mkdir() + + monkeypatch.setattr( + service, + "_run_model", + lambda **_: (Path(job_dir / "foreground.mp4"), Path(job_dir / "alpha.mp4")), + ) + result = service.run_job( + source_video_path=Path("input.mp4"), + mask_path=Path("mask.png"), + job_dir=job_dir, + template_frame_index=0, + process_start_frame_index=0, + process_end_frame_index=None, + ) + + assert result.foreground_video_path.name == "foreground.mp4" + assert result.alpha_video_path.name == "alpha.mp4" + + +def test_run_job_dispatches_bidirectional_flow_for_nonzero_template_frame_index( + tmp_path, + monkeypatch, +): + service = InferenceService(model_name="MatAnyone 2") + job_dir = tmp_path / "job-1" + job_dir.mkdir() + observed = {} + + def fake_bidirectional(**kwargs): + observed.update(kwargs) + return Path(job_dir / "foreground.mp4"), Path(job_dir / "alpha.mp4") + + monkeypatch.setattr(service, "_run_bidirectional_job", fake_bidirectional, raising=False) + + result = service.run_job( + source_video_path=Path("input.mp4"), + mask_path=Path("mask.png"), + job_dir=job_dir, + template_frame_index=12, + process_start_frame_index=0, + process_end_frame_index=None, + ) + + assert observed["template_frame_index"] == 12 + assert result.foreground_video_path.name == "foreground.mp4" + + +def test_run_job_uses_processing_range_clip_and_relative_anchor(tmp_path, monkeypatch): + service = InferenceService(model_name="MatAnyone 2") + job_dir = tmp_path / "job-1" + job_dir.mkdir() + observed = {} + + def fake_prepare_processing_clip(**kwargs): + observed["clip_request"] = kwargs + clip_path = job_dir / "processing_range.mp4" + clip_path.write_bytes(b"clip") + return clip_path, 1, 3.0 + + def fake_bidirectional(**kwargs): + observed["bidirectional"] = kwargs + return Path(job_dir / "foreground.mp4"), Path(job_dir / "alpha.mp4") + + monkeypatch.setattr(service, "_prepare_processing_clip", fake_prepare_processing_clip, raising=False) + monkeypatch.setattr(service, "_run_bidirectional_job", fake_bidirectional, raising=False) + + result = service.run_job( + source_video_path=Path("input.mp4"), + mask_path=Path("mask.png"), + job_dir=job_dir, + template_frame_index=3, + process_start_frame_index=2, + process_end_frame_index=4, + ) + + assert observed["clip_request"]["process_start_frame_index"] == 2 + assert observed["clip_request"]["process_end_frame_index"] == 4 + assert observed["bidirectional"]["source_video_path"] == job_dir / "processing_range.mp4" + assert observed["bidirectional"]["template_frame_index"] == 1 + assert result.foreground_video_path.name == "foreground.mp4" + + +def test_run_job_passes_resolved_inference_hyperparameters_to_model(tmp_path, monkeypatch): + service = InferenceService(model_name="MatAnyone 2") + job_dir = tmp_path / "job-1" + job_dir.mkdir() + observed = {} + + clip_path = job_dir / "processing_range.mp4" + clip_path.write_bytes(b"clip") + + monkeypatch.setattr( + service, + "_prepare_processing_clip", + lambda **kwargs: (clip_path, 0, 24.0), + raising=False, + ) + monkeypatch.setattr( + service, + "_resolve_inference_hyperparameters", + lambda **kwargs: {"n_warmup": 12, "r_erode": 14, "r_dilate": 16}, + raising=False, + ) + + def fake_run_model(**kwargs): + observed.update(kwargs) + return Path(job_dir / "foreground.mp4"), Path(job_dir / "alpha.mp4") + + monkeypatch.setattr(service, "_run_model", fake_run_model, raising=False) + + service.run_job( + source_video_path=Path("input.mp4"), + mask_path=Path("mask.png"), + job_dir=job_dir, + template_frame_index=0, + process_start_frame_index=0, + process_end_frame_index=None, + selected_mask_controls={"mask_001": {"edge_feather_radius": 6.0, "temporal_stability": 0.8}}, + selected_mask_presets={"mask_001": "hair"}, + ) + + assert observed["n_warmup"] == 12 + assert observed["r_erode"] == 14 + assert observed["r_dilate"] == 16 + + +def test_get_matanyone2_model_can_be_called_twice_in_same_process(monkeypatch, tmp_path): + checkpoint_path = tmp_path / "matanyone2.pth" + checkpoint_path.write_bytes(b"weights") + + class FakeModel: + def __init__(self, cfg, single_object): + self.cfg = cfg + self.single_object = single_object + self.loaded_weights = None + + def to(self, device): + self.device = device + return self + + def eval(self): + return self + + def load_weights(self, weights): + self.loaded_weights = weights + + monkeypatch.setattr( + "matanyone2.utils.get_default_model.MatAnyone2", + FakeModel, + ) + monkeypatch.setattr( + "matanyone2.utils.get_default_model.torch.load", + lambda path, map_location=None: {"path": str(path), "map_location": str(map_location)}, + ) + + first_model = get_matanyone2_model(str(checkpoint_path), device=torch.device("cpu")) + second_model = get_matanyone2_model(str(checkpoint_path), device=torch.device("cpu")) + + assert first_model.cfg.weights == str(checkpoint_path) + assert second_model.cfg.weights == str(checkpoint_path) diff --git a/tests/webapp/test_masking_service.py b/tests/webapp/test_masking_service.py new file mode 100644 index 0000000..8951731 --- /dev/null +++ b/tests/webapp/test_masking_service.py @@ -0,0 +1,342 @@ +import numpy as np +from PIL import Image +import pytest + +from matanyone2.webapp.models import DraftRecord +from matanyone2.webapp.services.masking import ( + MaskingService, + Sam2MaskController, + Sam3MaskController, + merge_masks, +) + + +def test_merge_masks_collapses_multiple_targets_into_single_uint8_mask(): + mask_a = np.array([[1, 0], [0, 0]], dtype=np.uint8) + mask_b = np.array([[0, 0], [1, 0]], dtype=np.uint8) + + merged = merge_masks([mask_a, mask_b]) + + assert merged.dtype == np.uint8 + assert merged.tolist() == [[255, 0], [255, 0]] + + +def test_apply_click_and_save_mask_persists_named_mask(tmp_path): + template_frame = tmp_path / "template.png" + Image.new("RGB", (4, 4), color=(0, 0, 0)).save(template_frame) + draft = DraftRecord( + draft_id="draft-1", + video_path=tmp_path / "input.mp4", + template_frame_path=template_frame, + width=4, + height=4, + fps=24.0, + frame_count=1, + duration_seconds=0.04, + ) + + class FakeController: + def __init__(self): + self.calls = [] + + def first_frame_click(self, image, points, labels, multimask=True): + self.calls.append((points.copy(), labels.copy(), multimask)) + mask = np.zeros((4, 4), dtype=np.uint8) + mask[1, 1] = 1 + return mask, np.zeros((4, 4), dtype=np.float32), Image.fromarray(image) + + controller = FakeController() + service = MaskingService(runtime_root=tmp_path, controller_factory=lambda: controller) + session = service.create_session(draft) + + result = service.apply_click(session, x=1, y=1, positive=True) + mask_name = service.save_current_mask(session) + + assert result.current_mask_path.exists() + assert mask_name == "mask_001" + assert session.saved_masks[mask_name].exists() + assert controller.calls[0][0].tolist() == [[1, 1]] + + +def test_save_current_mask_resets_click_history_for_next_target(tmp_path): + template_frame = tmp_path / "template.png" + Image.new("RGB", (10, 10), color=(0, 0, 0)).save(template_frame) + draft = DraftRecord( + draft_id="draft-1", + video_path=tmp_path / "input.mp4", + template_frame_path=template_frame, + width=10, + height=10, + fps=24.0, + frame_count=1, + duration_seconds=0.04, + ) + + class FakeController: + def __init__(self): + self.calls = [] + + def first_frame_click(self, image, points, labels, multimask=True): + self.calls.append((points.copy(), labels.copy(), multimask)) + mask = np.zeros((10, 10), dtype=np.uint8) + for x, y in points.tolist(): + mask[y, x] = 1 + return mask, np.zeros((10, 10), dtype=np.float32), Image.fromarray(image) + + controller = FakeController() + service = MaskingService(runtime_root=tmp_path, controller_factory=lambda: controller) + session = service.create_session(draft) + + service.apply_click(session, x=1, y=1, positive=True) + service.save_current_mask(session) + service.apply_click(session, x=8, y=8, positive=True) + + assert session.click_points == [(8, 8)] + assert session.click_labels == [1] + assert controller.calls[1][0].tolist() == [[8, 8]] + + +def test_save_current_mask_applies_hair_preset_and_records_metadata(tmp_path): + template_frame = tmp_path / "template.png" + Image.new("RGB", (9, 9), color=(0, 0, 0)).save(template_frame) + draft = DraftRecord( + draft_id="draft-1", + video_path=tmp_path / "input.mp4", + template_frame_path=template_frame, + width=9, + height=9, + fps=24.0, + frame_count=1, + duration_seconds=0.04, + ) + + service = MaskingService(runtime_root=tmp_path, controller_factory=lambda: None) + session = service.create_session(draft) + session.current_mask_path = session.session_dir / "current_mask.png" + Image.fromarray( + np.pad(np.array([[255]], dtype=np.uint8), ((4, 4), (4, 4))), + mode="L", + ).save(session.current_mask_path) + service.update_target(session, session.active_target_id, refine_preset="hair") + + mask_name = service.save_current_mask(session) + saved_mask = np.array(Image.open(session.saved_masks[mask_name]).convert("L")) + + assert mask_name == "mask_001" + assert session.saved_mask_presets[mask_name] == "hair" + assert int(saved_mask.sum()) > 255 + + +def test_save_current_mask_applies_edge_preset_to_tighten_mask(tmp_path): + template_frame = tmp_path / "template.png" + Image.new("RGB", (9, 9), color=(0, 0, 0)).save(template_frame) + draft = DraftRecord( + draft_id="draft-1", + video_path=tmp_path / "input.mp4", + template_frame_path=template_frame, + width=9, + height=9, + fps=24.0, + frame_count=1, + duration_seconds=0.04, + ) + + service = MaskingService(runtime_root=tmp_path, controller_factory=lambda: None) + session = service.create_session(draft) + original_mask = np.zeros((9, 9), dtype=np.uint8) + original_mask[2:7, 2:7] = 255 + session.current_mask_path = session.session_dir / "current_mask.png" + Image.fromarray(original_mask, mode="L").save(session.current_mask_path) + service.update_target(session, session.active_target_id, refine_preset="edge") + + mask_name = service.save_current_mask(session) + saved_mask = np.array(Image.open(session.saved_masks[mask_name]).convert("L")) + + assert session.saved_mask_presets[mask_name] == "edge" + assert int(saved_mask.sum()) < int(original_mask.sum()) + + +def test_update_target_reapplies_current_render_for_new_refine_preset(tmp_path): + template_frame = tmp_path / "template.png" + Image.new("RGB", (9, 9), color=(0, 0, 0)).save(template_frame) + draft = DraftRecord( + draft_id="draft-1", + video_path=tmp_path / "input.mp4", + template_frame_path=template_frame, + width=9, + height=9, + fps=24.0, + frame_count=1, + duration_seconds=0.04, + ) + + service = MaskingService(runtime_root=tmp_path, controller_factory=lambda: None) + session = service.create_session(draft) + base_mask = np.zeros((9, 9), dtype=np.uint8) + base_mask[4, 4] = 255 + session.current_mask_path = session.session_dir / "current_mask.png" + Image.fromarray(base_mask, mode="L").save(session.current_mask_path) + + service.update_target( + session, + session.active_target_id, + refine_preset="hair", + ) + + rerendered_mask = np.array(Image.open(session.current_mask_path).convert("L")) + + assert int(rerendered_mask.sum()) > int(base_mask.sum()) + assert session.current_preview_path is not None + assert session.current_preview_path.exists() + + +def test_update_target_applies_edge_feather_radius_to_current_render(tmp_path): + template_frame = tmp_path / "template.png" + Image.new("RGB", (9, 9), color=(0, 0, 0)).save(template_frame) + draft = DraftRecord( + draft_id="draft-1", + video_path=tmp_path / "input.mp4", + template_frame_path=template_frame, + width=9, + height=9, + fps=24.0, + frame_count=1, + duration_seconds=0.04, + ) + + service = MaskingService(runtime_root=tmp_path, controller_factory=lambda: None) + session = service.create_session(draft) + base_mask = np.zeros((9, 9), dtype=np.uint8) + base_mask[2:7, 2:7] = 255 + session.current_mask_path = session.session_dir / "current_mask.png" + Image.fromarray(base_mask, mode="L").save(session.current_mask_path) + + service.update_target( + session, + session.active_target_id, + edge_feather_radius=6.0, + ) + + feathered_mask = np.array(Image.open(session.current_mask_path).convert("L")) + + assert feathered_mask.dtype == np.uint8 + assert feathered_mask[1, 2] > 0 + assert feathered_mask[0, 0] == 0 + + +def test_update_target_mutates_name_visibility_and_lock_state(tmp_path): + template_frame = tmp_path / "template.png" + Image.new("RGB", (4, 4), color=(0, 0, 0)).save(template_frame) + draft = DraftRecord( + draft_id="draft-1", + video_path=tmp_path / "input.mp4", + template_frame_path=template_frame, + width=4, + height=4, + fps=24.0, + frame_count=1, + duration_seconds=0.04, + ) + + service = MaskingService(runtime_root=tmp_path, controller_factory=lambda: None) + session = service.create_session(draft) + target = service.create_target(session, name="Hero") + + updated = service.update_target( + session, + target.target_id, + name="Lead Actor", + visible=False, + locked=True, + ) + + assert updated.name == "Lead Actor" + assert updated.visible is False + assert updated.locked is True + + +def test_masking_service_defaults_to_sam3_backend(monkeypatch, tmp_path): + service = MaskingService(runtime_root=tmp_path) + sentinel = object() + + monkeypatch.setattr(service, "_build_sam3_controller", lambda: sentinel) + + assert service._get_controller() is sentinel + + +def test_sam3_mask_controller_selects_highest_scoring_mask_and_renders_preview(): + image = np.zeros((6, 6, 3), dtype=np.uint8) + image[..., 2] = 64 + points = np.array([[1, 1], [4, 4]], dtype=np.int32) + labels = np.array([1, 0], dtype=np.int32) + + class FakeInteractivePredictor: + def __init__(self): + self.calls = [] + + def set_image(self, image_value): + self.calls.append(("set_image", image_value.shape)) + + def predict(self, point_coords, point_labels, multimask_output): + self.calls.append( + ("predict", point_coords.copy(), point_labels.copy(), multimask_output) + ) + masks = np.zeros((3, 6, 6), dtype=np.uint8) + masks[2, 1:4, 1:4] = 1 + scores = np.array([0.1, 0.5, 0.95], dtype=np.float32) + logits = np.zeros((3, 6, 6), dtype=np.float32) + return masks, scores, logits + + controller = Sam3MaskController(FakeInteractivePredictor()) + + mask, scores, preview = controller.first_frame_click( + image=image, + points=points, + labels=labels, + multimask=True, + ) + + assert mask.shape == (6, 6) + assert mask.dtype == np.uint8 + assert mask[2, 2] == 1 + assert float(scores[2]) == pytest.approx(0.95) + assert preview.size == (6, 6) + + +def test_sam2_mask_controller_selects_highest_scoring_mask_and_renders_preview(): + image = np.zeros((6, 6, 3), dtype=np.uint8) + image[..., 1] = 32 + points = np.array([[1, 1], [4, 4]], dtype=np.int32) + labels = np.array([1, 0], dtype=np.int32) + + class FakePredictor: + def __init__(self): + self.calls = [] + + def set_image(self, image_value): + self.calls.append(("set_image", image_value.shape)) + + def predict(self, point_coords, point_labels, multimask_output): + self.calls.append( + ("predict", point_coords.copy(), point_labels.copy(), multimask_output) + ) + masks = np.zeros((3, 6, 6), dtype=np.uint8) + masks[1, 2:5, 2:5] = 1 + scores = np.array([0.1, 0.9, 0.2], dtype=np.float32) + logits = np.zeros((3, 6, 6), dtype=np.float32) + return masks, scores, logits + + controller = Sam2MaskController(FakePredictor()) + + mask, scores, preview = controller.first_frame_click( + image=image, + points=points, + labels=labels, + multimask=True, + ) + + assert mask.shape == (6, 6) + assert mask.dtype == np.uint8 + assert mask[3, 3] == 1 + assert float(scores[1]) == pytest.approx(0.9) + assert preview.size == (6, 6) diff --git a/tests/webapp/test_page_templates.py b/tests/webapp/test_page_templates.py new file mode 100644 index 0000000..0e186ed --- /dev/null +++ b/tests/webapp/test_page_templates.py @@ -0,0 +1,145 @@ +from fastapi.testclient import TestClient + + +def test_upload_page_renders_new_session_shell(app_client: TestClient): + response = app_client.get("/") + + assert response.status_code == 200 + assert 'class="app-shell"' in response.text + assert 'data-page="upload"' in response.text + assert 'id="dropzone-panel"' in response.text + assert 'id="media-info-card"' in response.text + + +def test_annotation_page_renders_workbench_layout( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + response = app_client.get(f"/drafts/{draft_id}/workspace") + + assert response.status_code == 200 + assert 'id="workspace-app"' in response.text + assert 'class="workspace-shell"' in response.text + assert 'data-default-canvas-mode="source"' in response.text + assert 'id="workflow-stepper"' in response.text + assert 'data-workflow-step="clip"' in response.text + assert 'data-workflow-step="mask"' in response.text + assert 'data-workflow-step="refine"' in response.text + assert 'data-workflow-step="review"' in response.text + assert 'id="workspace-monitor"' in response.text + assert 'id="workspace-sidebar"' in response.text + assert 'id="workspace-sidebar-tabs"' in response.text + assert 'id="sidebar-tab-targets"' in response.text + assert 'id="sidebar-tab-refine"' in response.text + assert 'id="sidebar-tab-export"' in response.text + assert 'id="sidebar-panel-targets"' in response.text + assert 'id="sidebar-panel-refine"' in response.text + assert 'id="sidebar-panel-export"' in response.text + assert 'id="monitor-view-tabs"' in response.text + assert 'data-canvas-mode="source"' in response.text + assert 'data-canvas-mode="overlay"' in response.text + assert 'data-canvas-mode="mask"' in response.text + assert 'data-canvas-mode="alpha"' in response.text + assert 'data-canvas-mode="foreground"' in response.text + assert 'id="workspace-monitor-frame"' in response.text + assert 'id="workspace-monitor-video"' in response.text + assert 'id="workspace-monitor-image"' in response.text + assert 'id="workspace-overlay-canvas"' in response.text + assert 'id="workspace-timeline-dock"' in response.text + assert 'id="clip-primary-rail"' in response.text + assert 'id="mark-range-in"' in response.text + assert 'id="mark-range-out"' in response.text + assert 'id="clear-range-selection"' in response.text + assert 'class="timeline-inline-actions timeline-inline-actions--compact"' in response.text + assert 'id="timeline-current-label"' in response.text + assert 'id="timeline-in-chip"' in response.text + assert 'id="timeline-out-chip"' in response.text + assert 'id="timeline-duration-chip"' in response.text + assert 'id="anchor-rail"' in response.text + assert 'id="anchor-frame-slider"' in response.text + assert 'id="undo-click"' in response.text + assert 'id="reset-target"' in response.text + assert 'id="brush-radius"' in response.text + assert 'id="overlay-opacity"' in response.text + assert 'id="preset-strength"' in response.text + assert 'id="motion-strength"' in response.text + assert 'id="temporal-stability"' in response.text + assert 'id="edge-feather-radius"' in response.text + assert 'id="workspace-review-sidebar"' in response.text + assert 'id="review-summary-list"' in response.text + assert 'id="target-review-list"' in response.text + assert 'id="artifact-summary-list"' in response.text + assert 'id="workspace-nav-back"' in response.text + assert 'id="workspace-nav-next"' in response.text + assert 'id="workspace-return-to-clip"' in response.text + assert 'id="workspace-return-to-refine"' in response.text + assert 'id="preview-compare-strip"' not in response.text + assert 'id="compare-toggle"' not in response.text + assert 'id="compare-drawer"' not in response.text + assert 'id="keyframe-video"' not in response.text + assert 'id="canvas-keyframe-panel"' not in response.text + assert 'id="anchor-frame-panel"' not in response.text + + +def test_job_page_renders_review_viewport(app_client: TestClient): + repository = app_client.app.state.repository + job = repository.create_job( + source_video_path="queued.mp4", + template_frame_index=0, + mask_path="queued.png", + params_json="{}", + ) + + response = app_client.get(f"/jobs/{job.job_id}") + + assert response.status_code == 200 + assert 'id="preview-viewport"' in response.text + assert 'id="preview-mode-tabs"' in response.text + assert 'id="artifact-panel"' in response.text + assert 'id="preview-overlay-canvas"' in response.text + assert 'id="overlay-foreground-video"' in response.text + assert 'id="overlay-alpha-video"' in response.text + assert 'id="review-summary-panel"' in response.text + assert 'id="target-review-panel"' in response.text + assert 'id="target-review-list"' in response.text + assert 'id="job-timeline"' in response.text + assert 'id="artifact-summary-list"' in response.text + assert 'id="warning-panel"' in response.text + + +def test_job_page_keeps_preview_streams_separate_from_download_artifacts(app_client: TestClient): + repository = app_client.app.state.repository + job = repository.create_job( + source_video_path="queued.mp4", + template_frame_index=0, + mask_path="queued.png", + params_json="{}", + ) + + response = app_client.get(f"/jobs/{job.job_id}") + + assert response.status_code == 200 + assert 'data-preview-foreground-endpoint' in response.text + assert 'data-preview-alpha-endpoint' in response.text + + +def test_annotate_route_keeps_workspace_compatibility( + app_client: TestClient, + sample_video_upload, +): + upload_response = app_client.post( + "/api/uploads", + files={"video": sample_video_upload}, + ) + draft_id = upload_response.json()["draft_id"] + + response = app_client.get(f"/drafts/{draft_id}/annotate") + + assert response.status_code == 200 + assert 'id="workspace-app"' in response.text diff --git a/tests/webapp/test_repository.py b/tests/webapp/test_repository.py new file mode 100644 index 0000000..2e929a0 --- /dev/null +++ b/tests/webapp/test_repository.py @@ -0,0 +1,21 @@ +from matanyone2.webapp.models import JobStatus +from matanyone2.webapp.repository import JobRepository + + +def test_repository_creates_job_and_reports_queue_position(tmp_path): + repo = JobRepository.from_path(tmp_path / "jobs.db") + first = repo.create_job( + source_video_path="a.mp4", + template_frame_index=0, + mask_path="a.png", + params_json="{}", + ) + second = repo.create_job( + source_video_path="b.mp4", + template_frame_index=0, + mask_path="b.png", + params_json="{}", + ) + + assert repo.get_job(first.job_id).status is JobStatus.QUEUED + assert repo.get_queue_position(second.job_id) == 2 diff --git a/tests/webapp/test_service_scripts.py b/tests/webapp/test_service_scripts.py new file mode 100644 index 0000000..0ee4373 --- /dev/null +++ b/tests/webapp/test_service_scripts.py @@ -0,0 +1,248 @@ +from pathlib import Path +import json +import subprocess +import time + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPTS_DIR = REPO_ROOT / "scripts" + + +def _run_powershell_script(script_name: str, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(SCRIPTS_DIR / script_name), + *args, + ], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def test_start_script_dry_run_outputs_expected_commands(tmp_path): + service_root = tmp_path / "service" + + result = _run_powershell_script( + "start_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + "8123", + "-DryRun", + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + + assert payload["status"] == "dry_run" + assert payload["python_path"].endswith(r".venv\Scripts\python.exe") + assert payload["base_url"] == "http://127.0.0.1:8123" + assert payload["webapp_args"] == [ + "-m", + "uvicorn", + "scripts.run_internal_webapp:app", + "--host", + "127.0.0.1", + "--port", + "8123", + ] + assert payload["worker_args"] == ["scripts/run_internal_worker.py"] + + +def test_check_script_reports_not_running_when_state_file_missing(tmp_path): + service_root = tmp_path / "service" + + result = _run_powershell_script( + "check_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + ) + + assert result.returncode == 1 + payload = json.loads(result.stdout) + assert payload["status"] == "not_running" + assert payload["state_file"].endswith("service.json") + + +def test_stop_script_dry_run_is_noop_without_state_file(tmp_path): + service_root = tmp_path / "service" + + result = _run_powershell_script( + "stop_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-DryRun", + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["status"] == "not_running" + assert payload["service_root"] == str(service_root) + + +def test_stop_script_waits_for_processes_to_exit(tmp_path): + service_root = tmp_path / "service" + service_root.mkdir(parents=True, exist_ok=True) + state_file = service_root / "service.json" + proc_one = subprocess.Popen( + ["powershell", "-NoProfile", "-Command", "Start-Sleep -Seconds 60"] + ) + proc_two = subprocess.Popen( + ["powershell", "-NoProfile", "-Command", "Start-Sleep -Seconds 60"] + ) + try: + state_file.write_text( + json.dumps( + { + "webapp_pid": proc_one.pid, + "worker_pid": proc_two.pid, + } + ), + encoding="utf-8", + ) + + result = _run_powershell_script( + "stop_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + ) + + assert result.returncode == 0, result.stderr + payload = json.loads(result.stdout) + assert payload["status"] == "stopped" + + for _ in range(10): + if proc_one.poll() is not None and proc_two.poll() is not None: + break + time.sleep(0.2) + + assert proc_one.poll() is not None + assert proc_two.poll() is not None + assert not state_file.exists() + finally: + proc_one.kill() + proc_two.kill() + + +def test_start_script_and_check_script_report_running_state(tmp_path): + service_root = tmp_path / "service" + port = "8137" + + start_result = _run_powershell_script( + "start_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) + + assert start_result.returncode == 0, start_result.stderr + start_payload = json.loads(start_result.stdout) + assert start_payload["status"] == "started" + + try: + check_payload = None + check_result = None + for _ in range(20): + check_result = _run_powershell_script( + "check_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) + check_payload = json.loads(check_result.stdout) + if check_result.returncode == 0 and check_payload["status"] == "running": + break + time.sleep(0.5) + + assert check_result is not None + assert check_result.returncode == 0, check_result.stderr + assert check_payload is not None + assert check_payload["status"] == "running" + assert check_payload["http_status"] == 200 + + time.sleep(1.5) + stable_result = _run_powershell_script( + "check_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) + stable_payload = json.loads(stable_result.stdout) + + assert stable_result.returncode == 0, stable_result.stderr + assert stable_payload["status"] == "running" + assert stable_payload["http_status"] == 200 + finally: + _run_powershell_script( + "stop_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) + + +def test_stop_script_cleans_up_started_service_processes(tmp_path): + service_root = tmp_path / "service" + port = "8138" + + start_result = _run_powershell_script( + "start_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) + + assert start_result.returncode == 0, start_result.stderr + + try: + for _ in range(20): + check_result = _run_powershell_script( + "check_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) + if check_result.returncode == 0: + break + time.sleep(0.5) + + stop_result = _run_powershell_script( + "stop_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) + dry_run_result = _run_powershell_script( + "stop_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + "-DryRun", + ) + + assert stop_result.returncode == 0, stop_result.stderr + assert dry_run_result.returncode == 0, dry_run_result.stderr + assert json.loads(dry_run_result.stdout)["status"] == "not_running" + finally: + _run_powershell_script( + "stop_internal_webapp.ps1", + "-ServiceRoot", + str(service_root), + "-Port", + port, + ) diff --git a/tests/webapp/test_smoke.py b/tests/webapp/test_smoke.py new file mode 100644 index 0000000..58f7086 --- /dev/null +++ b/tests/webapp/test_smoke.py @@ -0,0 +1,168 @@ +from dataclasses import dataclass + +import pytest +import requests + +from matanyone2.webapp.smoke import build_service_env +from matanyone2.webapp.smoke import poll_jobs +from matanyone2.webapp.smoke import wait_for_server + + +@dataclass +class _FakeResponse: + status_code: int + payload: dict + + def json(self): + return self.payload + + +class _FakeSession: + def __init__(self, responses): + self._responses = responses + + def get(self, url, timeout): + response = self._responses.pop(0) + if isinstance(response, Exception): + raise response + return response + + +def test_wait_for_server_retries_until_ready(): + session = _FakeSession( + [ + requests.RequestException("not ready"), + _FakeResponse(503, {}), + _FakeResponse(200, {}), + ] + ) + sleep_calls = [] + monotonic_values = iter([0.0, 0.05, 0.10, 0.15]) + + wait_for_server( + session, + "http://127.0.0.1:8010", + timeout_seconds=1.0, + poll_interval_seconds=0.1, + sleep=sleep_calls.append, + monotonic=lambda: next(monotonic_values), + ) + + assert sleep_calls == [0.1, 0.1] + + +def test_poll_jobs_tracks_queued_follow_up_job(): + session = _FakeSession( + [ + _FakeResponse(200, {"job_id": "job-1", "status": "running"}), + _FakeResponse(200, {"job_id": "job-2", "status": "queued"}), + _FakeResponse(200, {"job_id": "job-1", "status": "completed"}), + _FakeResponse(200, {"job_id": "job-2", "status": "running"}), + _FakeResponse(200, {"job_id": "job-1", "status": "completed"}), + _FakeResponse(200, {"job_id": "job-2", "status": "completed"}), + ] + ) + sleep_calls = [] + monotonic_values = iter([0.0, 0.1, 0.2, 0.3]) + + statuses = poll_jobs( + session, + "http://127.0.0.1:8010", + ["job-1", "job-2"], + timeout_seconds=5.0, + poll_interval_seconds=0.25, + sleep=sleep_calls.append, + monotonic=lambda: next(monotonic_values), + ) + + assert statuses["job-1"]["status"] == "completed" + assert statuses["job-2"]["status"] == "completed" + assert sleep_calls == [0.25, 0.25] + + +def test_poll_jobs_requires_follow_up_job_to_queue(): + session = _FakeSession( + [ + _FakeResponse(200, {"job_id": "job-1", "status": "running"}), + _FakeResponse(200, {"job_id": "job-2", "status": "running"}), + _FakeResponse(200, {"job_id": "job-1", "status": "completed"}), + _FakeResponse(200, {"job_id": "job-2", "status": "completed"}), + ] + ) + monotonic_values = iter([0.0, 0.1, 0.2]) + + with pytest.raises(AssertionError, match="never entered queued status"): + poll_jobs( + session, + "http://127.0.0.1:8010", + ["job-1", "job-2"], + timeout_seconds=5.0, + poll_interval_seconds=0.25, + sleep=lambda _: None, + monotonic=lambda: next(monotonic_values), + ) + + +def test_poll_jobs_retries_transient_connection_errors(): + session = _FakeSession( + [ + _FakeResponse(200, {"job_id": "job-1", "status": "running"}), + _FakeResponse(200, {"job_id": "job-2", "status": "queued"}), + requests.RequestException("connection reset"), + _FakeResponse(200, {"job_id": "job-1", "status": "completed"}), + _FakeResponse(200, {"job_id": "job-2", "status": "completed"}), + ] + ) + sleep_calls = [] + monotonic_values = iter([0.0, 0.1, 0.2, 0.3]) + + statuses = poll_jobs( + session, + "http://127.0.0.1:8010", + ["job-1", "job-2"], + timeout_seconds=5.0, + poll_interval_seconds=0.25, + sleep=sleep_calls.append, + monotonic=lambda: next(monotonic_values), + ) + + assert statuses["job-1"]["status"] == "completed" + assert statuses["job-2"]["status"] == "completed" + assert sleep_calls == [0.25, 0.25] + + +def test_poll_jobs_accepts_completed_with_warning(): + session = _FakeSession( + [ + _FakeResponse(200, {"job_id": "job-1", "status": "running"}), + _FakeResponse(200, {"job_id": "job-2", "status": "queued"}), + _FakeResponse(200, {"job_id": "job-1", "status": "completed_with_warning"}), + _FakeResponse(200, {"job_id": "job-2", "status": "completed"}), + ] + ) + monotonic_values = iter([0.0, 0.1, 0.2]) + + statuses = poll_jobs( + session, + "http://127.0.0.1:8010", + ["job-1", "job-2"], + timeout_seconds=5.0, + poll_interval_seconds=0.25, + sleep=lambda _: None, + monotonic=lambda: next(monotonic_values), + ) + + assert statuses["job-1"]["status"] == "completed_with_warning" + assert statuses["job-2"]["status"] == "completed" + + +def test_build_service_env_defaults_to_sam3_and_preserves_overrides(monkeypatch, tmp_path): + monkeypatch.delenv("MATANYONE2_WEBAPP_SAM_BACKEND", raising=False) + monkeypatch.delenv("MATANYONE2_WEBAPP_SAM_MODEL_TYPE", raising=False) + monkeypatch.setenv("MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH", r"D:\models\sam3.pt") + + env = build_service_env(tmp_path / "runtime", enable_prores=True) + + assert env["MATANYONE2_WEBAPP_SAM_BACKEND"] == "sam3" + assert env["MATANYONE2_WEBAPP_SAM3_CHECKPOINT_PATH"] == r"D:\models\sam3.pt" + assert "MATANYONE2_WEBAPP_SAM_MODEL_TYPE" not in env diff --git a/tests/webapp/test_video_service.py b/tests/webapp/test_video_service.py new file mode 100644 index 0000000..12cfee2 --- /dev/null +++ b/tests/webapp/test_video_service.py @@ -0,0 +1,54 @@ +from pathlib import Path + +from matanyone2.webapp.services.video import VideoDraftService + + +def test_create_draft_extracts_template_frame_and_metadata(tmp_path, sample_video_path): + service = VideoDraftService( + runtime_root=tmp_path, + max_video_seconds=10, + max_upload_bytes=10_000_000, + ) + draft = service.create_draft(Path(sample_video_path)) + + assert draft.frame_count > 0 + assert draft.template_frame_path.exists() + assert draft.duration_seconds <= 10 + assert draft.template_frame_index == 0 + assert draft.process_start_frame_index == 0 + assert draft.process_end_frame_index == draft.frame_count - 1 + + +def test_create_draft_records_browser_preview_path(tmp_path, sample_video_path, monkeypatch): + service = VideoDraftService( + runtime_root=tmp_path, + max_video_seconds=10, + max_upload_bytes=10_000_000, + ) + + def fake_ensure_browser_preview(video_path, *, preview_path=None): + assert preview_path is not None + preview_path.write_bytes(b"preview") + return preview_path + + monkeypatch.setattr(service, "ensure_browser_preview", fake_ensure_browser_preview) + + draft = service.create_draft(Path(sample_video_path)) + + assert draft.browser_preview_path is not None + assert draft.browser_preview_path.name == "preview_source.mp4" + assert draft.browser_preview_path.exists() + + +def test_select_template_frame_updates_draft_metadata_and_file(tmp_path, sample_video_path): + service = VideoDraftService( + runtime_root=tmp_path, + max_video_seconds=10, + max_upload_bytes=10_000_000, + ) + draft = service.create_draft(Path(sample_video_path)) + + updated = service.select_template_frame(draft, 2) + + assert updated.template_frame_index == 2 + assert updated.template_frame_path.exists() diff --git a/tests/webapp/test_worker.py b/tests/webapp/test_worker.py new file mode 100644 index 0000000..e6a53fb --- /dev/null +++ b/tests/webapp/test_worker.py @@ -0,0 +1,190 @@ +from pathlib import Path +import time + +import pytest + +import scripts.run_internal_worker as run_internal_worker +from matanyone2.webapp.models import ExportResult +from matanyone2.webapp.models import JobStatus +from matanyone2.webapp.queue import QueueCoordinator +from matanyone2.webapp.repository import JobRepository +from matanyone2.webapp.config import WebAppSettings +from matanyone2.webapp.worker import WorkerLoop + + +def test_recover_running_jobs_marks_them_interrupted(tmp_path): + repo = JobRepository.from_path(tmp_path / "jobs.db") + job = repo.create_job( + source_video_path="a.mp4", + template_frame_index=0, + mask_path="a.png", + params_json="{}", + ) + repo.update_status(job.job_id, JobStatus.RUNNING) + + coordinator = QueueCoordinator(repo) + coordinator.recover_interrupted_jobs() + + assert repo.get_job(job.job_id).status is JobStatus.INTERRUPTED + + +def test_process_next_job_completes_with_warning_when_export_warns(tmp_path): + repo = JobRepository.from_path(tmp_path / "jobs.db") + job = repo.create_job( + source_video_path="a.mp4", + template_frame_index=0, + mask_path="a.png", + params_json='{"process_start_frame_index": 3, "process_end_frame_index": 9, "selected_mask_controls": {"mask_001": {"motion_strength": 0.6, "temporal_stability": 0.8}}}', + ) + + class FakeInferenceService: + def run_job( + self, + *, + source_video_path, + mask_path, + job_dir, + template_frame_index, + process_start_frame_index, + process_end_frame_index, + selected_mask_controls, + selected_mask_presets, + ): + assert process_start_frame_index == 3 + assert process_end_frame_index == 9 + assert selected_mask_controls["mask_001"]["motion_strength"] == 0.6 + assert selected_mask_controls["mask_001"]["temporal_stability"] == 0.8 + assert selected_mask_presets == {} + foreground = Path(job_dir) / "foreground.mp4" + alpha = Path(job_dir) / "alpha.mp4" + foreground.write_bytes(b"fg") + alpha.write_bytes(b"a") + return type( + "InferenceResultLike", + (), + { + "foreground_video_path": foreground, + "alpha_video_path": alpha, + }, + )() + + class FakeExportService: + def export_assets( + self, + foreground_video_path, + alpha_video_path, + job_dir, + *, + motion_strength, + temporal_stability, + edge_feather_radius, + ): + assert motion_strength == 0.6 + assert temporal_stability == 0.8 + assert edge_feather_radius == 0.0 + rgba_dir = Path(job_dir) / "rgba_png" + rgba_dir.mkdir(parents=True, exist_ok=True) + zip_path = Path(job_dir) / "rgba_png.zip" + zip_path.write_bytes(b"zip") + return ExportResult( + rgba_png_dir=rgba_dir, + png_zip_path=zip_path, + preview_foreground_path=Path(job_dir) / "preview_foreground.mp4", + preview_alpha_path=Path(job_dir) / "preview_alpha.mp4", + prores_path=None, + warning_text="ffmpeg failed", + ) + + worker = WorkerLoop( + coordinator=QueueCoordinator(repo), + repository=repo, + inference_service=FakeInferenceService(), + export_service=FakeExportService(), + runtime_root=tmp_path, + ) + + processed_job_id = worker.process_next_job() + updated_job = repo.get_job(job.job_id) + + assert processed_job_id == job.job_id + assert updated_job.status is JobStatus.COMPLETED_WITH_WARNING + assert updated_job.warning_text == "ffmpeg failed" + + +def test_run_forever_keeps_polling_after_idle(monkeypatch, tmp_path): + repo = JobRepository.from_path(tmp_path / "jobs.db") + worker = WorkerLoop( + coordinator=QueueCoordinator(repo), + repository=repo, + inference_service=object(), + export_service=object(), + runtime_root=tmp_path, + ) + + observed_calls = [] + process_results = iter([None, "job-1"]) + + def fake_process_next_job(): + observed_calls.append("tick") + try: + return next(process_results) + except StopIteration as exc: + raise RuntimeError("stop loop") from exc + + sleep_calls = [] + + monkeypatch.setattr(worker, "process_next_job", fake_process_next_job) + monkeypatch.setattr(time, "sleep", sleep_calls.append) + + with pytest.raises(RuntimeError, match="stop loop"): + worker.run_forever(poll_interval_seconds=0.25) + + assert sleep_calls == [0.25] + assert observed_calls == ["tick", "tick", "tick"] + + +def test_run_internal_worker_main_uses_run_forever(monkeypatch, tmp_path): + settings = WebAppSettings( + runtime_root=tmp_path / "runtime", + database_path=tmp_path / "runtime" / "jobs.db", + ) + observed = {} + fake_repository = object() + + class FakeWorkerLoop: + def __init__(self, coordinator, *, repository, inference_service, export_service, runtime_root): + observed["coordinator"] = coordinator + observed["repository"] = repository + observed["inference_service"] = inference_service + observed["export_service"] = export_service + observed["runtime_root"] = runtime_root + + def recover(self): + observed["recovered"] = True + + def run_forever(self, poll_interval_seconds=1.0): + observed["poll_interval_seconds"] = poll_interval_seconds + + class FakeJobRepository: + @staticmethod + def from_path(path): + observed["database_path"] = path + return fake_repository + + monkeypatch.setattr(run_internal_worker, "WebAppSettings", lambda: settings) + monkeypatch.setattr(run_internal_worker, "JobRepository", FakeJobRepository) + monkeypatch.setattr(run_internal_worker, "QueueCoordinator", lambda repo: ("queue", repo)) + monkeypatch.setattr(run_internal_worker, "InferenceService", lambda: "inference") + monkeypatch.setattr( + run_internal_worker, + "ExportService", + lambda enable_prores: ("export", enable_prores), + ) + monkeypatch.setattr(run_internal_worker, "WorkerLoop", FakeWorkerLoop) + + run_internal_worker.main() + + assert observed["database_path"] == settings.database_path + assert observed["repository"] is fake_repository + assert observed["recovered"] is True + assert observed["poll_interval_seconds"] == 1.0