diff --git a/scripts/notebook_tools/_papermill_meta.py b/scripts/notebook_tools/_papermill_meta.py index 62fb798c5d..3f1e385ef0 100644 --- a/scripts/notebook_tools/_papermill_meta.py +++ b/scripts/notebook_tools/_papermill_meta.py @@ -26,13 +26,28 @@ def strip_stale_papermill_metadata(nb): describe that pass (old dates, old duration) and would let a reviewer date the fresh outputs to the wrong run. An absent metadata is missing information; a stale one is misleading information (#11146). + + The same rationale applies per cell (#18305, suite #11146) : un executeur + qui reecrit un carnet touche les sorties de chaque cellule code mais laisse + les blocs ``cells[i].metadata.execution`` (``iopub.status.busy``/``iopub. + execute_input``) et ``cells[i].metadata.papermill`` (``start_time``, + ``duration``) dater des executions precedentes. On les retire aussi. """ metadata = nb.get("metadata") - if not metadata: - return - metadata.pop("papermill", None) - execution = metadata.get("execution") - if isinstance(execution, dict): - execution.pop("papermill", None) - if not execution: - metadata.pop("execution", None) + if metadata: + metadata.pop("papermill", None) + execution = metadata.get("execution") + if isinstance(execution, dict): + execution.pop("papermill", None) + if not execution: + metadata.pop("execution", None) + for cell in nb.get("cells", []) or []: + cell_meta = cell.get("metadata") + if not cell_meta: + continue + cell_meta.pop("papermill", None) + execution = cell_meta.get("execution") + if isinstance(execution, dict): + execution.pop("papermill", None) + if not execution: + cell_meta.pop("execution", None) diff --git a/scripts/notebook_tools/tests/test_strip_papermill_meta.py b/scripts/notebook_tools/tests/test_strip_papermill_meta.py new file mode 100644 index 0000000000..e43045ffc7 --- /dev/null +++ b/scripts/notebook_tools/tests/test_strip_papermill_meta.py @@ -0,0 +1,206 @@ +"""Tests for _papermill_meta.strip_stale_papermill_metadata — per-cell metadata sweep (#18305, suite #11146). + +Pins the contract on miniature notebooks: +- notebook-level metadata.papermill + metadata.execution.papermill are removed. +- per-cell metadata.papermill + metadata.execution.papermill are removed (new, #18305). +- empty per-cell metadata dicts are tolerated (not crashed). +- absent metadata (notebook or cell) is a no-op. +- markdown cells with metadata are also processed. +- non-papermill keys (eg `tags`, `deletable`) are preserved. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from _papermill_meta import strip_stale_papermill_metadata + + +PAPERMILL_NOTEBOOK = { + "start_time": "2026-08-20T12:27:00.000Z", + "end_time": "2026-08-20T12:27:11.000Z", + "duration": 11.0, + "input_path": "/tmp/old.ipynb", +} + + +def make_nb(cell_meta_specs=None, notebook_papermill=None, notebook_exec=None): + """Build a tiny notebook with per-cell metadata specs. + + cell_meta_specs: list of dicts (or None for no metadata on cell). + """ + cells = [] + for i, spec in enumerate(cell_meta_specs or []): + cell = { + "cell_type": "code", + "execution_count": 1, + "source": f"print({i})", + "outputs": [], + } + if spec is not None: + cell["metadata"] = dict(spec) + else: + cell["metadata"] = {} + cells.append(cell) + + nb = {"cells": cells, "metadata": {}, "nbformat": 4, "nbformat_minor": 5} + if notebook_papermill is not None: + nb["metadata"]["papermill"] = dict(notebook_papermill) + if notebook_exec is not None: + nb["metadata"]["execution"] = dict(notebook_exec) + return nb + + +def test_notebook_level_papermill_is_stripped(): + nb = make_nb(notebook_papermill=PAPERMILL_NOTEBOOK) + strip_stale_papermill_metadata(nb) + assert "papermill" not in nb["metadata"], ( + "notebook-level metadata.papermill should be removed" + ) + + +def test_notebook_level_execution_papermill_is_stripped(): + nb = make_nb( + notebook_exec={ + "iopub.execute_input": "2026-08-20T12:27:01Z", + "papermill": {"start_time": "2026-08-20T12:27:00Z"}, + } + ) + strip_stale_papermill_metadata(nb) + execution = nb["metadata"].get("execution") + assert execution is None or "papermill" not in execution, ( + "notebook-level metadata.execution.papermill should be removed " + "and the wrapper dropped if emptied" + ) + + +def test_cell_level_papermill_is_stripped(): + nb = make_nb( + cell_meta_specs=[ + {"papermill": {"start_time": "2026-08-20T12:27:00Z", "duration": 0.1}} + ] + ) + strip_stale_papermill_metadata(nb) + cell_meta = nb["cells"][0]["metadata"] + assert "papermill" not in cell_meta, ( + "cell-level metadata.papermill should be removed (#18305)" + ) + + +def test_cell_level_execution_papermill_is_stripped_and_wrapper_dropped(): + nb = make_nb( + cell_meta_specs=[ + { + "execution": { + "iopub.status.busy": "2026-08-20T12:27:00Z", + "papermill": {"start_time": "2026-08-20T12:27:00Z"}, + }, + "tags": ["keep-me"], + } + ] + ) + strip_stale_papermill_metadata(nb) + cell_meta = nb["cells"][0]["metadata"] + assert "papermill" not in cell_meta.get("execution", {}), ( + "cell-level metadata.execution.papermill should be removed" + ) + assert cell_meta.get("tags") == ["keep-me"], ( + "non-papermill metadata keys must be preserved" + ) + assert "execution" not in cell_meta or "papermill" not in cell_meta["execution"], ( + "empty execution wrapper should be dropped" + ) + + +def test_cell_level_only_execution_papermill_keeps_other_execution_keys(): + nb = make_nb( + cell_meta_specs=[ + { + "execution": { + "iopub.status.busy": "2026-08-20T12:27:00Z", + "iopub.execute_input": "2026-08-20T12:27:01Z", + "papermill": {"start_time": "2026-08-20T12:27:00Z"}, + } + } + ] + ) + strip_stale_papermill_metadata(nb) + execution = nb["cells"][0]["metadata"].get("execution") + assert execution is not None + assert execution.get("iopub.status.busy") == "2026-08-20T12:27:00Z" + assert execution.get("iopub.execute_input") == "2026-08-20T12:27:01Z" + assert "papermill" not in execution + + +def test_empty_cell_metadata_is_noop(): + nb = make_nb(cell_meta_specs=[{}, {}, {}]) + strip_stale_papermill_metadata(nb) + for cell in nb["cells"]: + assert cell["metadata"] == {} + + +def test_no_metadata_key_on_cell_is_noop(): + nb = { + "cells": [ + {"cell_type": "code", "execution_count": 1, "source": "x", "outputs": []} + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5, + } + strip_stale_papermill_metadata(nb) + assert nb["cells"][0] == { + "cell_type": "code", + "execution_count": 1, + "source": "x", + "outputs": [], + } + + +def test_no_notebook_metadata_is_noop(): + nb = {"cells": [], "nbformat": 4, "nbformat_minor": 5} + strip_stale_papermill_metadata(nb) + assert "metadata" not in nb + + +def test_markdown_cells_with_papermill_metadata_also_stripped(): + nb = make_nb( + cell_meta_specs=[ + {"papermill": {"start_time": "2026-08-20T12:27:00Z"}} + ] + ) + # rewrite first cell to markdown + nb["cells"][0]["cell_type"] = "markdown" + nb["cells"][0]["source"] = "# title" + nb["cells"][0].pop("execution_count", None) + nb["cells"][0].pop("outputs", None) + strip_stale_papermill_metadata(nb) + assert "papermill" not in nb["cells"][0]["metadata"] + + +def test_realistic_instance_min_mgs_02(): + """Pinned reproduction of the instance measured in #18305 : 11 cellules, + toutes portant un ``metadata.execution`` date du 2026-08-20, plus un + ``metadata.papermill`` par cellule. Apres strip, le carnet ne doit plus + dater les sorties d'un autre run. + """ + nb = make_nb(cell_meta_specs=[{} for _ in range(11)]) + for cell in nb["cells"]: + cell["metadata"] = { + "execution": { + "iopub.status.busy": "2026-08-20T12:27:00.123456Z", + "iopub.status.idle": "2026-08-20T12:27:00.456789Z", + "iopub.execute_input": "2026-08-20T12:27:00.234567Z", + "shell.execute_reply": "2026-08-20T12:27:00.345678Z", + "papermill": {"start_time": "2026-08-20T12:27:00Z", "duration": 0.1}, + }, + "papermill": {"start_time": "2026-08-20T12:27:00Z", "duration": 0.1}, + "tags": ["exercise"], + } + strip_stale_papermill_metadata(nb) + for i, cell in enumerate(nb["cells"]): + assert "papermill" not in cell["metadata"], f"cell {i}: papermill present" + execution = cell["metadata"].get("execution", {}) + assert "papermill" not in execution, f"cell {i}: execution.papermill present" + assert cell["metadata"].get("tags") == ["exercise"], f"cell {i}: tags lost"