Merge pull request #468 from deepujain/fix/460-workflow-budget

feat: make workflow deadline configurable

SkillEvaluator should have a better timeout considering the bump to 600sec for skillspector this change

Author: mohgupta-ship-it

.env.example

@@ -5,6 +5,10 @@ ENV=dev                                                         # options: dev|s
 # Options: openai | anthropic | anthropic_proxy | nv_build
 SKILLSPECTOR_PROVIDER=
 
+# Aggregate deadline for one complete scan workflow. Defaults to 600 seconds;
+# override this positive finite value when a different limit is required.
+# SKILLSPECTOR_MAX_WORKFLOW_SECONDS=600
+
 # Provider credentials — set the one matching SKILLSPECTOR_PROVIDER (or
 # leave SKILLSPECTOR_PROVIDER unset and set NVIDIA_INFERENCE_KEY for the
 # default nv_build path).

docs/ANALYSIS_RESOURCE_BOUNDS.md

@@ -16,7 +16,7 @@ Values below are implementation defaults. MiB means 1,048,576 bytes.
 | Discovery time | 30 seconds | One skill bundle |
 | Canonical cached source bytes | 64 MiB | Ordinary files and expanded nested members combined |
 | Cached bytes from one filesystem artifact | 16 MiB | One artifact |
-| End-to-end workflow time | 60 seconds | One graph execution |
+| End-to-end workflow time | 600 seconds | One graph execution |
 | Cache and context-processing time | 60 seconds | One skill bundle, within the workflow deadline |
 
 The 64 MiB ceiling accounts for the canonical raw bytes retained for analysis. Local decoded text
@@ -130,7 +130,7 @@ work consume the same allowance.
 | External targets | 32 | One traversal |
 | Downloaded and cached source bytes | 10 MiB | Root and dependencies combined |
 | Discovered and expanded artifacts | 10,000 | Root and dependencies combined |
-| Traversal time | 60 seconds | Root and dependencies combined |
+| Traversal time | 600 seconds | Root and dependencies combined |
 | Reference source records | 1,024 | One extraction |
 | Reference source bytes | 1,000,000 | One extraction |
 | Raw reference candidates | 4,096 | One extraction |
@@ -211,3 +211,13 @@ When relevant analysis is incomplete:
 
 A low score or zero findings must not be interpreted as complete coverage when
 `analysis_completeness.is_complete` is false.
+
+
+## Configuring the aggregate workflow deadline
+
+The scanner limits one complete workflow to 600 seconds by default. Set
+`SKILLSPECTOR_MAX_WORKFLOW_SECONDS` to a positive finite number of seconds to
+change that aggregate deadline. The setting applies to direct,
+CLI, recursive, and multi-skill scans; byte and artifact ceilings remain in
+effect. Invalid, zero, negative, infinite, or NaN values safely keep the
+600-second default.

src/skillspector/cli.py

@@ -56,7 +56,7 @@
 from skillspector.multi_skill import MultiSkillDetectionResult, SkillDirectory, detect_skills
 from skillspector.nodes.report import report
 from skillspector.sarif_models import SARIF_SCHEMA_URI, validate_sarif_report
-from skillspector.state import MAX_WORKFLOW_BYTES
+from skillspector.state import MAX_WORKFLOW_BYTES, MAX_WORKFLOW_SECONDS
 from skillspector.suppression import (
     Baseline,
     build_baseline_dict,
@@ -100,7 +100,7 @@ def _ensure_utf8_streams() -> None:
 
 _TRANSITIVE_MAX_TARGETS = 32
 _TRANSITIVE_MAX_BYTES = 10 * 1024 * 1024
-_TRANSITIVE_MAX_SECONDS = 60.0
+_TRANSITIVE_MAX_SECONDS = MAX_WORKFLOW_SECONDS
 _TRANSITIVE_MAX_ARTIFACTS = 10_000
 _TRANSITIVE_MAX_FINDINGS = 10_000
 _TRANSITIVE_MAX_COMPONENTS = 10_000

src/skillspector/state.py

@@ -17,7 +17,9 @@
 
 from __future__ import annotations
 
+import math
 import operator
+import os
 from dataclasses import dataclass, field
 from time import monotonic
 from typing import Annotated, NotRequired
@@ -36,9 +38,40 @@
     LedgerRecordType,
     ledger_event,
 )
+from skillspector.logging_config import get_logger
 from skillspector.models import Finding
 
-MAX_WORKFLOW_SECONDS = 60.0
+logger = get_logger(__name__)
+
+DEFAULT_MAX_WORKFLOW_SECONDS = 600.0
+
+
+def _workflow_max_seconds_from_environment(value: str | None) -> float:
+    """Return a positive finite workflow deadline or the safe default."""
+    if value is None:
+        return DEFAULT_MAX_WORKFLOW_SECONDS
+    try:
+        seconds = float(value)
+    except ValueError:
+        logger.warning(
+            "SKILLSPECTOR_MAX_WORKFLOW_SECONDS=%r is not numeric, using default %.1fs",
+            value,
+            DEFAULT_MAX_WORKFLOW_SECONDS,
+        )
+        return DEFAULT_MAX_WORKFLOW_SECONDS
+    if not math.isfinite(seconds) or seconds <= 0:
+        logger.warning(
+            "SKILLSPECTOR_MAX_WORKFLOW_SECONDS=%r must be finite and positive, using default %.1fs",
+            value,
+            DEFAULT_MAX_WORKFLOW_SECONDS,
+        )
+        return DEFAULT_MAX_WORKFLOW_SECONDS
+    return seconds
+
+
+MAX_WORKFLOW_SECONDS = _workflow_max_seconds_from_environment(
+    os.environ.get("SKILLSPECTOR_MAX_WORKFLOW_SECONDS")
+)
 MAX_WORKFLOW_BYTES = 64 * 1024 * 1024
 MAX_WORKFLOW_ARTIFACTS = 10_000
 MAX_WORKFLOW_LIMITATION_RECORDS = 256

tests/nodes/test_build_context.py

@@ -38,11 +38,13 @@
 from skillspector.providers.openai import OpenAIProvider
 from skillspector.python_ast import ParsedPythonFile, get_python_ast
 from skillspector.state import (
+    DEFAULT_MAX_WORKFLOW_SECONDS,
     MAX_WORKFLOW_ARTIFACTS,
     MAX_WORKFLOW_BYTES,
     MAX_WORKFLOW_SECONDS,
     SkillspectorState,
     WorkflowResourceBudget,
+    _workflow_max_seconds_from_environment,
 )
 
 _OMS_FIXTURE = Path(__file__).parents[1] / "fixtures" / "oms" / "mcore-split-pr.skill.oms.sig"
@@ -180,14 +182,31 @@ def test_build_context_starts_and_returns_default_graph_wide_budget(tmp_path: Pa
 
     budget = result["workflow_resource_budget"]
     assert isinstance(budget, WorkflowResourceBudget)
-    assert budget.max_seconds == MAX_WORKFLOW_SECONDS == 60.0
+    assert DEFAULT_MAX_WORKFLOW_SECONDS == 600.0
+    assert budget.max_seconds == MAX_WORKFLOW_SECONDS
     assert budget.max_bytes == MAX_WORKFLOW_BYTES == 64 * 1024 * 1024
     assert budget.max_artifacts == MAX_WORKFLOW_ARTIFACTS == 10_000
     assert budget.started_at is not None
     assert budget.scanned_bytes == len(payload)
     assert budget.scanned_artifacts == 1
 
 
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        (None, DEFAULT_MAX_WORKFLOW_SECONDS),
+        ("120", 120.0),
+        ("0.5", 0.5),
+        ("0", DEFAULT_MAX_WORKFLOW_SECONDS),
+        ("-1", DEFAULT_MAX_WORKFLOW_SECONDS),
+        ("nan", DEFAULT_MAX_WORKFLOW_SECONDS),
+        ("not-a-number", DEFAULT_MAX_WORKFLOW_SECONDS),
+    ],
+)
+def test_workflow_budget_seconds_environment_parsing(value: str | None, expected: float) -> None:
+    assert _workflow_max_seconds_from_environment(value) == expected
+
+
 def test_build_context_reuses_supplied_stricter_transitive_budget(tmp_path: Path) -> None:
     from skillspector.cli import _TransitiveBudget, _TransitiveTraversalState
 

tests/unit/test_cli.py

@@ -1760,14 +1760,16 @@ def fake_run_graph_scan(
         show_suppressed: bool = False,
         transitive_traversal=None,
     ) -> dict[str, object]:
-        fake_time["value"] += 61.0
+        fake_time["value"] += cli._TRANSITIVE_MAX_SECONDS + 1.0
         return _mock_graph_result(file_cache={"SKILL.md": "https://github.com/org/dep.git"})
 
     def fake_scan_transitive(*args, traversal=None, **kwargs) -> dict[str, object]:
         assert traversal is not None
         assert traversal.remaining_seconds() == 0.0
         assert traversal.can_scan_more() is False
-        assert traversal.truncation_reasons == ["time budget 60s reached"]
+        assert traversal.truncation_reasons == [
+            f"time budget {cli._TRANSITIVE_MAX_SECONDS:g}s reached"
+        ]
         return {
             "report_body": "{}",
             "filtered_findings": [],