Move kernelguard precheck to background (#499)

* Run KernelGuard precheck in background jobs

* Increase KernelGuard precheck timeout

---------

Co-authored-by: Sinatras <SinatrasC@users.noreply.github.com>

Author: SinatrasC

src/kernelbot/api/main.py

@@ -15,7 +15,6 @@
 from libkernelbot.background_submission_manager import BackgroundSubmissionManager
 from libkernelbot.consts import SubmissionMode
 from libkernelbot.db_types import IdentityType
-from libkernelbot.kernelguard import KernelGuardRejected, enforce_submission_precheck, should_precheck_submission
 from libkernelbot.leaderboard_db import LeaderboardDB, LeaderboardRankedEntry
 from libkernelbot.problem_sync import sync_problems
 from libkernelbot.submission import (
@@ -564,18 +563,6 @@ async def run_submission_async(
         if not req.gpus or len(req.gpus) != 1:
             raise HTTPException(status_code=400, detail="Invalid GPU type")
 
-        # run KernelGuard pre-check before enqueuing to avoid filling the queue with blocked submissions
-        if should_precheck_submission(submission_mode_enum):
-            try:
-                await asyncio.wait_for(
-                    asyncio.to_thread(enforce_submission_precheck, req.code, req.file_name),
-                    timeout=20.0,
-                )
-            except asyncio.TimeoutError as e:
-                raise HTTPException(status_code=504, detail="KernelGuard pre-check timed out") from e
-            except KernelGuardRejected as e:
-                raise HTTPException(status_code=400, detail=str(e)) from e
-
         # put submission request to background manager to run in background
         sub_id, job_status_id = await enqueue_background_job(
             req, submission_mode_enum, backend_instance, background_submission_manager

src/libkernelbot/background_submission_manager.py

@@ -234,7 +234,7 @@ async def heartbeat():
             reporter = BackgroundSubmissionManagerReporter()
             await asyncio.wait_for(
                 self.backend.submit_full(
-                    item.req, item.mode, reporter, sub_id, skip_precheck=True
+                    item.req, item.mode, reporter, sub_id, skip_precheck=False
                 ),
                 timeout=HARD_TIMEOUT_SEC,
             )

src/libkernelbot/kernelguard.py

@@ -11,7 +11,7 @@
 logger = setup_logging(__name__)
 
 _TRUE_VALUES = {"1", "true", "yes", "on"}
-_DEFAULT_TIMEOUT_SEC = 30
+_DEFAULT_TIMEOUT_SEC = 5 * 60
 _GUARDED_MODES = frozenset(
     {
         SubmissionMode.BENCHMARK,

tests/test_background_submission_manager.py

@@ -57,8 +57,11 @@ async def test_enqueue_and_run_job(mock_backend):
     )
     db_context.update_heartbeat_if_active = mock.Mock()
 
+    submit_calls = []
+
     # mock submit_full
     async def fake_submit_full(req, mode, reporter, sub_id, skip_precheck=False):
+        submit_calls.append((sub_id, skip_precheck))
         await asyncio.sleep(0.01)  # simulate a long-running job
         return None, None
 
@@ -90,6 +93,7 @@ async def fake_submit_full(req, mode, reporter, sub_id, skip_precheck=False):
         mock.call(42, status="succeeded", last_heartbeat=mock.ANY)
         in db_context.upsert_submission_job_status.call_args_list
     )
+    assert submit_calls == [(42, False)]
 
     await manager.stop()
 

tests/test_kernelguard.py

@@ -20,6 +20,16 @@ def test_should_precheck_submission_disabled(monkeypatch):
     assert not kernelguard.should_precheck_submission(SubmissionMode.BENCHMARK)
 
 
+def test_timeout_sec_defaults_to_five_minutes(monkeypatch):
+    monkeypatch.delenv("KERNELGUARD_TIMEOUT_SEC", raising=False)
+    assert kernelguard._timeout_sec() == 5 * 60
+
+
+def test_timeout_sec_can_be_overridden(monkeypatch):
+    monkeypatch.setenv("KERNELGUARD_TIMEOUT_SEC", "240")
+    assert kernelguard._timeout_sec() == 240
+
+
 def test_enforce_submission_precheck_rejects_filtered_code(monkeypatch):
     monkeypatch.setenv("KERNELGUARD_ENABLED", "1")
     monkeypatch.setattr(