diff --git a/lib/bob/runner.ex b/lib/bob/runner.ex index 4acacfc5..99c55af3 100644 --- a/lib/bob/runner.ex +++ b/lib/bob/runner.ex @@ -7,6 +7,13 @@ defmodule Bob.Runner do @local_timeout 1_000 @remote_timeout 60_000 + # A build that hangs (e.g. a wedged `docker build`) blocks its task forever and + # keeps counting its weight against the shared budget, so the agent stops + # pulling new builds. Reap a task that outlives the master's stale-job timeout + # (mirrors Bob.Queue.Maintenance @job_timeout_seconds) so the slot is freed and + # the job is reported failed. + @job_timeout 3 * 60 * 60 * 1000 + def start_link([]) do GenServer.start_link(__MODULE__, new_state(), name: __MODULE__) end @@ -46,7 +53,8 @@ defmodule Bob.Runner do end def handle_info({ref, result}, state) when is_reference(ref) do - {key, args, job_id, _pid} = Map.fetch!(state.tasks, ref) + {key, args, job_id, _pid, timer} = Map.fetch!(state.tasks, ref) + Process.cancel_timer(timer) case result do :ok -> @@ -74,7 +82,8 @@ defmodule Bob.Runner do {nil, _tasks} -> {:noreply, state} - {{key, args, job_id, _pid}, tasks} -> + {{key, args, job_id, _pid, timer}, tasks} -> + Process.cancel_timer(timer) if job_id, do: Bob.RemoteQueue.failure(job_id) Logger.error("FAILED #{inspect(key)} #{inspect(args)} (#{inspect(reason)})") state = start_any_jobs(%{state | tasks: tasks}) @@ -82,6 +91,24 @@ defmodule Bob.Runner do end end + # A task still running past @job_timeout is wedged (e.g. a hung `docker build`). + # Kill it so it stops leaking its weight, fail the job, and pull fresh work. + # The kill triggers an abnormal :DOWN, but the row is already gone so that + # clause no-ops. + def handle_info({:job_timeout, ref}, state) do + case Map.pop(state.tasks, ref) do + {nil, _tasks} -> + {:noreply, state} + + {{key, args, job_id, pid, _timer}, tasks} -> + Task.Supervisor.terminate_child(Bob.Tasks, pid) + if job_id, do: Bob.RemoteQueue.failure(job_id) + Logger.error("TIMED OUT #{inspect(key)} #{inspect(args)}") + state = start_any_jobs(%{state | tasks: tasks}) + {:noreply, state} + end + end + # Tasks are linked and exits are trapped; the :DOWN clauses above do the # bookkeeping for both normal and abnormal task ends. def handle_info({:EXIT, _pid, _reason}, state) do @@ -136,7 +163,7 @@ defmodule Bob.Runner do end def terminate(_reason, state) do - Enum.each(state.tasks, fn {_ref, {key, args, job_id, pid}} -> + Enum.each(state.tasks, fn {_ref, {key, args, job_id, pid, _timer}} -> # Kill the task before requeueing so a straggler completion report can't # finish a row that another node may have re-claimed by then. Task.Supervisor.terminate_child(Bob.Tasks, pid) @@ -149,7 +176,7 @@ defmodule Bob.Runner do end defp current_weight(state) do - Enum.reduce(state.tasks, %{}, fn {_ref, {module, _args, _id, _pid}}, weights -> + Enum.reduce(state.tasks, %{}, fn {_ref, {module, _args, _id, _pid, _timer}}, weights -> concurrency_key = apply_task(module, :concurrency, []) weight = apply_task(module, :weight, []) Map.update(weights, concurrency_key, weight, &(&1 + weight)) @@ -159,7 +186,8 @@ defmodule Bob.Runner do defp start_job(id, key, args, state) do Logger.info("STARTING #{inspect(key)} #{inspect(args)}") task = Task.Supervisor.async(Bob.Tasks, fn -> run_task(key, args) end) - put_in(state.tasks[task.ref], {key, args, id, task.pid}) + timer = Process.send_after(self(), {:job_timeout, task.ref}, @job_timeout) + put_in(state.tasks[task.ref], {key, args, id, task.pid, timer}) end defp new_state do diff --git a/lib/bob/script.ex b/lib/bob/script.ex index 9b83ef9f..f7b03f27 100644 --- a/lib/bob/script.ex +++ b/lib/bob/script.ex @@ -1,4 +1,6 @@ defmodule Bob.Script do + @kill_after "30s" + def run(action, args, dir) do File.mkdir_p!(dir) @@ -24,13 +26,42 @@ defmodule Bob.Script do end defp exec({:cmd, cmd}, [], dir, log) do - Porcelain.shell(cmd, out: {:file, log}, err: :out, dir: dir, env: env()) + Porcelain.shell(timeout_shell(cmd), out: {:file, log}, err: :out, dir: dir, env: env()) end defp exec({:script, script}, args, dir, log) do - Path.join(script_dir(), script) - |> Path.expand() - |> Porcelain.exec(args, out: {:file, log}, err: :out, dir: dir, env: env()) + script = Path.expand(Path.join(script_dir(), script)) + {exe, exe_args} = timeout_exec(script, args) + Porcelain.exec(exe, exe_args, out: {:file, log}, err: :out, dir: dir, env: env()) + end + + # Bound every job so one that hangs anywhere — a stuck `docker build`, + # `docker push`, `wget`, etc. — is killed instead of pinning a build slot + # forever. GNU timeout run without --foreground signals the whole process + # group, so the script's children die with it; killing the `docker` client + # also makes the daemon cancel the build. Mirrors the runner's per-task + # backstop. When timeout is unavailable (non-Linux dev) the script runs + # unwrapped. + defp timeout_exec(script, args) do + case timeout_cmd() do + nil -> {script, args} + cmd -> {cmd, ["--kill-after=#{@kill_after}", timeout(), script | args]} + end + end + + defp timeout_shell(cmd) do + case timeout_cmd() do + nil -> cmd + bin -> "#{bin} --kill-after=#{@kill_after} #{timeout()} #{cmd}" + end + end + + defp timeout_cmd() do + System.find_executable("timeout") || System.find_executable("gtimeout") + end + + defp timeout() do + Application.get_env(:bob, :script_timeout, "3h") end defp env() do diff --git a/test/bob/runner_test.exs b/test/bob/runner_test.exs index f8ab8b13..e62d13cb 100644 --- a/test/bob/runner_test.exs +++ b/test/bob/runner_test.exs @@ -81,7 +81,12 @@ defmodule Bob.RunnerTest do Bob.Queue.add(QuietJob, [:terminate_test]) {:ok, {id, [:terminate_test]}} = Bob.Queue.start(QuietJob) - state = %{tasks: %{task.ref => {QuietJob, [:terminate_test], {:local, id}, task.pid}}} + timer = Process.send_after(self(), :noop, 100_000) + + state = %{ + tasks: %{task.ref => {QuietJob, [:terminate_test], {:local, id}, task.pid, timer}} + } + Bob.Runner.terminate(:shutdown, state) refute Process.alive?(task.pid) @@ -93,7 +98,8 @@ defmodule Bob.RunnerTest do {:ok, {id, [:down_test]}} = Bob.Queue.start(QuietJob) ref = make_ref() - state = %{tasks: %{ref => {QuietJob, [:down_test], {:local, id}, self()}}} + timer = Process.send_after(self(), :noop, 100_000) + state = %{tasks: %{ref => {QuietJob, [:down_test], {:local, id}, self(), timer}}} {:noreply, new_state} = Bob.Runner.handle_info({:DOWN, ref, :process, self(), :killed}, state) @@ -101,6 +107,22 @@ defmodule Bob.RunnerTest do assert new_state.tasks == %{} assert Bob.Repo.get!(Bob.Queue.Job, id).state == "failed" end + + test "a task that outlives its timeout is killed and the job failed" do + task = Task.Supervisor.async_nolink(Bob.Tasks, fn -> Process.sleep(:infinity) end) + + Bob.Queue.add(QuietJob, [:timeout_test]) + {:ok, {id, [:timeout_test]}} = Bob.Queue.start(QuietJob) + + timer = Process.send_after(self(), :noop, 100_000) + state = %{tasks: %{task.ref => {QuietJob, [:timeout_test], {:local, id}, task.pid, timer}}} + + {:noreply, new_state} = Bob.Runner.handle_info({:job_timeout, task.ref}, state) + + refute Process.alive?(task.pid) + assert new_state.tasks == %{} + assert Bob.Repo.get!(Bob.Queue.Job, id).state == "failed" + end end describe "agent node (no repo)" do diff --git a/test/bob/script_test.exs b/test/bob/script_test.exs index 200a7f77..838d2f22 100644 --- a/test/bob/script_test.exs +++ b/test/bob/script_test.exs @@ -19,5 +19,15 @@ defmodule Bob.ScriptTest do assert String.trim(script_dir) == Application.app_dir(:bob, "priv/scripts") end + + @tag :timeout_binary + test "kills a job that exceeds the timeout" do + Application.put_env(:bob, :script_timeout, "1") + on_exit(fn -> Application.delete_env(:bob, :script_timeout) end) + + assert_raise RuntimeError, ~r/returned: (124|137)/, fn -> + Script.run({:cmd, "sleep 30"}, [], System.tmp_dir!()) + end + end end end diff --git a/test/test_helper.exs b/test/test_helper.exs index eeb8abf8..b163abaf 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,9 @@ -ExUnit.start() +# Tests tagged :timeout_binary shell out to GNU `timeout`, which is absent on +# non-Linux dev machines; skip them there. +timeout_exclude = + if System.find_executable("timeout") || System.find_executable("gtimeout"), + do: [], + else: [:timeout_binary] + +ExUnit.start(exclude: timeout_exclude) Ecto.Adapters.SQL.Sandbox.mode(Bob.Repo, :manual)