Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 33 additions & 5 deletions lib/bob/runner.ex
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,13 @@ defmodule Bob.Runner do
@local_timeout 1_000
@remote_timeout 60_000

# A build that hangs (e.g. a wedged `docker build`) blocks its task forever and
# keeps counting its weight against the shared budget, so the agent stops
# pulling new builds. Reap a task that outlives the master's stale-job timeout
# (mirrors Bob.Queue.Maintenance @job_timeout_seconds) so the slot is freed and
# the job is reported failed.
@job_timeout 3 * 60 * 60 * 1000

def start_link([]) do
GenServer.start_link(__MODULE__, new_state(), name: __MODULE__)
end
Expand Down Expand Up @@ -46,7 +53,8 @@ defmodule Bob.Runner do
end

def handle_info({ref, result}, state) when is_reference(ref) do
{key, args, job_id, _pid} = Map.fetch!(state.tasks, ref)
{key, args, job_id, _pid, timer} = Map.fetch!(state.tasks, ref)
Process.cancel_timer(timer)

case result do
:ok ->
Expand Down Expand Up @@ -74,14 +82,33 @@ defmodule Bob.Runner do
{nil, _tasks} ->
{:noreply, state}

{{key, args, job_id, _pid}, tasks} ->
{{key, args, job_id, _pid, timer}, tasks} ->
Process.cancel_timer(timer)
if job_id, do: Bob.RemoteQueue.failure(job_id)
Logger.error("FAILED #{inspect(key)} #{inspect(args)} (#{inspect(reason)})")
state = start_any_jobs(%{state | tasks: tasks})
{:noreply, state}
end
end

# A task still running past @job_timeout is wedged (e.g. a hung `docker build`).
# Kill it so it stops leaking its weight, fail the job, and pull fresh work.
# The kill triggers an abnormal :DOWN, but the row is already gone so that
# clause no-ops.
def handle_info({:job_timeout, ref}, state) do
case Map.pop(state.tasks, ref) do
{nil, _tasks} ->
{:noreply, state}

{{key, args, job_id, pid, _timer}, tasks} ->
Task.Supervisor.terminate_child(Bob.Tasks, pid)
if job_id, do: Bob.RemoteQueue.failure(job_id)
Logger.error("TIMED OUT #{inspect(key)} #{inspect(args)}")
state = start_any_jobs(%{state | tasks: tasks})
{:noreply, state}
end
end

# Tasks are linked and exits are trapped; the :DOWN clauses above do the
# bookkeeping for both normal and abnormal task ends.
def handle_info({:EXIT, _pid, _reason}, state) do
Expand Down Expand Up @@ -136,7 +163,7 @@ defmodule Bob.Runner do
end

def terminate(_reason, state) do
Enum.each(state.tasks, fn {_ref, {key, args, job_id, pid}} ->
Enum.each(state.tasks, fn {_ref, {key, args, job_id, pid, _timer}} ->
# Kill the task before requeueing so a straggler completion report can't
# finish a row that another node may have re-claimed by then.
Task.Supervisor.terminate_child(Bob.Tasks, pid)
Expand All @@ -149,7 +176,7 @@ defmodule Bob.Runner do
end

defp current_weight(state) do
Enum.reduce(state.tasks, %{}, fn {_ref, {module, _args, _id, _pid}}, weights ->
Enum.reduce(state.tasks, %{}, fn {_ref, {module, _args, _id, _pid, _timer}}, weights ->
concurrency_key = apply_task(module, :concurrency, [])
weight = apply_task(module, :weight, [])
Map.update(weights, concurrency_key, weight, &(&1 + weight))
Expand All @@ -159,7 +186,8 @@ defmodule Bob.Runner do
defp start_job(id, key, args, state) do
Logger.info("STARTING #{inspect(key)} #{inspect(args)}")
task = Task.Supervisor.async(Bob.Tasks, fn -> run_task(key, args) end)
put_in(state.tasks[task.ref], {key, args, id, task.pid})
timer = Process.send_after(self(), {:job_timeout, task.ref}, @job_timeout)
put_in(state.tasks[task.ref], {key, args, id, task.pid, timer})
end

defp new_state do
Expand Down
39 changes: 35 additions & 4 deletions lib/bob/script.ex
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
defmodule Bob.Script do
@kill_after "30s"

def run(action, args, dir) do
File.mkdir_p!(dir)

Expand All @@ -24,13 +26,42 @@ defmodule Bob.Script do
end

defp exec({:cmd, cmd}, [], dir, log) do
Porcelain.shell(cmd, out: {:file, log}, err: :out, dir: dir, env: env())
Porcelain.shell(timeout_shell(cmd), out: {:file, log}, err: :out, dir: dir, env: env())
end

defp exec({:script, script}, args, dir, log) do
Path.join(script_dir(), script)
|> Path.expand()
|> Porcelain.exec(args, out: {:file, log}, err: :out, dir: dir, env: env())
script = Path.expand(Path.join(script_dir(), script))
{exe, exe_args} = timeout_exec(script, args)
Porcelain.exec(exe, exe_args, out: {:file, log}, err: :out, dir: dir, env: env())
end

# Bound every job so one that hangs anywhere — a stuck `docker build`,
# `docker push`, `wget`, etc. — is killed instead of pinning a build slot
# forever. GNU timeout run without --foreground signals the whole process
# group, so the script's children die with it; killing the `docker` client
# also makes the daemon cancel the build. Mirrors the runner's per-task
# backstop. When timeout is unavailable (non-Linux dev) the script runs
# unwrapped.
defp timeout_exec(script, args) do
case timeout_cmd() do
nil -> {script, args}
cmd -> {cmd, ["--kill-after=#{@kill_after}", timeout(), script | args]}
end
end

defp timeout_shell(cmd) do
case timeout_cmd() do
nil -> cmd
bin -> "#{bin} --kill-after=#{@kill_after} #{timeout()} #{cmd}"
end
end

defp timeout_cmd() do
System.find_executable("timeout") || System.find_executable("gtimeout")
end

defp timeout() do
Application.get_env(:bob, :script_timeout, "3h")
end

defp env() do
Expand Down
26 changes: 24 additions & 2 deletions test/bob/runner_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,12 @@ defmodule Bob.RunnerTest do
Bob.Queue.add(QuietJob, [:terminate_test])
{:ok, {id, [:terminate_test]}} = Bob.Queue.start(QuietJob)

state = %{tasks: %{task.ref => {QuietJob, [:terminate_test], {:local, id}, task.pid}}}
timer = Process.send_after(self(), :noop, 100_000)

state = %{
tasks: %{task.ref => {QuietJob, [:terminate_test], {:local, id}, task.pid, timer}}
}

Bob.Runner.terminate(:shutdown, state)

refute Process.alive?(task.pid)
Expand All @@ -93,14 +98,31 @@ defmodule Bob.RunnerTest do
{:ok, {id, [:down_test]}} = Bob.Queue.start(QuietJob)

ref = make_ref()
state = %{tasks: %{ref => {QuietJob, [:down_test], {:local, id}, self()}}}
timer = Process.send_after(self(), :noop, 100_000)
state = %{tasks: %{ref => {QuietJob, [:down_test], {:local, id}, self(), timer}}}

{:noreply, new_state} =
Bob.Runner.handle_info({:DOWN, ref, :process, self(), :killed}, state)

assert new_state.tasks == %{}
assert Bob.Repo.get!(Bob.Queue.Job, id).state == "failed"
end

test "a task that outlives its timeout is killed and the job failed" do
task = Task.Supervisor.async_nolink(Bob.Tasks, fn -> Process.sleep(:infinity) end)

Bob.Queue.add(QuietJob, [:timeout_test])
{:ok, {id, [:timeout_test]}} = Bob.Queue.start(QuietJob)

timer = Process.send_after(self(), :noop, 100_000)
state = %{tasks: %{task.ref => {QuietJob, [:timeout_test], {:local, id}, task.pid, timer}}}

{:noreply, new_state} = Bob.Runner.handle_info({:job_timeout, task.ref}, state)

refute Process.alive?(task.pid)
assert new_state.tasks == %{}
assert Bob.Repo.get!(Bob.Queue.Job, id).state == "failed"
end
end

describe "agent node (no repo)" do
Expand Down
10 changes: 10 additions & 0 deletions test/bob/script_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -19,5 +19,15 @@ defmodule Bob.ScriptTest do

assert String.trim(script_dir) == Application.app_dir(:bob, "priv/scripts")
end

@tag :timeout_binary
test "kills a job that exceeds the timeout" do
Application.put_env(:bob, :script_timeout, "1")
on_exit(fn -> Application.delete_env(:bob, :script_timeout) end)

assert_raise RuntimeError, ~r/returned: (124|137)/, fn ->
Script.run({:cmd, "sleep 30"}, [], System.tmp_dir!())
end
end
end
end
9 changes: 8 additions & 1 deletion test/test_helper.exs
Original file line number Diff line number Diff line change
@@ -1,2 +1,9 @@
ExUnit.start()
# Tests tagged :timeout_binary shell out to GNU `timeout`, which is absent on
# non-Linux dev machines; skip them there.
timeout_exclude =
if System.find_executable("timeout") || System.find_executable("gtimeout"),
do: [],
else: [:timeout_binary]

ExUnit.start(exclude: timeout_exclude)
Ecto.Adapters.SQL.Sandbox.mode(Bob.Repo, :manual)
Loading