Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions config/config.exs
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,13 @@ config :bob,
remote_jobs: [],
hexpm_impl: Bob.Hexpm.Impl

# Metrics are served by a standalone Bandit listener (see Bob.Application)
# on :metrics_port instead of PromEx's built-in cowboy server
config :bob, Bob.PromEx,
disabled: false,
metrics_server: :disabled,
grafana: :disabled

config :mime, :types, %{
"application/vnd.bob+erlang" => ["erlang"]
}
Expand Down
3 changes: 2 additions & 1 deletion config/dev.exs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ config :bob,
agent_schedule: [],
hexpm_url: "http://localhost:4000",
oauth_client_id: "b0b00000-0000-4000-8000-000000000b0b",
oauth_client_secret: "dev_secret_for_testing"
oauth_client_secret: "dev_secret_for_testing",
metrics_port: 9568

config :bob, Bob.Repo,
username: "postgres",
Expand Down
4 changes: 4 additions & 0 deletions config/runtime.exs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ if config_env() == :prod do
bob_hostname: System.fetch_env!("BOB_HOSTNAME")
}

if metrics_port = System.get_env("BOB_METRICS_PORT") do
config :bob, metrics_port: String.to_integer(metrics_port)
end

bob_host = System.fetch_env!("BOB_HOST")

config :bob, BobWeb.Endpoint,
Expand Down
2 changes: 2 additions & 0 deletions config/test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ config :bob,
oauth_client_id: "b0b00000-0000-4000-8000-000000000b0b",
oauth_client_secret: "test_secret"

config :bob, Bob.PromEx, disabled: true

config :ex_aws,
access_key_id: "test",
secret_access_key: "test",
Expand Down
13 changes: 12 additions & 1 deletion lib/bob/application.ex
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ defmodule Bob.Application do
children =
repo_children() ++
[
Bob.PromEx,
{Finch, name: Bob.Finch},
{Task.Supervisor, [name: Bob.Tasks]},
{Phoenix.PubSub, name: Bob.PubSub},
Expand All @@ -22,9 +23,12 @@ defmodule Bob.Application do
Bob.DockerHub.Auth,
runner_spec(),
{Bob.Schedule, [schedule()]},
BobWeb.Endpoint
BobWeb.Endpoint,
metrics_server_spec()
]

children = Enum.reject(children, &is_nil/1)

opts = [strategy: :one_for_one, name: Bob.Supervisor]
Supervisor.start_link(children, opts)
end
Expand All @@ -37,6 +41,13 @@ defmodule Bob.Application do
end
end

# Serves Prometheus metrics on a port separate from the main endpoint
defp metrics_server_spec() do
if port = Application.get_env(:bob, :metrics_port) do
{Bandit, plug: {PromEx.Plug, prom_ex_module: Bob.PromEx}, port: port}
end
end

if Mix.env() == :test do
defp maintenance_children(), do: []
else
Expand Down
42 changes: 42 additions & 0 deletions lib/bob/prom_ex.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
defmodule Bob.PromEx do
use PromEx, otp_app: :bob

alias PromEx.Plugins

@impl true
def plugins do
[
Plugins.Application,
Plugins.Beam,
{Plugins.Phoenix, router: BobWeb.Router, endpoint: BobWeb.Endpoint},
Bob.PromEx.Plugins.Bob
] ++ master_plugins()
end

# Agents run without Bob.Repo, so the Ecto plugin would crash them
defp master_plugins() do
if Application.get_env(:bob, :master?) do
[{Plugins.Ecto, repos: [Bob.Repo]}]
else
[]
end
end

@impl true
def dashboard_assigns do
[
datasource_id: "prometheus",
default_selected_interval: "5m"
]
end

@impl true
def dashboards do
[
{:prom_ex, "application.json"},
{:prom_ex, "beam.json"},
{:prom_ex, "phoenix.json"},
{:prom_ex, "ecto.json"}
]
end
end
138 changes: 138 additions & 0 deletions lib/bob/prom_ex/plugins/bob.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
defmodule Bob.PromEx.Plugins.Bob do
@moduledoc """
PromEx plugin for Bob's build queue and runner.

Event metrics are emitted by `Bob.Runner` on every node. Queue depth is
polled from Postgres, so that polling group only runs on the master.
"""

use PromEx.Plugin

@impl true
def event_metrics(_opts) do
Event.build(:bob_job_event_metrics, [
distribution("bob.job.run.duration.milliseconds",
event_name: [:bob, :job, :stop],
measurement: :duration,
unit: {:native, :millisecond},
description: "Build job duration.",
reporter_options: [
buckets: [
1_000,
5_000,
15_000,
60_000,
300_000,
900_000,
1_800_000,
3_600_000,
7_200_000
]
],
tags: [:job_type, :result],
tag_values: &job_stop_tag_values/1
),
counter("bob.job.timeout.total",
event_name: [:bob, :job, :timeout],
description: "Build jobs killed after exceeding the job timeout.",
tags: [:job_type],
tag_values: &job_tag_values/1
),
counter("bob.job.crash.total",
event_name: [:bob, :job, :crash],
description: "Build jobs that died from an uncaught throw or exit.",
tags: [:job_type],
tag_values: &job_tag_values/1
)
])
end

@impl true
def polling_metrics(opts) do
poll_rate = Keyword.get(opts, :poll_rate, 15_000)

[runner_metrics(poll_rate)] ++
if Application.get_env(:bob, :master?) do
[queue_metrics(poll_rate)]
else
[]
end
end

defp runner_metrics(poll_rate) do
Polling.build(
:bob_runner_polling_metrics,
poll_rate,
{__MODULE__, :execute_runner_metrics, []},
[
last_value("bob.runner.tasks.count",
event_name: [:bob, :runner, :tasks],
measurement: :count,
description: "Number of build jobs currently running on this node."
)
]
)
end

defp queue_metrics(poll_rate) do
Polling.build(
:bob_queue_polling_metrics,
poll_rate,
{__MODULE__, :execute_queue_metrics, []},
[
last_value("bob.queue.size.count",
event_name: [:bob, :queue, :size],
measurement: :count,
description: "Number of queued build jobs.",
tags: [:job_type],
tag_values: &job_tag_values/1
)
]
)
end

@doc false
def execute_runner_metrics() do
if GenServer.whereis(Bob.Runner) do
%{tasks: tasks} = Bob.Runner.state()
:telemetry.execute([:bob, :runner, :tasks], %{count: map_size(tasks)}, %{})
end

:ok
end

@doc false
def execute_queue_metrics() do
sizes = Map.new(Bob.Queue.queue_sizes())

known_keys =
Application.get_env(:bob, :local_jobs, []) ++ Application.get_env(:bob, :remote_jobs, [])

# Emit zeroes for configured but empty queues so gauges don't go stale
for key <- Enum.uniq(known_keys ++ Map.keys(sizes)) do
:telemetry.execute([:bob, :queue, :size], %{count: Map.get(sizes, key, 0)}, %{key: key})
end

:ok
rescue
# Polling must not crash while the Repo is unavailable (e.g. while it is
# restarting); ArgumentError covers the repo not yet being registered
DBConnection.ConnectionError -> :ok
ArgumentError -> :ok
end

defp job_stop_tag_values(%{key: key, result: result}) do
%{job_type: job_label(key), result: result}
end

defp job_tag_values(%{key: key}) do
%{job_type: job_label(key)}
end

@doc """
Formats a queue module key (`Bob.Job.BuildOTP` or `{Bob.Job.BuildOTP, "amd64"}`)
as a low-cardinality metric label like `"BuildOTP-amd64"`.
"""
def job_label({module, key}), do: "#{job_label(module)}-#{key}"
def job_label(module) when is_atom(module), do: module |> Module.split() |> List.last()
end
28 changes: 22 additions & 6 deletions lib/bob/runner.ex
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ defmodule Bob.Runner do
Process.cancel_timer(timer)
if job_id, do: Bob.RemoteQueue.failure(job_id)
Logger.error("FAILED #{inspect(key)} #{inspect(args)} (#{inspect(reason)})")
:telemetry.execute([:bob, :job, :crash], %{count: 1}, %{key: key})
state = start_any_jobs(%{state | tasks: tasks})
{:noreply, state}
end
Expand All @@ -104,6 +105,7 @@ defmodule Bob.Runner do
Task.Supervisor.terminate_child(Bob.Tasks, pid)
if job_id, do: Bob.RemoteQueue.failure(job_id)
Logger.error("TIMED OUT #{inspect(key)} #{inspect(args)}")
:telemetry.execute([:bob, :job, :timeout], %{count: 1}, %{key: key})
state = start_any_jobs(%{state | tasks: tasks})
{:noreply, state}
end
Expand Down Expand Up @@ -133,12 +135,26 @@ defmodule Bob.Runner do
end

defp run_task(key, args) do
{time, _} = :timer.tc(fn -> run_task_fun(key, args) end)
Logger.info("COMPLETED #{inspect(key)} #{inspect(args)} (#{time / 1_000_000}s)")
:ok
rescue
exception ->
{:error, exception, __STACKTRACE__}
start = System.monotonic_time()

try do
run_task_fun(key, args)
duration = System.monotonic_time() - start
time = System.convert_time_unit(duration, :native, :microsecond)
Logger.info("COMPLETED #{inspect(key)} #{inspect(args)} (#{time / 1_000_000}s)")
:telemetry.execute([:bob, :job, :stop], %{duration: duration}, %{key: key, result: "ok"})
:ok
rescue
exception ->
duration = System.monotonic_time() - start

:telemetry.execute([:bob, :job, :stop], %{duration: duration}, %{
key: key,
result: "error"
})

{:error, exception, __STACKTRACE__}
end
end

defp run_task_fun({module, key}, args), do: apply(module, :run, [key | args])
Expand Down
1 change: 1 addition & 0 deletions lib/bob_web/endpoint.ex
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ defmodule BobWeb.Endpoint do
end

plug(Plug.RequestId)
plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
plug(Logster.Plugs.Logger, excludes: [:params])
plug(BobWeb.Plugs.Secret, api_only: true)

Expand Down
1 change: 1 addition & 0 deletions mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ defmodule Bob.Mixfile do
{:phoenix_live_view, "~> 1.0"},
{:phoenix_live_reload, "~> 1.5", only: :dev},
{:bandit, "~> 1.5"},
{:prom_ex, "~> 1.11"},
{:esbuild, "~> 0.8", runtime: Mix.env() == :dev},
{:tailwind, "~> 0.2", runtime: Mix.env() == :dev},
{:igniter, "~> 0.8.1", only: [:dev, :test], runtime: false},
Expand Down
Loading
Loading