Skip to content

Commit cc1a9f3

Browse files
authored
Diff expected Docker tags in SQL instead of loading full tag history (#259)
DockerChecker previously pulled every tag of all six hexpm Docker repos out of the docker_tags mirror on each run - twelve full per-repo reads, about five million rows regex-parsed in the BEAM - which took ~15 minutes in production and held GB-scale working sets in memory. Each pass now computes its small expected set and asks Postgres only about that: the erlang and elixir passes probe presence with tag = ANY(expected) on the unique (repo, tag) index, the elixir pass scopes its erlang read to current base-image os_versions via the search metadata, and the manifest pass became a single anti-join that returns only mismatched tags (normally none). On a production-scale snapshot the data path drops from ~41s to ~7s locally; in production the win is larger since ~190k rows transfer instead of ~4.9M. A junk tag in a per-arch repo no longer crashes the whole run; the manifest pass now skips it with a log line. The GitHub refs fetch pages at per_page=100, cutting ~20 sequential requests to ~7.
1 parent 501b9e2 commit cc1a9f3

5 files changed

Lines changed: 339 additions & 154 deletions

File tree

‎lib/bob/artifacts.ex‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,73 @@ defmodule Bob.Artifacts do
388388
)
389389
end
390390

391+
@doc """
392+
The subset of `tags` that exist for `repo`, as a set.
393+
394+
The checker probes its (small) expected tag set with this instead of loading
395+
a repo's full tag history; the lookup runs on the unique `(repo, tag)` index.
396+
"""
397+
def docker_tags_present(_repo, []), do: MapSet.new()
398+
399+
def docker_tags_present(repo, tags) do
400+
Repo.all(
401+
from(d in DockerTag,
402+
where: d.repo == ^repo and d.tag in ^tags,
403+
select: d.tag
404+
),
405+
timeout: @long_query_timeout
406+
)
407+
|> MapSet.new()
408+
end
409+
410+
@doc """
411+
`{repo, tag}` for every tag of `repos` whose search metadata carries one of
412+
`os_versions`. Scopes the checker's erlang reads to current base images —
413+
a fraction of the full history.
414+
"""
415+
def erlang_tags_for_os_versions(repos, os_versions) do
416+
Repo.all(
417+
from(d in DockerTag,
418+
where:
419+
d.repo in ^repos and
420+
fragment("?->>'os_version' = ANY(?)", d.search, ^os_versions),
421+
select: {d.repo, d.tag}
422+
),
423+
timeout: @long_query_timeout
424+
)
425+
end
426+
427+
@doc """
428+
`{tag, archs}` for every per-arch tag whose manifest is missing or lacks one
429+
of the built archs. The whole diff runs in Postgres; only mismatches (normally
430+
none) reach the caller. Tags that exist only in the manifest repo are ignored,
431+
as is a manifest carrying more archs than were built.
432+
"""
433+
def manifest_mismatches(manifest_repo, amd64_repo, arm64_repo) do
434+
# The ::text[] cast is required: archs is varchar[] and array containment
435+
# does not unify varchar[] with the aggregated text[].
436+
%{rows: rows} =
437+
Repo.query!(
438+
"""
439+
SELECT p.tag, p.archs
440+
FROM (
441+
SELECT tag,
442+
array_agg(DISTINCT CASE WHEN repo = $2 THEN 'amd64' ELSE 'arm64' END) AS archs
443+
FROM docker_tags
444+
WHERE repo IN ($2, $3)
445+
GROUP BY tag
446+
) AS p
447+
LEFT JOIN docker_tags m ON m.repo = $1 AND m.tag = p.tag
448+
WHERE m.id IS NULL OR NOT (m.archs::text[] @> p.archs)
449+
ORDER BY p.tag COLLATE "C"
450+
""",
451+
[manifest_repo, amd64_repo, arm64_repo],
452+
timeout: @long_query_timeout
453+
)
454+
455+
Enum.map(rows, fn [tag, archs] -> {tag, archs} end)
456+
end
457+
391458
def base_image_tags(repo) do
392459
Repo.all(
393460
from(b in BaseImageTag,

‎lib/bob/github.ex‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ defmodule Bob.GitHub do
22
@github_url "https://api.github.com/"
33

44
def fetch_repo_refs(repo) do
5-
branches = github_request(@github_url <> "repos/#{repo}/branches")
6-
tags = github_request(@github_url <> "repos/#{repo}/tags")
5+
branches = github_request(@github_url <> "repos/#{repo}/branches?per_page=100")
6+
tags = github_request(@github_url <> "repos/#{repo}/tags?per_page=100")
77
response_to_refs(branches) ++ response_to_refs(tags)
88
end
99

‎lib/bob/job/docker_checker.ex‎

Lines changed: 75 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
defmodule Bob.Job.DockerChecker do
2+
require Logger
3+
24
@erlang_tag_regex ~r"^(.+)-(alpine|ubuntu|debian)-(.+)$"
35
@elixir_tag_regex ~r"^(.+)-erlang-(.+)-(alpine|ubuntu|debian)-(.+)$"
46

57
@archs ["amd64", "arm64"]
8+
@erlang_arch_repos Enum.map(@archs, &"hexpm/erlang-#{&1}")
69

710
def builds() do
811
[
@@ -64,13 +67,25 @@ defmodule Bob.Job.DockerChecker do
6467
def concurrency(), do: :shared
6568

6669
def erlang() do
67-
diff(expected_erlang_tags(), erlang_tags())
70+
expected_erlang_tags()
71+
|> Enum.group_by(fn {_erlang, _os, _os_version, arch} -> arch end)
72+
|> Enum.flat_map(fn {arch, expected} ->
73+
present =
74+
Bob.Artifacts.docker_tags_present(
75+
"hexpm/erlang-#{arch}",
76+
Enum.map(expected, &erlang_tag_name/1)
77+
)
78+
79+
Enum.reject(expected, &MapSet.member?(present, erlang_tag_name(&1)))
80+
end)
6881
|> Enum.map(fn {erlang, os, os_version, arch} ->
6982
{{Bob.Job.BuildDockerErlang, arch}, [erlang, os, os_version]}
7083
end)
7184
|> Bob.Queue.add_many()
7285
end
7386

87+
defp erlang_tag_name({erlang, os, os_version, _arch}), do: "#{erlang}-#{os}-#{os_version}"
88+
7489
def expected_erlang_tags() do
7590
refs = erlang_refs()
7691

@@ -254,32 +269,33 @@ defmodule Bob.Job.DockerChecker do
254269
{components, pre || []}
255270
end
256271

257-
def erlang_tags() do
258-
Enum.flat_map(@archs, &erlang_tags/1)
259-
end
260-
261-
def erlang_tags(arch) do
262-
"hexpm/erlang-#{arch}"
263-
|> Bob.Artifacts.docker_tags()
264-
|> Enum.map(fn {tag, [^arch]} ->
265-
[erlang, os, os_version] = Regex.run(@erlang_tag_regex, tag, capture: :all_but_first)
266-
{erlang, os, os_version, arch}
267-
end)
268-
end
269-
270272
def elixir() do
271-
diff(expected_elixir_tags(), elixir_tags())
273+
expected_elixir_tags()
274+
|> Enum.group_by(fn {_elixir, _erlang, _os, _os_version, arch} -> arch end)
275+
|> Enum.flat_map(fn {arch, expected} ->
276+
present =
277+
Bob.Artifacts.docker_tags_present(
278+
"hexpm/elixir-#{arch}",
279+
Enum.map(expected, &elixir_tag_name/1)
280+
)
281+
282+
Enum.reject(expected, &MapSet.member?(present, elixir_tag_name(&1)))
283+
end)
272284
|> Enum.map(fn {elixir, erlang, os, os_version, arch} ->
273285
{{Bob.Job.BuildDockerElixir, arch}, [elixir, erlang, os, os_version]}
274286
end)
275287
|> Bob.Queue.add_many()
276288
end
277289

290+
defp elixir_tag_name({elixir, erlang, os, os_version, _arch}) do
291+
"#{elixir}-erlang-#{erlang}-#{os}-#{os_version}"
292+
end
293+
278294
def expected_elixir_tags() do
279295
builds = builds()
280296
refs = elixir_builds()
281297

282-
Stream.flat_map(erlang_tags(), fn {erlang, os, os_version, erlang_arch} ->
298+
Stream.flat_map(current_erlang_tags(builds), fn {erlang, os, os_version, erlang_arch} ->
283299
if not skip_elixir_for_erlang?(erlang) and os_version in builds[os] do
284300
Stream.flat_map(refs, fn {"v" <> elixir, otp_major} ->
285301
if compatible_elixir_and_erlang?(otp_major, erlang) do
@@ -294,6 +310,20 @@ defmodule Bob.Job.DockerChecker do
294310
end)
295311
end
296312

313+
# Erlang per-arch tags scoped to the current base-image os_versions — the
314+
# only ones expected_elixir_tags/0 can use. Rows returned here always carry
315+
# search metadata, so the tag is guaranteed to match @erlang_tag_regex.
316+
defp current_erlang_tags(builds) do
317+
os_versions = builds |> Map.values() |> List.flatten()
318+
319+
@erlang_arch_repos
320+
|> Bob.Artifacts.erlang_tags_for_os_versions(os_versions)
321+
|> Enum.map(fn {"hexpm/erlang-" <> arch, tag} ->
322+
[erlang, os, os_version] = Regex.run(@erlang_tag_regex, tag, capture: :all_but_first)
323+
{erlang, os, os_version, arch}
324+
end)
325+
end
326+
297327
def elixir_builds() do
298328
"builds/elixir"
299329
|> Bob.Store.fetch_built_refs()
@@ -305,21 +335,6 @@ defmodule Bob.Job.DockerChecker do
305335
|> Enum.reject(fn {"v" <> elixir, _otp} -> skip_elixir?(elixir) end)
306336
end
307337

308-
def elixir_tags() do
309-
Stream.flat_map(@archs, &elixir_tags/1)
310-
end
311-
312-
def elixir_tags(arch) do
313-
"hexpm/elixir-#{arch}"
314-
|> Bob.Artifacts.docker_tags()
315-
|> Enum.map(fn {tag, [^arch]} ->
316-
[elixir, erlang, os, os_version] =
317-
Regex.run(@elixir_tag_regex, tag, capture: :all_but_first)
318-
319-
{elixir, erlang, os, os_version, arch}
320-
end)
321-
end
322-
323338
defp split_elixir_build(build_name) do
324339
case String.split(build_name, "-otp-") do
325340
[elixir, major_otp] -> {elixir, major_otp}
@@ -358,18 +373,6 @@ defmodule Bob.Job.DockerChecker do
358373
end
359374
end
360375

361-
def diff(expected, current) do
362-
current = MapSet.new(current)
363-
364-
Stream.flat_map(expected, fn key ->
365-
if MapSet.member?(current, key) do
366-
[]
367-
else
368-
[key]
369-
end
370-
end)
371-
end
372-
373376
defp compatible_elixir_and_erlang?(otp_major, erlang) do
374377
String.starts_with?(erlang, otp_major <> ".")
375378
end
@@ -386,53 +389,39 @@ defmodule Bob.Job.DockerChecker do
386389
end
387390

388391
def manifest() do
389-
erlang_tags = group_archs(erlang_tags())
390-
erlang_manifest_tags = erlang_manifest_tags()
391-
diff_manifests("erlang", erlang_tags, erlang_manifest_tags)
392-
393-
elixir_tags = group_archs(elixir_tags())
394-
elixir_manifest_tags = elixir_manifest_tags()
395-
diff_manifests("elixir", elixir_tags, elixir_manifest_tags)
396-
end
397-
398-
def erlang_manifest_tags() do
399-
"hexpm/erlang"
400-
|> Bob.Artifacts.docker_tags()
401-
|> Map.new(fn {tag, archs} ->
402-
[erlang, os, os_version] = Regex.run(@erlang_tag_regex, tag, capture: :all_but_first)
403-
{{erlang, os, os_version}, archs}
404-
end)
405-
end
406-
407-
def elixir_manifest_tags() do
408-
"hexpm/elixir"
409-
|> Bob.Artifacts.docker_tags()
410-
|> Map.new(fn {tag, archs} ->
411-
[elixir, erlang, os, os_version] =
412-
Regex.run(@elixir_tag_regex, tag, capture: :all_but_first)
413-
414-
{{elixir, erlang, os, os_version}, archs}
392+
check_manifests("erlang")
393+
check_manifests("elixir")
394+
end
395+
396+
# The whole per-arch vs manifest diff runs in Postgres; only mismatched tags
397+
# (normally none) come back, so just those few need parsing into job args.
398+
defp check_manifests(kind) do
399+
"hexpm/#{kind}"
400+
|> Bob.Artifacts.manifest_mismatches("hexpm/#{kind}-amd64", "hexpm/#{kind}-arm64")
401+
|> Enum.flat_map(fn {tag, _archs} ->
402+
case manifest_key(kind, tag) do
403+
{:ok, key} ->
404+
[{Bob.Job.DockerManifest, [kind, key]}]
405+
406+
:error ->
407+
Logger.error("DOCKER CHECKER skipping unparseable #{kind} tag #{inspect(tag)}")
408+
[]
409+
end
415410
end)
411+
|> Bob.Queue.add_many()
416412
end
417413

418-
def group_archs(enum) do
419-
Enum.group_by(
420-
enum,
421-
&Tuple.delete_at(&1, tuple_size(&1) - 1),
422-
&elem(&1, tuple_size(&1) - 1)
423-
)
414+
defp manifest_key("erlang", tag) do
415+
case Regex.run(@erlang_tag_regex, tag, capture: :all_but_first) do
416+
[erlang, os, os_version] -> {:ok, {erlang, os, os_version}}
417+
nil -> :error
418+
end
424419
end
425420

426-
def diff_manifests(kind, expected, current) do
427-
expected
428-
|> Enum.sort()
429-
|> Enum.flat_map(fn {key, expected_archs} ->
430-
if expected_archs -- Map.get(current, key, []) != [] do
431-
[{Bob.Job.DockerManifest, [kind, key]}]
432-
else
433-
[]
434-
end
435-
end)
436-
|> Bob.Queue.add_many()
421+
defp manifest_key("elixir", tag) do
422+
case Regex.run(@elixir_tag_regex, tag, capture: :all_but_first) do
423+
[elixir, erlang, os, os_version] -> {:ok, {elixir, erlang, os, os_version}}
424+
nil -> :error
425+
end
437426
end
438427
end

0 commit comments

Comments
 (0)