fix(aws-lambda): a network blip no longer stops the release ffmpeg build

packages/aws-lambda/image/Dockerfile

@@ -5,17 +5,19 @@ ARG BASE=public.ecr.aws/lambda/nodejs:22@sha256:d138eff445f6ea04c51091d8e6cbb064
 
 FROM ${BASE} AS build
 ENV SOURCE_DATE_EPOCH=1722643200
-RUN rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \
-    microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \
+COPY fetch.sh /fetch.sh
+RUN . /fetch.sh && rpm -qa --qf '%{NAME}\n' | sort >/build-base-rpms.txt && \
+    retry microdnf install -y --nodocs --setopt=install_weak_deps=0 --releasever=2023.12.20260914 \
       gcc gcc-c++ make cmake meson ninja-build autoconf automake libtool pkgconf-pkg-config diffutils \
       git xz bzip2 tar gzip which patchelf gnutls-devel zlib-devel $([ "$(uname -m)" = x86_64 ] && echo nasm) && \
     microdnf clean all && mkdir /build && mv /build-base-rpms.txt /build/base-rpms.txt
 COPY sources.txt build-ffmpeg.sh /build/
 RUN bash /build/build-ffmpeg.sh
 
 FROM ${BASE} AS certs
+COPY fetch.sh /fetch.sh
 # certtool, not openssl: the base's openssl-libs cannot run the repo's openssl CLI.
-RUN microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \
+RUN . /fetch.sh && retry microdnf install -y --nodocs --releasever=2023.12.20260914 gnutls-utils && mkdir /certs && cd /certs && \
     printf 'cn = check-ca\nca\ncert_signing_key\nexpiration_days = 2\n' >ca.tmpl && \
     printf 'cn = localhost\ndns_name = localhost\ntls_www_server\nexpiration_days = 2\n' >server.tmpl && \
     certtool -p --outfile ca.key && certtool -s --load-privkey ca.key --template ca.tmpl --outfile ca.crt && \

packages/aws-lambda/image/build-ffmpeg.sh

@@ -11,18 +11,12 @@ export PKG_CONFIG_PATH="$DEPS/lib/pkgconfig:$DEPS/lib64/pkgconfig"
 export CFLAGS="-O2 -fPIC" CXXFLAGS="-O2 -fPIC"
 mkdir -p "$SRC" "$DEPS" "$PREFIX/share/licenses" /out
 
-fetch() {
-  local name=$1 sum url
-  read -r sum url < <(awk -v n="$name" '$1 == n { print $2, $3 }' /build/sources.txt)
-  curl -fsSL --retry 3 -o "$SRC/$name" "$url"
-  echo "$sum  $SRC/$name" | sha256sum -c --quiet
-  tar -xf "$SRC/$name" -C "$SRC"
-}
+source /fetch.sh
 
 license() { mkdir -p "$PREFIX/share/licenses/$1" && cp "${@:2}" "$PREFIX/share/licenses/$1/"; }
 
 read -r x264_commit x264_url < <(awk '$1 == "x264" { print $2, $3 }' /build/sources.txt)
-git clone -q "$x264_url" "$SRC/x264"
+retry git clone -q "$x264_url" "$SRC/x264"
 git -C "$SRC/x264" checkout -q "$x264_commit"
 [ "$(git -C "$SRC/x264" rev-parse HEAD)" = "$x264_commit" ]
 (cd "$SRC/x264" && ./configure --prefix="$DEPS" --enable-static --enable-pic --disable-cli &&

packages/aws-lambda/image/fetch.sh

@@ -0,0 +1,19 @@
+# Source downloads for the ffmpeg build. One place for the retry policy: curl alone does not retry a reset connection.
+SOURCES=${SOURCES:-/build/sources.txt}
+
+retry() {
+  local n
+  for n in 1 2 3 4 5; do
+    "$@" && return 0
+    sleep $((n * 3))
+  done
+  return 1
+}
+
+fetch() {
+  local name=$1 sum url
+  read -r sum url <<<"$(awk -v n="$name" '$1 == n { print $2, $3 }' "$SOURCES")"
+  curl -fsSL --retry 5 --retry-all-errors --retry-delay 3 --connect-timeout 20 --max-time 600 -o "$SRC/$name" "$url"
+  echo "$sum  $SRC/$name" | sha256sum -c --quiet
+  tar -xf "$SRC/$name" -C "$SRC"
+}

packages/aws-lambda/image/fetch.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from "bun:test";
+import { createHash } from "node:crypto";
+import { mkdtempSync, readFileSync, writeFileSync } from "node:fs";
+import { createServer } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+const tarGz = (() => {
+  const dir = mkdtempSync(join(tmpdir(), "fetch-src-"));
+  writeFileSync(join(dir, "hello.txt"), "hi");
+  const out = join(dir, "pkg.tar.gz");
+  Bun.spawnSync(["tar", "-czf", out, "-C", dir, "hello.txt"]);
+  return readFileSync(out);
+})();
+
+async function fetchAfterResets(resets: number) {
+  let hits = 0;
+  const server = createServer((socket) => {
+    socket.once("data", () => {
+      if (++hits <= resets) return socket.resetAndDestroy();
+      socket.write(
+        `HTTP/1.1 200 OK\r\nContent-Length: ${tarGz.length}\r\nConnection: close\r\n\r\n`,
+      );
+      socket.end(tarGz);
+    });
+  }).listen(0, "127.0.0.1");
+  const { port } = server.address() as { port: number };
+  const dir = mkdtempSync(join(tmpdir(), "fetch-run-"));
+  const sum = createHash("sha256").update(tarGz).digest("hex");
+  writeFileSync(
+    join(dir, "sources.txt"),
+    `pkg.tar.gz ${sum} http://127.0.0.1:${port}/pkg.tar.gz\n`,
+  );
+  const proc = Bun.spawn(
+    ["bash", "-c", `set -euo pipefail; source ${import.meta.dir}/fetch.sh; fetch pkg.tar.gz`],
+    {
+      env: { ...process.env, SRC: dir, SOURCES: join(dir, "sources.txt") },
+      stdout: "pipe",
+      stderr: "pipe",
+    },
+  );
+  const code = await proc.exited;
+  server.close();
+  return { code, hits, extracted: readFileSync(join(dir, "hello.txt"), "utf8") };
+}
+
+describe("ffmpeg image fetch", () => {
+  it("retries a reset connection and still checks the sha256", async () => {
+    const r = await fetchAfterResets(2);
+    expect(r).toMatchObject({ code: 0, hits: 3, extracted: "hi" });
+  }, 30_000);
+});