From 6633d1795607a94f29f4d621885dbf9d22ac4933 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Mon, 7 Sep 2026 02:03:14 -0400 Subject: [PATCH 01/17] fix(node): bound REST blob reads Resolve REST blob paths to immutable object IDs, enforce size and output ceilings, and retain admission through response delivery. Refs #204 --- .env.example | 6 +- README.md | 4 +- crates/gitlawb-node/src/api/repos.rs | 185 +++++++++-- crates/gitlawb-node/src/auth/mod.rs | 3 + crates/gitlawb-node/src/config.rs | 11 +- crates/gitlawb-node/src/error.rs | 18 + crates/gitlawb-node/src/git/store.rs | 308 +++++++++++++++++- .../gitlawb-node/src/git/visibility_pack.rs | 102 +++++- crates/gitlawb-node/src/main.rs | 3 + crates/gitlawb-node/src/state.rs | 20 +- crates/gitlawb-node/src/test_support.rs | 3 + 11 files changed, 601 insertions(+), 62 deletions(-) diff --git a/.env.example b/.env.example index 81c60824..f2e9deeb 100644 --- a/.env.example +++ b/.env.example @@ -139,8 +139,10 @@ GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # cost centers. Must be positive; set very large to effectively disable. Default 30. GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 -# Max concurrent git READ ops (upload-pack + the upload-pack info/refs -# advertisement) served at once, a global pool separate from the push pool below. +# Max concurrent git READ ops (upload-pack, its info/refs advertisement, and REST +# blob reads) served at once, a global pool separate from the push pool below. +# REST blobs also have a fixed four-response sub-pool and a 32 MiB per-response +# ceiling; their permits remain held until the response body finishes or disconnects. # The anon receive-pack info/refs advertisement has its OWN pool (see below), not # this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from # here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one diff --git a/README.md b/README.md index 3a092bf2..a3f64ae3 100644 --- a/README.md +++ b/README.md @@ -396,9 +396,9 @@ Important node settings: | `GITLAWB_ENFORCE_OWNER_PUSH` | Require the authenticated pusher to be the repo owner on `git-receive-pack`. **Defaults to `true`.** A `did:key` signature is authentication, not authorization — anyone can mint a key and sign — so with this off every signed caller may push to every repository, private ones included. Delegated and CI keys count as non-owners: a UCAN `git/push` capability is verified but not yet honored for authorization, so they cannot push while this is on. Set `false` only for a rolling upgrade; see [`docs/RUN-A-NODE.md`](docs/RUN-A-NODE.md). | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, `info/refs` advertisement, or REST blob read may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | -| `GITLAWB_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack and its `info/refs` advertisement) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. | +| `GITLAWB_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack, its `info/refs` advertisement, and REST blob reads) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. REST blob responses are limited to 32 MiB each and a dedicated four-request pool holds admission through body delivery, bounding retained blob data to 128 MiB. | | `GITLAWB_MAX_CONCURRENT_GIT_PUSHES` | Max concurrent `git-receive-pack` POST operations, in a pool separate from the read pool. The anon receive-pack `info/refs` advertisement runs in a third pool of the same size, disjoint from both, so an advertisement flood cannot shed a push either. Two per-source push caps are derived from this value (`/8`, floor 1) and have no env var of their own. Over-cap sheds a 503 + Retry-After. Default 32. | | `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER` | Max concurrent read ops a single caller may hold, so one caller cannot monopolize the read pool. Keyed on the resolved source IP, never the DID, and only as granular as `GITLAWB_TRUSTED_PROXY`: left unset, a node behind an edge or NAT keys every caller on the edge IP and this collapses to one global cap. Default 16. | | `GITLAWB_MAX_CONCURRENT_PIN_TASKS` | Max post-push pin loops (IPFS + Pinata) running concurrently across all repos. This caps how many loops RUN at once, not how much object-id list memory the node retains: on the local IPFS path a loop parked waiting for a permit still holds its full list. Do not size memory from this knob alone. A loop over cap waits, never drops a pin. Default 8. | diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4e327c42..4a8dffaa 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2,7 +2,7 @@ use axum::extract::{Extension, Path, Query, State}; use axum::http::StatusCode; use axum::response::Response; use axum::Json; -use bytes::Bytes; +use bytes::{Buf, Bytes}; use std::sync::Arc; use crate::auth::{caller_authorized_to_push, AuthenticatedDid}; @@ -21,6 +21,40 @@ use crate::webhooks; /// The git all-zeros object id — the create/delete sentinel in a ref update. const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; +/// REST blob responses share the same served-object ceiling as `/ipfs/{cid}`. +const MAX_SERVED_BLOB_BYTES: u64 = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + +/// A dedicated pool bounds retained REST blob bodies independently of smart-HTTP +/// traffic. At the per-response ceiling, the default caps live blob data at 128 MiB. +pub(crate) const MAX_CONCURRENT_BLOB_READS: usize = 4; + +const BLOB_RESPONSE_CHUNK_BYTES: usize = 64 * 1024; + +struct BlobResponseStream { + content: Bytes, + _git_permit: tokio::sync::OwnedSemaphorePermit, + _blob_permit: tokio::sync::OwnedSemaphorePermit, + _caller_permit: Option, +} + +impl futures::Stream for BlobResponseStream { + type Item = std::result::Result; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + if this.content.is_empty() { + return std::task::Poll::Ready(None); + } + let len = this.content.len().min(BLOB_RESPONSE_CHUNK_BYTES); + let chunk = Bytes::copy_from_slice(&this.content[..len]); + this.content.advance(len); + std::task::Poll::Ready(Some(Ok(chunk))) + } +} + /// The set of blob OIDs withheld from **anonymous** replication for a repo, or /// `None` when the repo must not replicate at all (private / mode A / /// undetermined — fail closed). This is the anonymous replication gate: @@ -413,47 +447,94 @@ pub async fn list_commits( pub async fn get_blob( State(state): State, Path((owner, name, file_path)): Path<(String, String, String)>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, auth: Option>, ) -> Result { use axum::http::header; - use axum::response::IntoResponse; // Unnormalized paths ("../..", "./", "//") can't resolve in `git show` // and crawlers combinatorially explode them from relative links — that's // a client error, not a 500. let file_path = file_path.trim_matches('/'); if file_path.is_empty() - || file_path - .split('/') - .any(|seg| seg.is_empty() || seg == "." || seg == "..") + || file_path.split('/').any(|seg| { + seg.is_empty() || seg == "." || seg == ".." || seg.chars().any(char::is_control) + }) { return Err(AppError::BadRequest("invalid file path".into())); } + if state.git_read_semaphore.available_permits() == 0 + || state.git_blob_semaphore.available_permits() == 0 + { + tracing::warn!( + "served-git concurrency cap reached; shedding blob request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let gate_path = format!("/{file_path}"); let (record, _rules) = crate::api::authorize_repo_read(&state, &owner, &name, caller, &gate_path).await?; - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; - let head_ref = store::resolve_head(&disk_path, &record.default_branch); - let content = store::read_file(&disk_path, &head_ref, file_path).map_err(|e| { - let msg = e.to_string(); - // `git show ref:path` on a path absent from the tree is a 404, - // not a server error - if msg.contains("does not exist in") - || msg.contains("invalid object name") - || msg.contains("exists on disk, but not in") - { - AppError::NotFound(format!("file not found: {file_path}")) - } else { - AppError::Git(msg) + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + &name, + "REST blob", + )?; + let blob_permit = git_permit(&state.git_blob_semaphore)?; + let permit = git_permit(&state.git_read_semaphore)?; + + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding blob request with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; + + let default_branch = record.default_branch.clone(); + let read_path = file_path.to_string(); + let git_bin = state.git_bin.clone(); + let deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let (read, permit, blob_permit, caller_permit) = tokio::task::spawn_blocking(move || { + let read = store::read_file_bounded( + &git_bin, + &disk_path, + &default_branch, + &read_path, + MAX_SERVED_BLOB_BYTES, + deadline, + ); + (read, permit, blob_permit, caller_permit) + }) + .await + .map_err(|e| AppError::Internal(anyhow::anyhow!("blob read task failed: {e}")))?; + let read = read.map_err(|e| git_service_app_error(&e))?; + + let content = match read { + store::BoundedFileRead::Found(content) => content, + store::BoundedFileRead::Missing => { + return Err(AppError::NotFound(format!("file not found: {file_path}"))); } - })?; + store::BoundedFileRead::TooLarge { size, max } => { + tracing::warn!(repo = %name, path = %file_path, size, max, "REST blob exceeds served size limit"); + return Err(AppError::PayloadTooLarge(format!( + "file exceeds the maximum served size of {max} bytes" + ))); + } + }; // Guess content type let mime = match file_path.rsplit('.').next() { @@ -467,7 +548,24 @@ pub async fn get_blob( _ => "application/octet-stream", }; - Ok(([(header::CONTENT_TYPE, mime)], content).into_response()) + let content_len = content.len(); + let stream = BlobResponseStream { + content: Bytes::from(content), + _git_permit: permit, + _blob_permit: blob_permit, + _caller_permit: caller_permit, + }; + let mut response = Response::new(axum::body::Body::from_stream(stream)); + response.headers_mut().insert( + header::CONTENT_TYPE, + axum::http::HeaderValue::from_static(mime), + ); + response.headers_mut().insert( + header::CONTENT_LENGTH, + axum::http::HeaderValue::from_str(&content_len.to_string()) + .expect("a decimal content length is a valid header value"), + ); + Ok(response) } /// GET /api/v1/repos/:owner/:repo/tree (root listing) @@ -3343,6 +3441,45 @@ mod tests { const OWNER_SHORT: &str = "z6MkpTHR8VNsBxYAAWHut2Geadd9jSwuBV8xRoAnwWsdvktH"; const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; + #[tokio::test] + async fn blob_response_holds_admission_until_the_body_is_dropped() { + use futures::StreamExt; + + let git = Arc::new(tokio::sync::Semaphore::new(1)); + let blob = Arc::new(tokio::sync::Semaphore::new(1)); + let callers = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let source = vec![b'x'; BLOB_RESPONSE_CHUNK_BYTES * 2]; + let source_start = source.as_ptr() as usize; + let source_end = source_start + source.len(); + let mut stream = BlobResponseStream { + content: Bytes::from(source), + _git_permit: git.clone().try_acquire_owned().unwrap(), + _blob_permit: blob.clone().try_acquire_owned().unwrap(), + _caller_permit: callers.try_acquire("source"), + }; + + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.len(), BLOB_RESPONSE_CHUNK_BYTES); + assert!(git.clone().try_acquire_owned().is_err()); + assert!(blob.clone().try_acquire_owned().is_err()); + assert!(callers.try_acquire("source").is_none()); + + let second = stream.next().await.unwrap().unwrap(); + assert!(stream.next().await.is_none()); + for chunk in [&first, &second] { + let chunk_start = chunk.as_ptr() as usize; + assert!( + chunk_start < source_start || chunk_start >= source_end, + "emitted chunks must not retain the full source allocation" + ); + } + + drop(stream); + assert!(git.try_acquire_owned().is_ok()); + assert!(blob.try_acquire_owned().is_ok()); + assert!(callers.try_acquire("source").is_some()); + } + #[test] fn upload_pack_request_finalizes_only_with_done_pktline() { let want = "0032want 1111111111111111111111111111111111111111\n"; diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 27b67786..337485df 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -532,6 +532,9 @@ mod tests { peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_blob_semaphore: Arc::new(tokio::sync::Semaphore::new( + crate::api::repos::MAX_CONCURRENT_BLOB_READS, + )), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 1fbf4376..5c12954a 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -314,8 +314,8 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK - /// `info/refs` advertisement only. The authenticated push POST draws from a + /// This is the READ pool (`git_read_semaphore`): upload-pack, the UPLOAD-PACK + /// `info/refs` advertisement, and REST blob reads. The authenticated push POST draws from a /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can /// never reach, and each read caller is additionally bounded by /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual @@ -422,9 +422,10 @@ pub struct Config { )] pub max_concurrent_pin_tasks: usize, - /// Maximum concurrent read operations (`upload-pack` and the upload-pack - /// `info/refs` advertisement) a single caller may hold at once, so one caller - /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// Maximum concurrent read operations (`upload-pack`, the upload-pack + /// `info/refs` advertisement, and REST blob reads) a single caller may hold at + /// once, so one caller cannot monopolize the `max_concurrent_git_ops` read pool + /// (#174). Callers are /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 474408e5..8b99ef0a 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -35,6 +35,9 @@ pub enum AppError { #[error("invalid request: {0}")] BadRequest(String), + #[error("payload too large: {0}")] + PayloadTooLarge(String), + /// A DID was well-formed enough to carry to a resolver but no verifying key /// could be derived from it. Its own code rather than plain `bad_request` /// because the auth middleware already answers `unresolvable_did` for the @@ -162,6 +165,11 @@ impl IntoResponse for AppError { // IcaptchaProofRequired is handled above (it carries extra headers/fields). AppError::IcaptchaProofRequired { .. } => unreachable!("handled before this match"), AppError::BadRequest(msg) => (StatusCode::BAD_REQUEST, "bad_request", msg.clone()), + AppError::PayloadTooLarge(msg) => ( + StatusCode::PAYLOAD_TOO_LARGE, + "payload_too_large", + msg.clone(), + ), AppError::UnresolvableDid(msg) => { (StatusCode::BAD_REQUEST, "unresolvable_did", msg.clone()) } @@ -274,6 +282,16 @@ mod tests { ); } + #[test] + fn payload_too_large_maps_to_413() { + assert_eq!( + AppError::PayloadTooLarge("x".into()) + .into_response() + .status(), + StatusCode::PAYLOAD_TOO_LARGE + ); + } + #[test] fn overloaded_maps_to_503_with_retry_after() { let resp = AppError::Overloaded("x".into()).into_response(); diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5617b419..ae9a478d 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -213,21 +213,174 @@ pub fn ls_tree(repo_path: &Path, refname: &str, tree_path: &str) -> Result Result> { +#[derive(Debug, PartialEq, Eq)] +pub enum BoundedFileRead { + Found(Vec), + Missing, + TooLarge { size: u64, max: u64 }, +} + +fn ref_resolves_bounded( + git_bin: &str, + repo_path: &Path, + refname: &str, + deadline: std::time::Instant, +) -> Result { + let commit = format!("{refname}^{{commit}}"); + let (status, _, _) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["rev-parse", "--verify", &commit], + repo_path, + &[], + deadline, + )?; + Ok(status.success()) +} + +/// Resolve the same ref preference as [`resolve_head`] without running an unbounded +/// child on an async worker. Every command shares the caller's deadline. +fn resolve_head_bounded( + git_bin: &str, + repo_path: &Path, + preferred_branch: &str, + deadline: std::time::Instant, +) -> Result { + if ref_resolves_bounded(git_bin, repo_path, "HEAD", deadline)? { + return Ok("HEAD".into()); + } + + let preferred = format!("refs/heads/{preferred_branch}"); + if ref_resolves_bounded(git_bin, repo_path, &preferred, deadline)? { + return Ok(preferred); + } + + for candidate in ["refs/heads/main", "refs/heads/master", "refs/heads/develop"] { + if candidate != preferred.as_str() + && ref_resolves_bounded(git_bin, repo_path, candidate, deadline)? + { + return Ok(candidate.into()); + } + } + + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ + "for-each-ref", + "--count=1", + "--format=%(refname)", + "refs/heads/", + ], + repo_path, + &[], + deadline, + )?; + let first = String::from_utf8(out).context("git returned a non-UTF-8 ref name")?; + let first = first.trim(); + Ok(if first.is_empty() { "HEAD" } else { first }.to_string()) +} + +/// Read one file for the REST blob endpoint without materializing unbounded child +/// output. The mutable ref is resolved to one immutable blob OID, its declared size is +/// checked before content capture, and the stdout drain retains at most `max_bytes`. +/// All Git children share `deadline`; async callers must invoke this in `spawn_blocking`. +pub fn read_file_bounded( + git_bin: &str, + repo_path: &Path, + preferred_branch: &str, + file_path: &str, + max_bytes: u64, + deadline: std::time::Instant, +) -> Result { + if file_path.contains('\r') || file_path.contains('\n') { + bail!("file path contains a line break"); + } + + let refname = resolve_head_bounded(git_bin, repo_path, preferred_branch, deadline)?; let spec = format!("{refname}:{file_path}"); - let output = Command::new("git") - .args(["show", &spec]) - .current_dir(repo_path) - .output() - .context("failed to run git show")?; + let stdin = format!("{spec}\n"); + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "--batch-check"], + repo_path, + stdin.as_bytes(), + deadline, + )?; + if !status.success() { + bail!( + "git cat-file --batch-check failed: {}", + String::from_utf8_lossy(&stderr) + ); + } - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git show failed: {stderr}"); + let stderr = String::from_utf8_lossy(&stderr); + if stderr + .lines() + .any(|line| line.starts_with("error:") || line.starts_with("fatal:")) + { + bail!("git cat-file --batch-check reported: {}", stderr.trim()); + } + + let output = String::from_utf8(stdout).context("git returned non-UTF-8 object metadata")?; + let mut lines = output.lines(); + let line = lines.next().unwrap_or_default().trim(); + if lines.next().is_some() { + bail!("git returned multiple object metadata records"); + } + if line.split_whitespace().last() == Some("missing") { + return Ok(BoundedFileRead::Missing); + } + + let mut parts = line.split_whitespace(); + let oid = parts.next().context("git omitted the blob object ID")?; + let kind = parts.next().context("git omitted the object type")?; + let size = parts + .next() + .context("git omitted the object size")? + .parse::() + .context("git returned an invalid object size")?; + if parts.next().is_some() + || !matches!(oid.len(), 40 | 64) + || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + bail!("git returned invalid object metadata"); + } + if kind != "blob" { + return Ok(BoundedFileRead::Missing); + } + if size > max_bytes { + return Ok(BoundedFileRead::TooLarge { + size, + max: max_bytes, + }); + } + + let max_stdout = usize::try_from(max_bytes).context("blob limit exceeds platform capacity")?; + let (status, content, stderr, exceeded) = + crate::git::visibility_pack::run_bounded_git_raw_capped( + git_bin, + &["cat-file", "blob", oid], + repo_path, + &[], + deadline, + max_stdout, + )?; + if !status.success() { + bail!( + "git cat-file blob failed: {}", + String::from_utf8_lossy(&stderr) + ); + } + if exceeded { + bail!("git emitted blob content beyond the served size limit"); + } + if content.len() as u64 != size { + bail!( + "git blob size changed between metadata and content reads (expected {size}, got {})", + content.len() + ); } - Ok(output.stdout) + Ok(BoundedFileRead::Found(content)) } #[derive(Debug, Clone, serde::Serialize)] @@ -882,6 +1035,139 @@ mod tests { use std::path::Path; use std::process::Command; + #[test] + fn bounded_file_read_rejects_packed_blob_and_preserves_allowed_content() { + let td = tempfile::TempDir::new().unwrap(); + let work = td.path(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(work) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let content = vec![b'a'; 64 * 1024]; + std::fs::write(work.join("large.txt"), &content).unwrap(); + run(&["add", "large.txt"]); + run(&["commit", "-qm", "add blob"]); + run(&["gc", "-q"]); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + assert_eq!( + super::read_file_bounded("git", work, "main", "large.txt", 1024, deadline).unwrap(), + super::BoundedFileRead::TooLarge { + size: content.len() as u64, + max: 1024, + } + ); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + assert_eq!( + super::read_file_bounded( + "git", + work, + "main", + "large.txt", + content.len() as u64, + deadline, + ) + .unwrap(), + super::BoundedFileRead::Found(content) + ); + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_does_not_capture_rejected_content() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let marker = td.path().join("content-called"); + let oid = "a".repeat(40); + let script = td.path().join("fakegit"); + std::fs::write( + &script, + format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 4096\"; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"blob\" ]; then touch \"{}\"; printf x; exit 0; fi\n\ + exit 1\n", + marker.display() + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&script, permissions).unwrap(); + + let result = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "large.txt", + 1024, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap(); + assert_eq!( + result, + super::BoundedFileRead::TooLarge { + size: 4096, + max: 1024, + } + ); + assert!( + !marker.exists(), + "content command must not run after an oversized metadata result" + ); + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_caps_content_that_exceeds_preflight_size() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let oid = "b".repeat(40); + let script = td.path().join("fakegit"); + std::fs::write( + &script, + format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 3\"; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"blob\" ] && [ \"$3\" = \"{oid}\" ]; then printf 0123456789abcdef0123456789abcdef; exit 0; fi\n\ + exit 1\n" + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&script, permissions).unwrap(); + + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "large.txt", + 16, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap_err(); + assert!( + err.to_string().contains("beyond the served size limit"), + "unexpected error: {err:#}" + ); + } + #[test] fn branch_diff_names_lists_changed_paths() { let td = tempfile::TempDir::new().unwrap(); diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 08666994..985184e9 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -29,6 +29,29 @@ const WALK_TIMEOUT: Duration = Duration::from_secs(600); #[cfg(unix)] const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); +/// Drain stdout completely while retaining no more than `limit` bytes. Continuing to +/// drain after the retained buffer is full prevents a child from deadlocking on its +/// stdout pipe without allowing its output to keep growing resident memory. +fn drain_stdout( + reader: &mut impl std::io::Read, + limit: Option, +) -> std::io::Result<(Vec, bool)> { + let mut out = Vec::new(); + let mut exceeded = false; + let mut chunk = [0_u8; 8192]; + loop { + let read = reader.read(&mut chunk)?; + if read == 0 { + return Ok((out, exceeded)); + } + let retained = limit + .map(|limit| read.min(limit.saturating_sub(out.len()))) + .unwrap_or(read); + out.extend_from_slice(&chunk[..retained]); + exceeded |= retained < read; + } +} + /// Run one git child under a shared `deadline` with process-group teardown, /// BLOCKING, and return its stdout. The child runs in its own process group; a /// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, @@ -67,13 +90,14 @@ fn child_terminated_without_reaping(pid: i32) -> bool { } #[cfg(unix)] -pub(crate) fn run_bounded_git_raw( +fn run_bounded_git_raw_with_limit( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result<(std::process::ExitStatus, Vec, Vec)> { + stdout_limit: Option, +) -> Result<(std::process::ExitStatus, Vec, Vec, bool)> { use std::io::{Read, Write}; use std::os::unix::process::CommandExt; use std::sync::mpsc::RecvTimeoutError; @@ -149,14 +173,13 @@ pub(crate) fn run_bounded_git_raw( err }); let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut out = Vec::new(); + let read_result = drain_stdout(&mut stdout, stdout_limit); // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block // until the kernel returns, pinning the walk thread and its permit. That residual // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. - let read_result = stdout.read_to_end(&mut out); // The drain has returned, but that only means all stdout write ends are closed — // NOT that the child has exited. A group member, or the leader itself, can close // stdout and keep running; standing the watchdog down on the drain alone (as the @@ -183,7 +206,7 @@ pub(crate) fn run_bounded_git_raw( let status = child.wait().context("git wait failed")?; let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - read_result.context("failed to read git stdout")?; + let (out, stdout_exceeded) = read_result.context("failed to read git stdout")?; // The watchdog runs off a wall clock that can race a child finishing right at the // deadline. A child that exited on its own (success) is not a timeout even if the // watchdog fired late; only a child that did not exit successfully is a genuine @@ -191,9 +214,46 @@ pub(crate) fn run_bounded_git_raw( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err, stdout_exceeded)) +} + +#[cfg(unix)] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + let (status, out, err, _) = + run_bounded_git_raw_with_limit(git_bin, args, repo_path, stdin_bytes, deadline, None)?; Ok((status, out, err)) } +/// Run a bounded git child while retaining at most `max_stdout_bytes` from stdout. +/// The pipe is still drained after the limit so the child cannot block on a full pipe; +/// `true` in the fourth tuple field reports that bytes were discarded. This is the +/// output-side companion to the process deadline for callers serving attacker-chosen +/// objects: a child that emits more than its preflight size cannot grow memory past the +/// caller's ceiling. +pub(crate) fn run_bounded_git_raw_capped( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, + max_stdout_bytes: usize, +) -> Result<(std::process::ExitStatus, Vec, Vec, bool)> { + run_bounded_git_raw_with_limit( + git_bin, + args, + repo_path, + stdin_bytes, + deadline, + Some(max_stdout_bytes), + ) +} + /// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin /// wrapper the walk callers use. Probes that must distinguish exit classes — /// `git cat-file` absence vs an object-store access failure — call @@ -227,13 +287,14 @@ pub(crate) fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -pub(crate) fn run_bounded_git_raw( +fn run_bounded_git_raw_with_limit( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result<(std::process::ExitStatus, Vec, Vec)> { + stdout_limit: Option, +) -> Result<(std::process::ExitStatus, Vec, Vec, bool)> { use std::io::{Read, Write}; use std::sync::mpsc::RecvTimeoutError; @@ -283,8 +344,7 @@ pub(crate) fn run_bounded_git_raw( }) }; - let mut out = Vec::new(); - let read_result = stdout.read_to_end(&mut out); + let read_result = drain_stdout(&mut stdout, stdout_limit); // The drain has returned (child exited or was killed), so taking the lock here // cannot deadlock against the watchdog. let status = child @@ -296,10 +356,23 @@ pub(crate) fn run_bounded_git_raw( let killed = watchdog.join().unwrap_or(false); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - read_result.context("failed to read git stdout")?; + let (out, stdout_exceeded) = read_result.context("failed to read git stdout")?; if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err, stdout_exceeded)) +} + +#[cfg(not(unix))] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + let (status, out, err, _) = + run_bounded_git_raw_with_limit(git_bin, args, repo_path, stdin_bytes, deadline, None)?; Ok((status, out, err)) } @@ -1225,6 +1298,15 @@ pub fn withheld_blob_recipients_bounded( mod tests { use super::*; + #[test] + fn stdout_drain_discards_bytes_past_the_retention_limit() { + let mut input = std::io::Cursor::new(b"0123456789".to_vec()); + let (out, exceeded) = drain_stdout(&mut input, Some(4)).unwrap(); + assert_eq!(out, b"0123"); + assert!(exceeded); + assert_eq!(input.position(), 10, "the pipe must still be fully drained"); + } + /// Write an executable fake `git` shell script into `dir` and return its path, /// so a test can drive the walk's process-group teardown without a real git and /// without mutating the process-global PATH (the crate's only injection seam). diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 66bfa096..bf85679e 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -444,6 +444,9 @@ async fn main() -> Result<()> { peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_blob_semaphore: Arc::new(tokio::sync::Semaphore::new( + crate::api::repos::MAX_CONCURRENT_BLOB_READS, + )), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 24607e5a..482f3dc5 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -194,14 +194,18 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, - /// Bounds concurrent served git READ operations (upload-pack and its own - /// `info/refs` advertisement ONLY — the receive-pack advertisement draws from - /// `git_push_advert_semaphore`, so sizing this pool for both would undercount - /// the read capacity an operator gets). A read handler acquires a permit before - /// spawning git and holds it for the op; when none are free the request is shed - /// with a 503. Writes draw from `git_write_semaphore` so a read flood cannot + /// Bounds concurrent served git READ operations (upload-pack, its own + /// `info/refs` advertisement, and REST blob reads — the receive-pack advertisement + /// draws from `git_push_advert_semaphore`, so sizing this pool for both would + /// undercount the read capacity an operator gets). A read handler acquires a permit + /// before spawning git and holds it for the op; when none are free the request is + /// shed with a 503. Writes draw from `git_write_semaphore` so a read flood cannot /// shed an authenticated push at admission (#174). pub git_read_semaphore: Arc, + /// Bounds REST blob bodies that are being read or delivered. The response stream + /// owns this permit until EOF or disconnect, so slow clients cannot recycle a Git + /// execution slot while retaining another full blob in memory. + pub git_blob_semaphore: Arc, /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate /// from `git_read_semaphore` so an anonymous READ flood can never shed an /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from @@ -279,8 +283,8 @@ pub struct AppState { /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` - /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` - /// advertisement. + /// (#174). Applied by `git_upload_pack`, the upload-pack `info/refs` + /// advertisement, and REST blob reads. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` /// advertisement: each source IP may hold at most a small share of the DEDICATED diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 430c0600..1e73bd2c 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -118,6 +118,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { shutdown_tx: tokio::sync::watch::channel(false).0, // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_blob_semaphore: Arc::new(tokio::sync::Semaphore::new( + crate::api::repos::MAX_CONCURRENT_BLOB_READS, + )), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), From 95d295cc57d784e0e5ac826107e4e9201fc38f42 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Mon, 7 Sep 2026 03:39:24 -0400 Subject: [PATCH 02/17] test(node): cover blob authorization denials. --- crates/gitlawb-node/src/test_support.rs | 65 +++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 1e73bd2c..5797b860 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -655,6 +655,71 @@ mod tests { ); } + /// A signed non-reader and an anonymous caller get the same opaque 404 for + /// a withheld blob path. The gate runs before repository acquisition, so + /// the test needs no on-disk repository and any leaked record detail is a + /// direct authorization regression. + #[sqlx::test] + async fn get_blob_denies_withheld_path_without_leaking_details(pool: PgPool) { + use crate::db::VisibilityMode; + + let owner = "did:key:zBLOBOWNERAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let stranger = "did:key:zBLOBSTRANGERBBBBBBBBBBBBBBBBBBBBBBBBBBB"; + let state = test_state(pool).await; + let mut repo = seed_repo(owner, "blob-repo"); + repo.description = Some("protected-description-marker".into()); + repo.default_branch = "protected-branch-marker".into(); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[], owner) + .await + .expect("set rule"); + + let router = || { + Router::new() + .route( + "/api/v1/repos/{owner}/{repo}/blob/{*path}", + axum::routing::get(crate::api::repos::get_blob), + ) + .with_state(state.clone()) + }; + let uri = format!("/api/v1/repos/{owner}/blob-repo/blob/secret/protected-file-marker.txt"); + + for (caller, request) in [ + ( + "authenticated non-reader", + signed_request_as(stranger, Method::GET, &uri, Body::empty()), + ), + ("anonymous caller", anon_get(&uri)), + ] { + let response = router().oneshot(request).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "{caller} must be denied as if the repository did not exist" + ); + let body = json_body(response).await; + assert_eq!(body["error"], "repo_not_found", "wrong denial for {caller}"); + assert_eq!( + body["message"], + format!("repository '{owner}/blob-repo' not found"), + "{caller} must receive the opaque repository denial" + ); + let body = body.to_string(); + for protected_detail in [ + "protected-file-marker", + "protected-description-marker", + "protected-branch-marker", + ] { + assert!( + !body.contains(protected_detail), + "{caller} denial leaked protected detail: {protected_detail}" + ); + } + } + } + fn seed_task(id: &str, delegator: &str) -> AgentTask { let now = Utc::now().to_rfc3339(); AgentTask { From d824cfac512a6876bc871a4f51e2895b057eae3c Mon Sep 17 00:00:00 2001 From: euxaristia Date: Tue, 8 Sep 2026 20:32:54 -0400 Subject: [PATCH 03/17] fix(node): Hide Git diagnostics from blob error responses. Refs #204 --- .env.example | 2 ++ crates/gitlawb-node/src/api/repos.rs | 40 +++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index f2e9deeb..9c5a2238 100644 --- a/.env.example +++ b/.env.example @@ -125,6 +125,7 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # rather than giving each stage a full budget: a walk that consumes it leaves the # serve nothing and the clone gets a 504. Serving large path-scoped repos may # need a higher value here than when each stage was budgeted separately. +# REST blob size probes and content reads also share this deadline and return 504 on timeout. # Must be 1..=3153600000 (100 years): the node derives deadlines from this value, # and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 @@ -184,6 +185,7 @@ GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 # collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a # high-fanout caller (CI behind one NAT) then needs the operator to raise this. # Default 16. +# REST blob downloads also acquire this per-caller read allowance. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 # Two further per-source concurrency caps exist on the PUSH side but have NO diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4a8dffaa..689c4511 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -521,7 +521,12 @@ pub async fn get_blob( }) .await .map_err(|e| AppError::Internal(anyhow::anyhow!("blob read task failed: {e}")))?; - let read = read.map_err(|e| git_service_app_error(&e))?; + let read = read.map_err(|e| { + if e.downcast_ref::().is_some() { + return AppError::Timeout("git service timed out".into()); + } + AppError::Internal(e) + })?; let content = match read { store::BoundedFileRead::Found(content) => content, @@ -3625,6 +3630,39 @@ mod tests { ); } + #[cfg(unix)] + #[sqlx::test] + async fn blob_cat_file_failure_is_opaque(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git( + tmp.path(), + "#!/bin/sh\necho 'private-git-stderr /internal/repo.git' >&2\nexit 1\n", + ); + let state = f4_state_with_repo(pool, tmp.path(), &fake, "z6bloberror", "repo", false).await; + let response = get_blob( + State(state), + Path(("z6bloberror".into(), "repo".into(), "file.txt".into())), + crate::rate_limit::PeerAddr(None), + axum::http::HeaderMap::new(), + None, + ) + .await + .unwrap_err() + .into_response(); + assert_eq!( + response.status(), + axum::http::StatusCode::INTERNAL_SERVER_ERROR + ); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["message"], crate::error::INTERNAL_ERROR_MESSAGE); + let body = body.to_string(); + assert!(!body.contains("private-git-stderr")); + assert!(!body.contains("/internal/repo.git")); + } + #[test] fn git_service_app_error_classifies_timeout_bad_request_and_git() { // GitServiceTimeout carried through anyhow -> 504 Timeout. From 3d7032b356f9f1e188588f0766576f83e32a10d3 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Wed, 9 Sep 2026 02:25:18 -0400 Subject: [PATCH 04/17] fix(node): close REST blob admission and error gaps Acquire the per-source read permit before repository visibility queries and keep storage acquisition failures opaque. Cover blob route size, capacity, timeout, and Git diagnostic responses, including permit release and source isolation. Refs #204 --- crates/gitlawb-node/src/api/repos.rs | 196 +++++++++++++++++++++++---- 1 file changed, 172 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 689c4511..9499b9c0 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -476,11 +476,8 @@ pub async fn get_blob( )); } - let caller = auth.as_ref().map(|e| e.0 .0.as_str()); - let gate_path = format!("/{file_path}"); - let (record, _rules) = - crate::api::authorize_repo_read(&state, &owner, &name, caller, &gate_path).await?; - + // Charge the source before visibility queries, retaining its permit through + // authorization, Git work, and response delivery. let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, @@ -488,6 +485,11 @@ pub async fn get_blob( &name, "REST blob", )?; + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); + let gate_path = format!("/{file_path}"); + let (record, _rules) = + crate::api::authorize_repo_read(&state, &owner, &name, caller, &gate_path).await?; + let blob_permit = git_permit(&state.git_blob_semaphore)?; let permit = git_permit(&state.git_read_semaphore)?; @@ -501,7 +503,7 @@ pub async fn get_blob( tracing::warn!(repo = %name, "repo acquire timed out; shedding blob request with 503"); AppError::Overloaded("git service acquisition timed out, retry shortly".into()) })? - .map_err(|e| AppError::Git(e.to_string()))?; + .map_err(AppError::Internal)?; let default_branch = record.default_branch.clone(); let read_path = file_path.to_string(); @@ -3633,24 +3635,66 @@ mod tests { #[cfg(unix)] #[sqlx::test] async fn blob_cat_file_failure_is_opaque(pool: sqlx::PgPool) { - use axum::response::IntoResponse; use http_body_util::BodyExt; let tmp = tempfile::TempDir::new().unwrap(); - let fake = write_fake_git( - tmp.path(), - "#!/bin/sh\necho 'private-git-stderr /internal/repo.git' >&2\nexit 1\n", + for (index, failure) in [ + "echo 'private-git-stderr /internal/repo.git' >&2; exit 1", + "echo 'error: private-git-stderr /internal/repo.git' >&2; exit 0", + "if [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 blob 1'; exit 0; fi\necho 'private-git-stderr /internal/repo.git' >&2; exit 1", + ].iter().enumerate() { + let fake = write_fake_git(tmp.path(), &format!( + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\n{failure}\n" + )); + let owner = format!("z6bloberror{index}"); + let state = f4_state_with_repo(pool.clone(), tmp.path(), &fake, &owner, "repo", false).await; + let response = blob_route_request(state, &owner, "203.0.113.31:5000").await; + assert_eq!(response.status(), axum::http::StatusCode::INTERNAL_SERVER_ERROR); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["message"], crate::error::INTERNAL_ERROR_MESSAGE); + let body = body.to_string(); + assert!(!body.contains("private-git-stderr")); + assert!(!body.contains("/internal/repo.git")); + } + } + + async fn blob_route_request(state: AppState, owner: &str, peer: &str) -> Response { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::Request; + use tower::ServiceExt; + let router = axum::Router::new() + .route( + "/repos/{owner}/{repo}/blob/{*path}", + axum::routing::get(get_blob), + ) + .with_state(state); + let mut request = Request::builder() + .uri(format!("/repos/{owner}/repo/blob/file.txt")) + .body(Body::empty()) + .unwrap(); + request + .extensions_mut() + .insert(ConnectInfo(peer.parse::().unwrap())); + router.oneshot(request).await.unwrap() + } + + #[sqlx::test] + async fn blob_acquire_failure_is_opaque(pool: sqlx::PgPool) { + use http_body_util::BodyExt; + let mut state = crate::test_support::test_state(pool.clone()).await; + state + .db + .upsert_mirror_repo("z6blobacquire", "repo", "/unused", None, false) + .await + .unwrap(); + // An invalid configured storage root makes the real acquire() fail before Git. + let tmp = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + tmp.path().join("private-storage").join("..").join("repos"), + pool, ); - let state = f4_state_with_repo(pool, tmp.path(), &fake, "z6bloberror", "repo", false).await; - let response = get_blob( - State(state), - Path(("z6bloberror".into(), "repo".into(), "file.txt".into())), - crate::rate_limit::PeerAddr(None), - axum::http::HeaderMap::new(), - None, - ) - .await - .unwrap_err() - .into_response(); + let response = blob_route_request(state, "z6blobacquire", "203.0.113.31:5000").await; assert_eq!( response.status(), axum::http::StatusCode::INTERNAL_SERVER_ERROR @@ -3658,9 +3702,113 @@ mod tests { let bytes = response.into_body().collect().await.unwrap().to_bytes(); let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(body["message"], crate::error::INTERNAL_ERROR_MESSAGE); - let body = body.to_string(); - assert!(!body.contains("private-git-stderr")); - assert!(!body.contains("/internal/repo.git")); + assert!(!body.to_string().contains("parent-directory")); + } + + #[tokio::test] + async fn blob_capacity_sheds_before_database_access() { + use axum::http::StatusCode; + use http_body_util::BodyExt; + for capacity in ["caller", "blob", "global"] { + let mut state = crate::test_support::test_state_lazy(); + // A misplaced DB lookup returns db_unavailable immediately, without a timeout. + state.db.pool().close().await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let caller_slot = (capacity == "caller").then(|| { + state + .git_read_per_caller + .try_acquire("203.0.113.31") + .unwrap() + }); + if capacity == "blob" { + state.git_blob_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(0)); + } + if capacity == "global" { + state.git_read_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(0)); + } + let response = + blob_route_request(state.clone(), "z6blobcap", "203.0.113.31:5000").await; + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{capacity}" + ); + assert_eq!(response.headers()[axum::http::header::RETRY_AFTER], "1"); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "overloaded", "{capacity}"); + if capacity == "caller" { + let other = + blob_route_request(state.clone(), "z6blobcap", "203.0.113.32:5000").await; + assert_eq!(other.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = other.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], crate::error::DB_UNAVAILABLE_CODE); + drop(caller_slot); + let released = blob_route_request(state, "z6blobcap", "203.0.113.31:5000").await; + assert_eq!(released.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = released.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], crate::error::DB_UNAVAILABLE_CODE); + } + } + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_maps_oversize_and_timeout(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + for (owner, command, expected) in [ + ( + "z6blobsize", + format!( + "echo '1111111111111111111111111111111111111111 blob {}'", + MAX_SERVED_BLOB_BYTES + 1 + ), + StatusCode::PAYLOAD_TOO_LARGE, + ), + ( + "z6blobtime", + "exec sleep 30".into(), + StatusCode::GATEWAY_TIMEOUT, + ), + ] { + let fake = write_fake_git( + tmp.path(), + &format!("#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\n{command}\n"), + ); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &fake, owner, "repo", false).await; + let mut config = (*state.config).clone(); + config.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(config); + let response = tokio::time::timeout( + std::time::Duration::from_secs(10), + blob_route_request(state.clone(), owner, "203.0.113.31:5000"), + ) + .await + .expect("blob deadline must terminate the Git child"); + assert_eq!(response.status(), expected); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"], + if expected == StatusCode::PAYLOAD_TOO_LARGE { + "payload_too_large" + } else { + "git_timeout" + } + ); + assert!(state.git_read_semaphore.available_permits() > 0); + assert!(state.git_blob_semaphore.available_permits() > 0); + assert!(state + .git_read_per_caller + .try_acquire("203.0.113.31") + .is_some()); + } } #[test] From fc2d043b5e6815312c94cdee50f8c8b0f4cf4029 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Thu, 10 Sep 2026 17:30:08 -0400 Subject: [PATCH 05/17] fix(node): acquire blob permits pre-DB and cover route/fallback arms Acquire per-caller, blob, and global read permits atomically prior to visibility database queries in get_blob. Add handler-level route tests for 400, 404, and 200 blob responses. Cover all fallback arms of resolve_head_bounded with an unborn HEAD. Refs #407 --- crates/gitlawb-node/src/api/repos.rs | 104 ++++++++++++++++++++++----- crates/gitlawb-node/src/git/store.rs | 86 ++++++++++++++++++++++ 2 files changed, 172 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 9499b9c0..7ac740ad 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -465,19 +465,8 @@ pub async fn get_blob( return Err(AppError::BadRequest("invalid file path".into())); } - if state.git_read_semaphore.available_permits() == 0 - || state.git_blob_semaphore.available_permits() == 0 - { - tracing::warn!( - "served-git concurrency cap reached; shedding blob request with 503 (pre-DB)" - ); - return Err(AppError::Overloaded( - "git service at capacity, retry shortly".into(), - )); - } - - // Charge the source before visibility queries, retaining its permit through - // authorization, Git work, and response delivery. + // Charge the source and acquire blob/global read permits before any database work. + // The OwnedSemaphorePermits release on drop if authorization or subsequent work fails. let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); let caller_permit = acquire_read_caller_permit( &state.git_read_per_caller, @@ -485,14 +474,14 @@ pub async fn get_blob( &name, "REST blob", )?; + let blob_permit = git_permit(&state.git_blob_semaphore)?; + let permit = git_permit(&state.git_read_semaphore)?; + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let gate_path = format!("/{file_path}"); let (record, _rules) = crate::api::authorize_repo_read(&state, &owner, &name, caller, &gate_path).await?; - let blob_permit = git_permit(&state.git_blob_semaphore)?; - let permit = git_permit(&state.git_read_semaphore)?; - let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); let disk_path = tokio::time::timeout( acquire_deadline, @@ -3658,7 +3647,12 @@ mod tests { } } - async fn blob_route_request(state: AppState, owner: &str, peer: &str) -> Response { + async fn blob_route_request_path( + state: AppState, + owner: &str, + path: &str, + peer: &str, + ) -> Response { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::Request; @@ -3670,7 +3664,7 @@ mod tests { ) .with_state(state); let mut request = Request::builder() - .uri(format!("/repos/{owner}/repo/blob/file.txt")) + .uri(format!("/repos/{owner}/repo/blob/{path}")) .body(Body::empty()) .unwrap(); request @@ -3679,6 +3673,10 @@ mod tests { router.oneshot(request).await.unwrap() } + async fn blob_route_request(state: AppState, owner: &str, peer: &str) -> Response { + blob_route_request_path(state, owner, "file.txt", peer).await + } + #[sqlx::test] async fn blob_acquire_failure_is_opaque(pool: sqlx::PgPool) { use http_body_util::BodyExt; @@ -3811,6 +3809,76 @@ mod tests { } } + #[tokio::test] + async fn blob_route_rejects_invalid_paths() { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let state = crate::test_support::test_state_lazy(); + for invalid_path in [ + "file%01.txt", + "file%1f.txt", + "foo/%2e%2e/bar.txt", + "foo/%2e/bar.txt", + "foo//bar.txt", + ] { + let response = + blob_route_request_path(state.clone(), "z6test", invalid_path, "203.0.113.31:5000") + .await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "path: {invalid_path:?}" + ); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "bad_request"); + } + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_returns_not_found_on_missing_blob(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git( + tmp.path(), + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo 'missing'; exit 0; fi\nexit 1\n", + ); + let state = + f4_state_with_repo(pool, tmp.path(), &fake, "z6blobmissing", "repo", false).await; + let response = + blob_route_request_path(state, "z6blobmissing", "missing.txt", "203.0.113.31:5000") + .await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "not_found"); + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_returns_200_with_expected_headers_and_content(pool: sqlx::PgPool) { + use axum::http::{header, StatusCode}; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git( + tmp.path(), + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 blob 13'; exit 0; fi\nprintf '{\"hello\":123}'; exit 0\n", + ); + let state = f4_state_with_repo(pool, tmp.path(), &fake, "z6blobhappy", "repo", false).await; + let response = + blob_route_request_path(state, "z6blobhappy", "data.json", "203.0.113.31:5000").await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/json; charset=utf-8" + ); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "13"); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&bytes[..], b"{\"hello\":123}"); + } + #[test] fn git_service_app_error_classifies_timeout_bad_request_and_git() { // GitServiceTimeout carried through anyhow -> 504 Timeout. diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index ae9a478d..13541790 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -1168,6 +1168,92 @@ mod tests { ); } + #[test] + fn resolve_head_bounded_covers_all_fallback_arms() { + let td = tempfile::TempDir::new().unwrap(); + let work = td.path(); + let run = |args: &[&str]| { + let status = std::process::Command::new("git") + .args(args) + .current_dir(work) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + std::fs::write(work.join("hello.txt"), b"hello").unwrap(); + run(&["add", "hello.txt"]); + run(&["commit", "-qm", "initial"]); + run(&["branch", "-M", "custom-feature"]); + // Detach or point HEAD to an unborn branch so HEAD itself does not resolve. + run(&["symbolic-ref", "HEAD", "refs/heads/unborn-branch"]); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // 1. Preferred branch arm: "custom-feature" resolves to refs/heads/custom-feature. + let resolved = + super::resolve_head_bounded("git", work, "custom-feature", deadline).unwrap(); + assert_eq!(resolved, "refs/heads/custom-feature"); + let read = + super::read_file_bounded("git", work, "custom-feature", "hello.txt", 1024, deadline) + .unwrap(); + assert_eq!(read, super::BoundedFileRead::Found(b"hello".to_vec())); + + // 2. Candidate branch arm (main, master, develop): create "master". + run(&["branch", "-M", "custom-feature", "master"]); + // Asking for a nonexistent preferred branch falls back to "master". + let resolved_master = + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(); + assert_eq!(resolved_master, "refs/heads/master"); + let read_master = + super::read_file_bounded("git", work, "nonexistent", "hello.txt", 1024, deadline) + .unwrap(); + assert_eq!( + read_master, + super::BoundedFileRead::Found(b"hello".to_vec()) + ); + + // 3. for-each-ref fallback arm: rename "master" to "isolated-branch" (neither main, master, nor develop). + run(&["branch", "-M", "master", "isolated-branch"]); + let resolved_isolated = + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(); + assert_eq!(resolved_isolated, "refs/heads/isolated-branch"); + let read_isolated = + super::read_file_bounded("git", work, "nonexistent", "hello.txt", 1024, deadline) + .unwrap(); + assert_eq!( + read_isolated, + super::BoundedFileRead::Found(b"hello".to_vec()) + ); + + // 4. Empty refs fallback: a completely empty repo with unborn HEAD. + let td_empty = tempfile::TempDir::new().unwrap(); + let run_empty = |args: &[&str]| { + let status = std::process::Command::new("git") + .args(args) + .current_dir(td_empty.path()) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + run_empty(&["init", "-q"]); + let resolved_empty = + super::resolve_head_bounded("git", td_empty.path(), "nonexistent", deadline).unwrap(); + assert_eq!(resolved_empty, "HEAD"); + let read_empty = super::read_file_bounded( + "git", + td_empty.path(), + "nonexistent", + "hello.txt", + 1024, + deadline, + ) + .unwrap(); + assert_eq!(read_empty, super::BoundedFileRead::Missing); + } + #[test] fn branch_diff_names_lists_changed_paths() { let td = tempfile::TempDir::new().unwrap(); From c63527cf9520ad451803db6a9fbe4ade189f3b3b Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 02:27:21 -0400 Subject: [PATCH 06/17] fix(node): confirm blob absence against a readable store Check object-store readability and re-probe clean missing results so unavailable packs produce an opaque error instead of a false 404. Fixup for fc2d043b5e6815312c94cdee50f8c8b0f4cf4029. Refs #407 --- crates/gitlawb-node/src/api/repos.rs | 30 +++++ crates/gitlawb-node/src/git/store.rs | 192 +++++++++++++++++++++++---- 2 files changed, 193 insertions(+), 29 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 7ac740ad..d10ce288 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3856,6 +3856,36 @@ mod tests { assert_eq!(body["error"], "not_found"); } + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_unreadable_pack_is_opaque_error(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git(tmp.path(), "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nread spec\necho \"$spec missing\"\n"); + let state = + f4_state_with_repo(pool, tmp.path(), &fake, "z6blobunreadable", "repo", false).await; + let record = state + .db + .get_repo("z6blobunreadable", "repo") + .await + .unwrap() + .unwrap(); + let path = state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + .unwrap(); + std::os::unix::fs::symlink("removed-pack", path.join("objects/pack/unreadable.pack")) + .unwrap(); + let response = + blob_route_request_path(state, "z6blobunreadable", "file.txt", "203.0.113.31:5000") + .await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + use http_body_util::BodyExt; + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["message"], crate::error::INTERNAL_ERROR_MESSAGE); + } + #[cfg(unix)] #[sqlx::test] async fn blob_route_returns_200_with_expected_headers_and_content(pool: sqlx::PgPool) { diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 13541790..617bf319 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -298,34 +298,8 @@ pub fn read_file_bounded( let refname = resolve_head_bounded(git_bin, repo_path, preferred_branch, deadline)?; let spec = format!("{refname}:{file_path}"); let stdin = format!("{spec}\n"); - let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( - git_bin, - &["cat-file", "--batch-check"], - repo_path, - stdin.as_bytes(), - deadline, - )?; - if !status.success() { - bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&stderr) - ); - } - - let stderr = String::from_utf8_lossy(&stderr); - if stderr - .lines() - .any(|line| line.starts_with("error:") || line.starts_with("fatal:")) - { - bail!("git cat-file --batch-check reported: {}", stderr.trim()); - } - - let output = String::from_utf8(stdout).context("git returned non-UTF-8 object metadata")?; - let mut lines = output.lines(); - let line = lines.next().unwrap_or_default().trim(); - if lines.next().is_some() { - bail!("git returned multiple object metadata records"); - } + let output = blob_metadata_bounded(git_bin, repo_path, &spec, stdin.as_bytes(), deadline)?; + let line = output.trim(); if line.split_whitespace().last() == Some("missing") { return Ok(BoundedFileRead::Missing); } @@ -379,10 +353,72 @@ pub fn read_file_bounded( content.len() ); } - Ok(BoundedFileRead::Found(content)) } +fn blob_metadata_bounded( + git_bin: &str, + repo_path: &Path, + spec: &str, + stdin: &[u8], + deadline: std::time::Instant, +) -> Result { + let probe_started = std::time::Instant::now(); + for attempt in 0..2 { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "--batch-check"], + repo_path, + stdin, + deadline, + )?; + if !status.success() { + bail!( + "git cat-file --batch-check failed: {}", + String::from_utf8_lossy(&stderr) + ); + } + + let stderr = String::from_utf8_lossy(&stderr); + if stderr + .lines() + .any(|line| line.starts_with("error:") || line.starts_with("fatal:")) + { + bail!("git cat-file --batch-check reported: {}", stderr.trim()); + } + + let output = String::from_utf8(stdout).context("git returned non-UTF-8 object metadata")?; + let mut lines = output.lines(); + let line = lines.next().unwrap_or_default().trim(); + if lines.next().is_some() { + bail!("git returned multiple object metadata records"); + } + if line.split_whitespace().last() == Some("missing") { + // A clean missing response also occurs for unreadable packs. Use the + // same out-of-band check as object_type_bounded, then confirm once. + let worktree_git = repo_path.join(".git"); + let git_dir = if worktree_git.is_dir() { + worktree_git.as_path() + } else { + repo_path + }; + if !object_store_readable(git_dir, spec) { + bail!("git cat-file inconclusive: object store not readable"); + } + if attempt == 0 { + if deadline.saturating_duration_since(std::time::Instant::now()) + < probe_started.elapsed() + { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + continue; + } + } + return Ok(output); + } + unreachable!("the confirming probe returns its result") +} + #[derive(Debug, Clone, serde::Serialize)] pub struct CommitInfo { pub hash: String, @@ -1035,6 +1071,28 @@ mod tests { use std::path::Path; use std::process::Command; + #[cfg(unix)] + fn write_blob_probe_fixture(dir: &Path, content: &str) -> std::path::PathBuf { + use std::io::Write; + let script = dir.join("fakegit"); + // Write in a child so parallel tests cannot inherit an open writable + // script descriptor when they fork (which would cause ETXTBSY). + let mut writer = Command::new("sh") + .args(["-c", "cat > \"$1\" && chmod 755 \"$1\"", "fixture-writer"]) + .arg(&script) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + writer + .stdin + .take() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + assert!(writer.wait().unwrap().success()); + script + } + #[test] fn bounded_file_read_rejects_packed_blob_and_preserves_allowed_content() { let td = tempfile::TempDir::new().unwrap(); @@ -1083,6 +1141,82 @@ mod tests { ); } + #[cfg(unix)] + #[test] + fn bounded_file_read_confirms_missing_and_recovers_after_reprobe() { + for recover in [false, true] { + let td = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(td.path().join("objects")).unwrap(); + let oid = "a".repeat(40); + let script = write_blob_probe_fixture( + td.path(), + &format!( + r#"#!/bin/sh +if [ "$1" = rev-parse ]; then echo {oid}; exit 0; fi +if [ "$2" = --batch-check ]; then + read spec + echo probe >> probes + if [ -f probed ] && [ "{recover}" = true ]; then echo '{oid} blob 5'; else echo "$spec missing"; fi + touch probed + exit 0 +fi +if [ "$2" = blob ]; then printf hello; exit 0; fi +exit 1 +"# + ), + ); + let result = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "hello.txt", + 1024, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap(); + assert_eq!( + result, + if recover { + super::BoundedFileRead::Found(b"hello".to_vec()) + } else { + super::BoundedFileRead::Missing + } + ); + assert_eq!( + std::fs::read_to_string(td.path().join("probes")) + .unwrap() + .lines() + .count(), + 2 + ); + } + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_does_not_report_missing_for_unreadable_pack() { + use std::os::unix::fs::symlink; + let td = tempfile::TempDir::new().unwrap(); + let pack = td.path().join("objects/pack"); + std::fs::create_dir_all(&pack).unwrap(); + // A disappearing pack is unopenable even when CI runs as root. + symlink("removed-during-repack", pack.join("unreadable.pack")).unwrap(); + let script = write_blob_probe_fixture(td.path(), "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nread spec\necho \"$spec missing\"\n"); + let error = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "hello.txt", + 1024, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap_err(); + assert!( + error.to_string().contains("object store not readable"), + "{error:#}" + ); + } + #[cfg(unix)] #[test] fn bounded_file_read_does_not_capture_rejected_content() { From 8977ed4ecebdbceb593609dfd1f109f6972fa287 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 02:27:22 -0400 Subject: [PATCH 07/17] fix(node): bound REST blob response delivery Use an independent delivery timer and a bounded channel to release admission even when clients stop polling the body. Fixup for fc2d043b5e6815312c94cdee50f8c8b0f4cf4029. Refs #407 --- .env.example | 1 + README.md | 3 + crates/gitlawb-node/src/api/repos.rs | 163 +++++++++++++++++++++++++++ 3 files changed, 167 insertions(+) diff --git a/.env.example b/.env.example index 9c5a2238..ba93aa73 100644 --- a/.env.example +++ b/.env.example @@ -126,6 +126,7 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # serve nothing and the clone gets a 504. Serving large path-scoped repos may # need a higher value here than when each stage was budgeted separately. # REST blob size probes and content reads also share this deadline and return 504 on timeout. +# Body delivery gets the same time allowance; expiry aborts the response and releases admission. # Must be 1..=3153600000 (100 years): the node derives deadlines from this value, # and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 diff --git a/README.md b/README.md index a3f64ae3..f6b3801a 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,9 @@ GET /{owner}/{repo}/info/refs POST /{owner}/{repo}/git-upload-pack ``` +Body delivery has its own allowance equal to `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`. +If delivery exceeds it, the response is interrupted and its admission slots are released. + Signed write routes include: ```txt diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index d10ce288..a62d2d89 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -55,6 +55,73 @@ impl futures::Stream for BlobResponseStream { } } +/// The producer owns admission and the full blob; its timer runs even when +/// the HTTP server stops polling the body. Only one copied chunk is queued. +struct BlobDeliveryStream { + receiver: tokio::sync::mpsc::Receiver, + producer: tokio::task::JoinHandle>, + finished: bool, +} + +impl BlobDeliveryStream { + fn new(mut source: BlobResponseStream, timeout: std::time::Duration) -> Self { + let (sender, receiver) = tokio::sync::mpsc::channel(1); + let producer = tokio::spawn(async move { + use futures::StreamExt; + let delivery = async { + while let Some(Ok(chunk)) = source.next().await { + if sender.send(chunk).await.is_err() { + return; + } + } + // Keep admission until the last queued chunk is consumed, even + // for a one-chunk response whose first send never had to wait. + let _drained = sender.reserve().await; + }; + tokio::time::timeout(timeout, delivery).await.map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "blob delivery timed out") + }) + }); + Self { + receiver, + producer, + finished: false, + } + } +} + +impl futures::Stream for BlobDeliveryStream { + type Item = std::io::Result; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + use std::future::Future; + use std::task::Poll; + let this = self.get_mut(); + if this.finished { + return Poll::Ready(None); + } + if let Some(chunk) = std::task::ready!(this.receiver.poll_recv(cx)) { + return Poll::Ready(Some(Ok(chunk))); + } + let result = std::task::ready!(std::pin::Pin::new(&mut this.producer).poll(cx)); + this.finished = true; + match result { + Ok(Ok(())) => Poll::Ready(None), + Ok(Err(error)) => Poll::Ready(Some(Err(error))), + Err(_) => Poll::Ready(Some(Err(std::io::Error::other("blob delivery failed")))), + } + } +} + +impl Drop for BlobDeliveryStream { + fn drop(&mut self) { + self.producer.abort(); + } +} + /// The set of blob OIDs withheld from **anonymous** replication for a repo, or /// `None` when the repo must not replicate at all (private / mode A / /// undetermined — fail closed). This is the anonymous replication gate: @@ -551,6 +618,10 @@ pub async fn get_blob( _blob_permit: blob_permit, _caller_permit: caller_permit, }; + let stream = BlobDeliveryStream::new( + stream, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ); let mut response = Response::new(axum::body::Body::from_stream(stream)); response.headers_mut().insert( header::CONTENT_TYPE, @@ -3476,6 +3547,64 @@ mod tests { assert!(callers.try_acquire("source").is_some()); } + #[tokio::test(start_paused = true)] + async fn blob_delivery_deadline_releases_admission_without_body_polls() { + use futures::StreamExt; + for (chunks, read_first_chunk) in [(1, false), (8, false), (8, true)] { + let git = Arc::new(tokio::sync::Semaphore::new(1)); + let blob = Arc::new(tokio::sync::Semaphore::new(1)); + let callers = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let source = BlobResponseStream { + content: Bytes::from(vec![b'x'; BLOB_RESPONSE_CHUNK_BYTES * chunks]), + _git_permit: git.clone().try_acquire_owned().unwrap(), + _blob_permit: blob.clone().try_acquire_owned().unwrap(), + _caller_permit: callers.try_acquire("source"), + }; + let mut stream = BlobDeliveryStream::new(source, std::time::Duration::from_secs(10)); + tokio::task::yield_now().await; + if read_first_chunk { + assert_eq!( + stream.next().await.unwrap().unwrap().len(), + BLOB_RESPONSE_CHUNK_BYTES + ); + tokio::task::yield_now().await; + } + assert_eq!(blob.available_permits(), 0); + tokio::time::advance(std::time::Duration::from_secs(11)).await; + tokio::task::yield_now().await; + // The body still exists and has not been polled during the deadline. + assert_eq!(git.available_permits(), 1); + assert_eq!(blob.available_permits(), 1); + assert!(callers.try_acquire("source").is_some()); + assert!(stream.next().await.unwrap().is_ok()); // one queued chunk + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert!(stream.next().await.is_none()); + } + } + + #[tokio::test(start_paused = true)] + async fn blob_delivery_disconnect_releases_admission_before_deadline() { + let git = Arc::new(tokio::sync::Semaphore::new(1)); + let blob = Arc::new(tokio::sync::Semaphore::new(1)); + let callers = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let stream = BlobDeliveryStream::new( + BlobResponseStream { + content: Bytes::from(vec![b'x'; BLOB_RESPONSE_CHUNK_BYTES * 8]), + _git_permit: git.clone().try_acquire_owned().unwrap(), + _blob_permit: blob.clone().try_acquire_owned().unwrap(), + _caller_permit: callers.try_acquire("source"), + }, + std::time::Duration::from_secs(600), + ); + tokio::task::yield_now().await; + drop(stream); + tokio::task::yield_now().await; + assert_eq!(git.available_permits(), 1); + assert_eq!(blob.available_permits(), 1); + assert!(callers.try_acquire("source").is_some()); + } + #[test] fn upload_pack_request_finalizes_only_with_done_pktline() { let want = "0032want 1111111111111111111111111111111111111111\n"; @@ -3856,6 +3985,40 @@ mod tests { assert_eq!(body["error"], "not_found"); } + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_delivery_uses_configured_timeout(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git(tmp.path(), "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 blob 524288'; exit 0; fi\nhead -c 524288 /dev/zero\n"); + let mut state = + f4_state_with_repo(pool, tmp.path(), &fake, "z6blobdelivery", "repo", false).await; + let mut config = (*state.config).clone(); + config.git_service_timeout_secs = 1; + state.config = Arc::new(config); + let response = blob_route_request_path( + state.clone(), + "z6blobdelivery", + "file.txt", + "203.0.113.31:5000", + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + state.git_blob_semaphore.available_permits(), + MAX_CONCURRENT_BLOB_READS - 1 + ); + // Keep the body alive without ever polling it; the timer must run anyway. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while state.git_blob_semaphore.available_permits() != MAX_CONCURRENT_BLOB_READS { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("configured deadline must release blob admission"); + use http_body_util::BodyExt; + assert!(response.into_body().collect().await.is_err()); + } + #[cfg(unix)] #[sqlx::test] async fn blob_route_unreadable_pack_is_opaque_error(pool: sqlx::PgPool) { From e1fe66e57c3504c94480c6b88d45af0bdda09dac Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 02:27:22 -0400 Subject: [PATCH 08/17] test(git): distinguish every bounded ref fallback Use competing branches and a resolving HEAD so each fallback preference changes an observable test result. Fixup for fc2d043b5e6815312c94cdee50f8c8b0f4cf4029. Refs #407 --- crates/gitlawb-node/src/git/store.rs | 33 +++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 617bf319..c12c6191 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -1321,11 +1321,15 @@ exit 1 run(&["add", "hello.txt"]); run(&["commit", "-qm", "initial"]); run(&["branch", "-M", "custom-feature"]); + run(&["branch", "aaa-earlier"]); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + assert_eq!( + super::resolve_head_bounded("git", work, "aaa-earlier", deadline).unwrap(), + "HEAD" + ); // Detach or point HEAD to an unborn branch so HEAD itself does not resolve. run(&["symbolic-ref", "HEAD", "refs/heads/unborn-branch"]); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); - // 1. Preferred branch arm: "custom-feature" resolves to refs/heads/custom-feature. let resolved = super::resolve_head_bounded("git", work, "custom-feature", deadline).unwrap(); @@ -1349,8 +1353,31 @@ exit 1 super::BoundedFileRead::Found(b"hello".to_vec()) ); - // 3. for-each-ref fallback arm: rename "master" to "isolated-branch" (neither main, master, nor develop). + // Main must precede master, which must precede develop; all precede + // the alphabetically earlier branch chosen by for-each-ref. + run(&["branch", "develop", "master"]); + run(&["branch", "main", "master"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/main" + ); + run(&["branch", "-m", "main", "z-retired-main"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/master" + ); run(&["branch", "-M", "master", "isolated-branch"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/develop" + ); + run(&["branch", "-m", "develop", "z-retired-develop"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/aaa-earlier" + ); + run(&["branch", "-m", "aaa-earlier", "z-retired-earlier"]); + // 3. for-each-ref fallback arm: only a nonstandard branch remains. let resolved_isolated = super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(); assert_eq!(resolved_isolated, "refs/heads/isolated-branch"); From 4344b77e5415a7c4434cb479eac393a21b56dcce Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 02:27:22 -0400 Subject: [PATCH 09/17] docs(node): describe the cat-file blob read path Name the current Git invocation and document file-only blob responses. Fixup for fc2d043b5e6815312c94cdee50f8c8b0f4cf4029. Refs #407 --- README.md | 1 + crates/gitlawb-node/src/api/repos.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f6b3801a..1e48da31 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,7 @@ GET /{owner}/{repo}/info/refs POST /{owner}/{repo}/git-upload-pack ``` +REST blob reads serve file content only; directory and gitlink paths return 404. Body delivery has its own allowance equal to `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`. If delivery exceeds it, the response is interrupted and its admission slots are released. diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index a62d2d89..80788972 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -520,7 +520,8 @@ pub async fn get_blob( ) -> Result { use axum::http::header; - // Unnormalized paths ("../..", "./", "//") can't resolve in `git show` + // Unnormalized paths ("../..", "./", "//") are invalid in the ref:path + // input sent to `git cat-file --batch-check`, // and crawlers combinatorially explode them from relative links — that's // a client error, not a 500. let file_path = file_path.trim_matches('/'); From 49aebb22da497ad140f1f5207077f2c3a32df2aa Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 02:27:23 -0400 Subject: [PATCH 10/17] fix(node): prevent caching of authorized blob content Set Cache-Control to no-store so private content is not reused after an identity change, and assert the response header. Fixup for fc2d043b5e6815312c94cdee50f8c8b0f4cf4029. Refs #407 --- README.md | 1 + crates/gitlawb-node/src/api/repos.rs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/README.md b/README.md index 1e48da31..9b6f1e0c 100644 --- a/README.md +++ b/README.md @@ -343,6 +343,7 @@ POST /{owner}/{repo}/git-upload-pack REST blob reads serve file content only; directory and gitlink paths return 404. Body delivery has its own allowance equal to `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`. If delivery exceeds it, the response is interrupted and its admission slots are released. +Blob responses use `Cache-Control: no-store` because they may contain private content. Signed write routes include: diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 80788972..2c2d0675 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -624,6 +624,10 @@ pub async fn get_blob( std::time::Duration::from_secs(state.config.git_service_timeout_secs), ); let mut response = Response::new(axum::body::Body::from_stream(stream)); + response.headers_mut().insert( + header::CACHE_CONTROL, + axum::http::HeaderValue::from_static("no-store"), + ); response.headers_mut().insert( header::CONTENT_TYPE, axum::http::HeaderValue::from_static(mime), @@ -4069,6 +4073,7 @@ mod tests { "application/json; charset=utf-8" ); assert_eq!(response.headers()[header::CONTENT_LENGTH], "13"); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(&bytes[..], b"{\"hello\":123}"); } From c566455d04ae88f728c2b11070ce4e034dda1fe7 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 16:43:42 -0400 Subject: [PATCH 11/17] fix(node): pin non-blob denial and map unreadable store to 503 Add regression coverage ensuring non-blob paths deny with 404, cover the storage acquire timeout on the blob route, pin size-mismatch and metadata parse error branches, map unreadable-store faults to 503, and make the 1s delivery-timeout test deadline-safe. Refs #407 --- crates/gitlawb-node/src/api/repos.rs | 80 +++++++++++- crates/gitlawb-node/src/git/store.rs | 187 +++++++++++++++++++++++++++ 2 files changed, 261 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2c2d0675..cc82bbf2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -584,6 +584,11 @@ pub async fn get_blob( if e.downcast_ref::().is_some() { return AppError::Timeout("git service timed out".into()); } + if e.to_string().contains("object store not readable") { + return AppError::Overloaded( + "object store temporarily unavailable, retry shortly".into(), + ); + } AppError::Internal(e) })?; @@ -4008,10 +4013,7 @@ mod tests { ) .await; assert_eq!(response.status(), StatusCode::OK); - assert_eq!( - state.git_blob_semaphore.available_permits(), - MAX_CONCURRENT_BLOB_READS - 1 - ); + assert!(state.git_blob_semaphore.available_permits() <= MAX_CONCURRENT_BLOB_READS - 1); // Keep the body alive without ever polling it; the timer must run anyway. tokio::time::timeout(std::time::Duration::from_secs(5), async { while state.git_blob_semaphore.available_permits() != MAX_CONCURRENT_BLOB_READS { @@ -4047,11 +4049,77 @@ mod tests { let response = blob_route_request_path(state, "z6blobunreadable", "file.txt", "203.0.113.31:5000") .await; - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); use http_body_util::BodyExt; let bytes = response.into_body().collect().await.unwrap().to_bytes(); let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); - assert_eq!(body["message"], crate::error::INTERNAL_ERROR_MESSAGE); + assert_eq!(body["error"], "overloaded"); + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_returns_not_found_on_non_blob_path(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git( + tmp.path(), + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 tree 1024'; exit 0; fi\nexit 1\n", + ); + let state = f4_state_with_repo(pool, tmp.path(), &fake, "z6blobtree", "repo", false).await; + let response = + blob_route_request_path(state, "z6blobtree", "somedir", "203.0.113.31:5000").await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "not_found"); + } + + #[sqlx::test] + async fn blob_route_acquire_timeout_sheds_503_and_releases_permits(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().to_path_buf(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let endpoint = crate::test_support::silent_http_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Repo exists in DB but not on disk, so acquire attempts Tigris and times out. + state + .db + .upsert_mirror_repo("z6blobacqtimeout", "repo", "/unused", None, false) + .await + .unwrap(); + + let response = blob_route_request_path( + state.clone(), + "z6blobacqtimeout", + "file.txt", + "203.0.113.31:5000", + ) + .await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "overloaded"); + assert!(state.git_read_semaphore.available_permits() > 0); + assert_eq!( + state.git_blob_semaphore.available_permits(), + MAX_CONCURRENT_BLOB_READS + ); + assert!(state + .git_read_per_caller + .try_acquire("203.0.113.31") + .is_some()); } #[cfg(unix)] diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index c12c6191..5d388148 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -1302,6 +1302,193 @@ exit 1 ); } + #[cfg(unix)] + #[test] + fn bounded_file_read_guards_size_mismatch_and_metadata_parse() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let script = td.path().join("fakegit"); + let write_fake = |body: &str| { + std::fs::write(&script, body).unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&script, permissions).unwrap(); + }; + + let oid = "c".repeat(40); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // 1. Size mismatch (emitted bytes fewer than declared size): + // Declares 20 bytes, emits 5 bytes ("hello"). max_bytes is 64 (so exceeded is false). + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 20\"; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"blob\" ]; then printf hello; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "mismatch.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string() + .contains("size changed between metadata and content reads"), + "expected size mismatch error, got: {err:#}" + ); + + // 2. Metadata parse: missing fields (only 2 fields, no size) + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob\"; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("omitted the object size"), + "expected missing fields error, got: {err:#}" + ); + + // 3. Metadata parse: non-hex OID + write_fake( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo nothex; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"not-a-valid-hex-oid-1234567890123456789012 blob 10\"; exit 0; fi\n\ + exit 1\n" + ); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("invalid object metadata"), + "expected non-hex oid error, got: {err:#}" + ); + + // 4. Metadata parse: trailing fields + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 10 trailing-junk\"; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("invalid object metadata"), + "expected trailing fields error, got: {err:#}" + ); + + // 5. Metadata parse: multi-record output + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 10\n{oid} blob 10\"; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("multiple object metadata records"), + "expected multi-record error, got: {err:#}" + ); + } + + #[cfg(unix)] + #[test] + fn blob_metadata_bounded_reprobe_budget_exhaustion_returns_timeout() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let script = td.path().join("fakegit"); + std::fs::write( + &script, + "#!/bin/sh\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then sleep 0.05; read spec; echo \"$spec missing\"; exit 0; fi\n\ + exit 1\n", + ) + .unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&script, permissions).unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(30); + let err = super::blob_metadata_bounded( + script.to_str().unwrap(), + td.path(), + "spec", + b"spec\n", + deadline, + ) + .unwrap_err(); + assert!( + err.downcast_ref::() + .is_some(), + "exhausted reprobe budget must return GitServiceTimeout, got: {err:#}" + ); + } + + #[test] + fn read_file_bounded_denies_non_blob_paths() { + let td = tempfile::TempDir::new().unwrap(); + let work = td.path(); + let run = |args: &[&str]| { + let status = std::process::Command::new("git") + .args(args) + .current_dir(work) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + std::fs::create_dir(work.join("subfolder")).unwrap(); + std::fs::write(work.join("subfolder/file.txt"), b"contents").unwrap(); + run(&["add", "subfolder"]); + run(&["commit", "-qm", "add directory"]); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + + // "subfolder" resolves to a tree object in git, not a blob. + let read = + super::read_file_bounded("git", work, "main", "subfolder", 1024, deadline).unwrap(); + assert_eq!(read, super::BoundedFileRead::Missing); + } + #[test] fn resolve_head_bounded_covers_all_fallback_arms() { let td = tempfile::TempDir::new().unwrap(); From df5a283c9f60cb10cc885d26dfd5cb648eca0276 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 20:41:19 -0400 Subject: [PATCH 12/17] style(repos): simplify blob semaphore permit assertion for clippy Use strict less-than comparison to satisfy clippy::int_plus_one lint. Refs #407 --- crates/gitlawb-node/src/api/repos.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index cc82bbf2..2589e592 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -4013,7 +4013,7 @@ mod tests { ) .await; assert_eq!(response.status(), StatusCode::OK); - assert!(state.git_blob_semaphore.available_permits() <= MAX_CONCURRENT_BLOB_READS - 1); + assert!(state.git_blob_semaphore.available_permits() < MAX_CONCURRENT_BLOB_READS); // Keep the body alive without ever polling it; the timer must run anyway. tokio::time::timeout(std::time::Duration::from_secs(5), async { while state.git_blob_semaphore.available_permits() != MAX_CONCURRENT_BLOB_READS { From f0d0025488102659a2b5fea01c3b2c51d1b82314 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sat, 12 Sep 2026 20:56:56 -0400 Subject: [PATCH 13/17] fix(git): classify unreadable store with ProbeError and tighten blob test asserts Return and downcast ProbeError::Transient for unreadable object store to drop string matching. Build lock pool and assert full permit and key release on acquire timeout. Use printf in fake git script and ensure reprobe budget test reaches the budget arm with spawn verification. Refs #407 --- crates/gitlawb-node/src/api/repos.rs | 17 +++++----- crates/gitlawb-node/src/git/store.rs | 47 +++++++++++++++++++++------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 2589e592..a02d0ac1 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -584,7 +584,10 @@ pub async fn get_blob( if e.downcast_ref::().is_some() { return AppError::Timeout("git service timed out".into()); } - if e.to_string().contains("object store not readable") { + if matches!( + e.downcast_ref::(), + Some(store::ProbeError::Transient(_)) + ) { return AppError::Overloaded( "object store temporarily unavailable, retry shortly".into(), ); @@ -4087,7 +4090,10 @@ mod tests { let tigris = crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) .await; - state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + let lock_pool = + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)); + state.repo_store = + crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), lock_pool); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; let mut cfg = (*state.config).clone(); cfg.git_acquire_timeout_secs = 1; @@ -4111,15 +4117,12 @@ mod tests { let bytes = response.into_body().collect().await.unwrap().to_bytes(); let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(body["error"], "overloaded"); - assert!(state.git_read_semaphore.available_permits() > 0); + assert_eq!(state.git_read_semaphore.available_permits(), 64); assert_eq!( state.git_blob_semaphore.available_permits(), MAX_CONCURRENT_BLOB_READS ); - assert!(state - .git_read_per_caller - .try_acquire("203.0.113.31") - .is_some()); + assert_eq!(state.git_read_per_caller.tracked_keys(), 0); } #[cfg(unix)] diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5d388148..45e39834 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -403,7 +403,10 @@ fn blob_metadata_bounded( repo_path }; if !object_store_readable(git_dir, spec) { - bail!("git cat-file inconclusive: object store not readable"); + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: object store not readable" + )) + .into()); } if attempt == 0 { if deadline.saturating_duration_since(std::time::Instant::now()) @@ -1410,7 +1413,7 @@ exit 1 write_fake(&format!( "#!/bin/sh\n\ if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ - if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 10\n{oid} blob 10\"; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; printf '%s\\n%s\\n' '{oid} blob 10' '{oid} blob 10'; exit 0; fi\n\ exit 1\n" )); let err = super::read_file_bounded( @@ -1434,22 +1437,35 @@ exit 1 use std::os::unix::fs::PermissionsExt; let td = tempfile::TempDir::new().unwrap(); - let script = td.path().join("fakegit"); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("spawns.log"); + let fake = td.path().join("fakegit"); std::fs::write( - &script, - "#!/bin/sh\n\ - if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then sleep 0.05; read spec; echo \"$spec missing\"; exit 0; fi\n\ - exit 1\n", + &fake, + format!( + "#!/bin/sh\n\ + echo call >> {}\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then \ + read spec; \ + sleep 0.15; \ + printf '%s\\n' \"$spec missing\"; \ + exit 0; \ + fi\n\ + exit 1\n", + log.display() + ), ) .unwrap(); - let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + let mut permissions = std::fs::metadata(&fake).unwrap().permissions(); permissions.set_mode(0o755); - std::fs::set_permissions(&script, permissions).unwrap(); + std::fs::set_permissions(&fake, permissions).unwrap(); - let deadline = std::time::Instant::now() + std::time::Duration::from_millis(30); + let budget = std::time::Duration::from_millis(200); + let deadline = std::time::Instant::now() + budget; let err = super::blob_metadata_bounded( - script.to_str().unwrap(), - td.path(), + fake.to_str().unwrap(), + &bare, "spec", b"spec\n", deadline, @@ -1460,6 +1476,13 @@ exit 1 .is_some(), "exhausted reprobe budget must return GitServiceTimeout, got: {err:#}" ); + let spawns = std::fs::read_to_string(&log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + spawns, 1, + "a confirming probe must not be spawned when remaining budget is insufficient" + ); } #[test] From eb1222df61450b8e12b646b38668d129ce9bf426 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sun, 13 Sep 2026 23:39:29 -0400 Subject: [PATCH 14/17] test(node): assert full blob admission release after read errors Require every read and blob permit to return and the per-caller entry to disappear after oversized and timed-out reads. Partial-capacity assertions could stay green despite a leaked permit. Refs #407 --- crates/gitlawb-node/src/api/repos.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index a02d0ac1..67c70d68 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3942,12 +3942,12 @@ mod tests { "git_timeout" } ); - assert!(state.git_read_semaphore.available_permits() > 0); - assert!(state.git_blob_semaphore.available_permits() > 0); - assert!(state - .git_read_per_caller - .try_acquire("203.0.113.31") - .is_some()); + assert_eq!(state.git_read_semaphore.available_permits(), 64); + assert_eq!( + state.git_blob_semaphore.available_permits(), + MAX_CONCURRENT_BLOB_READS + ); + assert_eq!(state.git_read_per_caller.tracked_keys(), 0); } } From c15acf5089be1e527054535b52bd96ffb6fbe27c Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sun, 13 Sep 2026 23:39:29 -0400 Subject: [PATCH 15/17] test(git): require probe completion before reprobe budget denial Record completion after the probe writes its result, so a watchdog kill cannot masquerade as the unaffordable-reprobe branch. Quote the log path and leave scheduler margin while spending more than half the budget. Refs #407 --- crates/gitlawb-node/src/git/store.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 45e39834..901fb27d 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -1439,21 +1439,22 @@ exit 1 let td = tempfile::TempDir::new().unwrap(); let bare = td.path().join("bare.git"); std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); - let log = td.path().join("spawns.log"); + let log = td.path().join("probe spawns.log"); + let quoted_log = format!("'{}'", log.display().to_string().replace('\'', "'\"'\"'")); let fake = td.path().join("fakegit"); std::fs::write( &fake, format!( "#!/bin/sh\n\ - echo call >> {}\n\ + echo call >> {quoted_log}\n\ if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then \ read spec; \ - sleep 0.15; \ + sleep 2.5; \ printf '%s\\n' \"$spec missing\"; \ + echo done >> {quoted_log}; \ exit 0; \ fi\n\ - exit 1\n", - log.display() + exit 1\n" ), ) .unwrap(); @@ -1461,7 +1462,9 @@ exit 1 permissions.set_mode(0o755); std::fs::set_permissions(&fake, permissions).unwrap(); - let budget = std::time::Duration::from_millis(200); + // Spend over half the budget while leaving 1.5s for scheduler jitter. + // The completion marker below must still rule out a watchdog kill. + let budget = std::time::Duration::from_secs(4); let deadline = std::time::Instant::now() + budget; let err = super::blob_metadata_bounded( fake.to_str().unwrap(), @@ -1476,12 +1479,10 @@ exit 1 .is_some(), "exhausted reprobe budget must return GitServiceTimeout, got: {err:#}" ); - let spawns = std::fs::read_to_string(&log) - .map(|s| s.lines().count()) - .unwrap_or(0); assert_eq!( - spawns, 1, - "a confirming probe must not be spawned when remaining budget is insufficient" + std::fs::read_to_string(&log).unwrap(), + "call\ndone\n", + "the first probe must finish, without spawning an unaffordable confirming probe" ); } From 77e9b76e5562f126bf7471fb95db2ab509bc1647 Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sun, 13 Sep 2026 23:39:30 -0400 Subject: [PATCH 16/17] test(node): make Windows regression fixtures portable Gate Unix-only helpers with their callers, replace shell timing shims with native Rust fixtures, and normalize rooted fixture paths without a drive prefix. Distinguish admission rejection from a closed database by error code, since both correctly return HTTP 503. Refs #407 --- crates/gitlawb-node/src/api/ipfs.rs | 18 ++-- crates/gitlawb-node/src/api/repos.rs | 14 +++ crates/gitlawb-node/src/git/repo_store.rs | 4 +- crates/gitlawb-node/src/ipfs_pin.rs | 1 + crates/gitlawb-node/src/pinata.rs | 1 + crates/gitlawb-node/src/sync.rs | 24 ++++- crates/gitlawb-node/src/test_git_shim.rs | 75 ++++++++++++++++ crates/gitlawb-node/src/test_support.rs | 88 +++++-------------- .../gitlawb-node/tests/fixtures/git_shim.rs | 59 +++++++++++++ 9 files changed, 209 insertions(+), 75 deletions(-) create mode 100644 crates/gitlawb-node/src/test_git_shim.rs create mode 100644 crates/gitlawb-node/tests/fixtures/git_shim.rs diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 92d12980..40139804 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -8338,6 +8338,7 @@ mod tests { #[tokio::test] async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { let mut state = crate::test_support::test_state_lazy(); + state.db.pool().close().await; // Global pool has room; the per-source cap is 1. state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); @@ -8364,16 +8365,23 @@ mod tests { "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" ); + let bytes = axum::body::to_bytes(resp.into_body(), 1024).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "overloaded"); + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and - // proceeds (then errors on the lazy DB, which is not a 503). + // proceeds to the closed DB, which has a distinct db_unavailable error code. let resp = ipfs_router(state) .oneshot(get_cid(&cid, Some(other))) .await .unwrap(); - assert_ne!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "a different source must not be shed by the per-source cap" + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = axum::body::to_bytes(resp.into_body(), 1024).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"], + crate::error::DB_UNAVAILABLE_CODE, + "a different source must clear admission and reach the closed database" ); } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 67c70d68..f4a0aafc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -7785,6 +7785,7 @@ mod tests { /// A pkt-line receive-pack body carrying one branch-create ref update, so the /// handler's post-receive tail resolves a non-empty new-tip set (the delta /// scan's git stages run). + #[cfg(unix)] fn ref_update_body(new_sha: &str) -> axum::body::Bytes { let line = format!("{ZERO_SHA} {new_sha} refs/heads/main"); axum::body::Bytes::from(format!("{:04x}{}0000", line.len() + 4, line)) @@ -10242,6 +10243,7 @@ mod tests { ) } + #[cfg(unix)] fn f2a_log(log: &std::path::Path) -> String { std::fs::read_to_string(log).unwrap_or_default() } @@ -10249,6 +10251,7 @@ mod tests { /// Withheld-walk children run so far. `ls-tree` is the walk's signature child /// (`blob_paths` lists every reachable commit's tree); the delta scan and the /// full-scan fallback use `rev-list` / `cat-file` instead. + #[cfg(unix)] fn f2a_walks(log: &std::path::Path) -> usize { f2a_log(log) .lines() @@ -10260,6 +10263,7 @@ mod tests { /// the withheld walk actually runs rather than taking the no-rule shortcut). /// The repo's on-disk path is passed to the tail directly, so no repo_store or /// receive-pack plumbing is involved. + #[cfg(unix)] async fn f2a_state( pool: sqlx::PgPool, git_bin: &str, @@ -10291,6 +10295,7 @@ mod tests { (state, rec) } + #[cfg(unix)] fn f2a_update(ref_name: &str, new_sha: &str) -> Vec { vec![RefUpdate { old_sha: ZERO_SHA.to_string(), @@ -10299,6 +10304,7 @@ mod tests { }] } + #[cfg(unix)] const F2A_PUSHER: &str = "did:key:z6MkF2aPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; /// Scenario 1 (the finding). A second rapid push to the same repo coalesces @@ -10372,6 +10378,7 @@ mod tests { } /// Poll `cond` until it holds, with a bound so a regression fails the test /// rather than hanging the suite. + #[cfg(unix)] async fn f2a_wait_for(mut cond: impl FnMut() -> bool, what: &str) { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); while !cond() { @@ -10386,6 +10393,7 @@ mod tests { /// A `rev-list --objects` line names the tips a DELTA scan was asked to resolve, /// so it attributes that scan to one push's tips. The withheld walk's own /// `rev-list --all` / `ls-tree` lines never carry a tip as an argument this way. + #[cfg(unix)] fn f2a_delta_scanned(log: &std::path::Path, tip: &str) -> bool { f2a_log(log) .lines() @@ -10532,6 +10540,7 @@ mod tests { /// Mount a Pinata upload endpoint that assigns every object the same CID, and /// point the state at it. Returns the server (kept alive by the caller) and CID. + #[cfg(unix)] async fn f2a_pinata(state: &mut AppState) -> (mockito::ServerGuard, String) { let cid = "bafyf2acoalescedmapping".to_string(); let mut server = mockito::Server::new_async().await; @@ -10551,6 +10560,7 @@ mod tests { /// Poll the branch to CID table until the push's mapping lands (the Pinata /// worker is detached), bounded so a regression fails rather than hangs. + #[cfg(unix)] async fn f2a_wait_for_branch_cid( db: &crate::db::Db, slug: &str, @@ -10570,6 +10580,7 @@ mod tests { } } + #[cfg(unix)] fn f2a_slug(rec: &crate::db::RepoRecord) -> String { format!( "{}/{}", @@ -10703,6 +10714,7 @@ mod tests { // and `z6p2fail`), and owner-only push is on by default, so the identity has to // follow the repo each push targets rather than being fixed for both. + #[cfg(unix)] fn p2_push( state: &AppState, owner: &str, @@ -10721,6 +10733,7 @@ mod tests { ) } + #[cfg(unix)] fn p2_logged(log: &std::path::Path, prefix: &str) -> bool { f2a_log(log).lines().any(|l| l.starts_with(prefix)) } @@ -11020,6 +11033,7 @@ mod tests { /// runs exactly one `rev-list --all`, so this counts the walks that were attempted /// (the `ls-tree` counter above cannot: a walk whose enumeration fails never gets /// to `ls-tree`). + #[cfg(unix)] fn f2b_walk_attempts(log: &std::path::Path) -> usize { f2a_log(log) .lines() diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 45820746..2a465c59 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -83,7 +83,7 @@ impl RepoStore { /// Test-only: every guard from this store parks in `release` right before the /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future /// while it is parked reproduces a client disconnect inside `release`. - #[cfg(test)] + #[cfg(all(test, unix))] pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { self.pre_unlock_gate = Some(gate); self @@ -107,7 +107,7 @@ impl RepoStore { /// Test-only: how many write guards from this store have reached the Tigris upload /// site. See [`RepoStore::upload_site_reached`]. - #[cfg(test)] + #[cfg(all(test, unix))] pub fn tigris_upload_site_reached(&self) -> usize { self.upload_site_reached .load(std::sync::atomic::Ordering::SeqCst) diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3..9b0e929d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -2534,6 +2534,7 @@ mod tests { /// type probe carries its oid on stdin rather than in argv, so an oid appears in /// the log only once an object has already got past its probe, and a healthy /// object costs two invocations to a faulting one's one. + #[cfg(unix)] fn objects_attempted(log: &std::path::Path) -> usize { std::fs::read_to_string(log) .unwrap_or_default() diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 14f1d582..d3e29217 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -911,6 +911,7 @@ mod tests { /// probe carries its oid on stdin rather than in argv, so an oid appears in the log only /// once an object has already got past its probe, and a healthy object costs two /// invocations to a faulting one's one. + #[cfg(unix)] fn objects_attempted(log: &std::path::Path) -> usize { std::fs::read_to_string(log) .unwrap_or_default() diff --git a/crates/gitlawb-node/src/sync.rs b/crates/gitlawb-node/src/sync.rs index 0ed4a9f9..62fa898a 100644 --- a/crates/gitlawb-node/src/sync.rs +++ b/crates/gitlawb-node/src/sync.rs @@ -1250,6 +1250,18 @@ mod tests { .await; } + /// Use a rooted path without a Windows drive prefix in the remote fixture. + /// The destination still resolves on the temp directory's current drive, + /// while the remote's relative path contains no illegal colon component. + fn absolute_slug_path(path: &Path) -> String { + assert!(path.is_absolute()); + path.components() + .filter(|part| !matches!(part, std::path::Component::Prefix(_))) + .collect::() + .to_string_lossy() + .replace('\\', "/") + } + #[sqlx::test] async fn process_batch_rejects_slug_escaping_repos_dir(pool: PgPool) { // The verified escape from #272: `PathBuf::join` discards everything @@ -1265,12 +1277,13 @@ mod tests { // fails on but leaves the parent it created, so assert on both. let outside = TempDir::new().unwrap(); let escape_dir = outside.path().join("nest"); - let slug = format!("a/{}/escape", escape_dir.display()); + let escape_path = absolute_slug_path(&escape_dir); + let slug = format!("a/{escape_path}/escape"); let escape_target = escape_dir.join("escape.git"); // Serve the composed URL for real, so a run without the guard genuinely // clones outside the root rather than merely failing at git. - let rel = format!("a{}/escape", escape_dir.display()); + let rel = format!("a{escape_path}/escape"); let (_remote, peer_url) = rooted_remote(&[&rel]); let did = "did:key:z6MkAttacker"; @@ -1390,6 +1403,7 @@ mod tests { // ── canonical containment before the git call (issue #272) ─────────────── /// Every ref in `repo`, as one string, for a before/after comparison. + #[cfg(unix)] fn refs_of(repo: &Path) -> String { let out = Command::new("git") .args(["-C", repo.to_str().unwrap(), "for-each-ref"]) @@ -1722,6 +1736,7 @@ mod tests { /// starvation tests is fixed rather than dependent on how fast the loop /// runs. Two rows enqueued in the same microsecond would otherwise order /// arbitrarily. + #[cfg(unix)] async fn enqueue_at(db: &Db, pool: &PgPool, repo: &str, did: &str, enqueued_at: &str) { enqueue(db, repo, did).await; sqlx::query("UPDATE sync_queue SET enqueued_at = $1 WHERE repo = $2") @@ -1915,7 +1930,8 @@ mod tests { let outside = TempDir::new().unwrap(); let escape_dir = outside.path().join("nest"); - let slug = format!("a/{}/gitlawb-probe", escape_dir.display()); + let escape_path = absolute_slug_path(&escape_dir); + let slug = format!("a/{escape_path}/gitlawb-probe"); let escape_target = escape_dir.join("gitlawb-probe.git"); // Two things this fixture must get right or the test is green for the @@ -1926,7 +1942,7 @@ mod tests { // run without the guard genuinely clones outside the root instead of // just failing at git. Db::upsert_peer cannot seed this row: it gates on // is_public_http_url, which rejects file://. - let rel = format!("a{}/gitlawb-probe", escape_dir.display()); + let rel = format!("a{escape_path}/gitlawb-probe"); let (_remote, peer_url) = rooted_remote(&[&rel]); let did = "did:key:z6MkAttacker"; seed_local_peer(&pool, did, &peer_url).await; diff --git a/crates/gitlawb-node/src/test_git_shim.rs b/crates/gitlawb-node/src/test_git_shim.rs new file mode 100644 index 00000000..a53b05d6 --- /dev/null +++ b/crates/gitlawb-node/src/test_git_shim.rs @@ -0,0 +1,75 @@ +//! Native Git fixtures shared by timing tests on Unix and Windows. + +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +/// The fixture delays or hangs in its own process, so the production watchdog +/// can terminate it without depending on a shell or descendant process support. +pub(super) enum Behavior<'a> { + Delay(u64), + Hang, + HangOid(&'a str), + HangRepo(&'a str), + LogTypes(&'a Path), +} + +pub(super) struct GitShim { + _directory: tempfile::TempDir, + executable: PathBuf, +} + +impl Deref for GitShim { + type Target = Path; + + fn deref(&self) -> &Path { + &self.executable + } +} + +pub(super) fn create(name: &str, behavior: Behavior<'_>) -> GitShim { + static COMPILED: OnceLock = OnceLock::new(); + let compiled = COMPILED.get_or_init(|| { + let directory = tempfile::tempdir().expect("native Git fixture directory"); + let source = directory.path().join("git_shim.rs"); + std::fs::write(&source, include_str!("../tests/fixtures/git_shim.rs")) + .expect("write native Git fixture source"); + let executable = directory + .path() + .join(format!("git-shim{}", std::env::consts::EXE_SUFFIX)); + let output = + std::process::Command::new(std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into())) + .arg("--edition=2021") + .arg("-Dwarnings") + .arg(&source) + .arg("-o") + .arg(executable) + .output() + .expect("compile native Git fixture with the installed Rust toolchain"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + directory + }); + let directory = tempfile::Builder::new() + .prefix(name) + .tempdir() + .expect("Git fixture instance"); + let file_name = format!("git-shim{}", std::env::consts::EXE_SUFFIX); + let executable = directory.path().join(&file_name); + std::fs::copy(compiled.path().join(file_name), &executable).expect("copy native Git fixture"); + let config = match behavior { + Behavior::Delay(milliseconds) => format!("delay\n{milliseconds}"), + Behavior::Hang => "hang\n".to_owned(), + Behavior::HangOid(oid) => format!("hang-oid\n{oid}"), + Behavior::HangRepo(repo) => format!("hang-repo\n{repo}"), + Behavior::LogTypes(path) => format!("log-types\n{}", path.display()), + }; + std::fs::write(directory.path().join("config"), config).expect("configure native Git fixture"); + GitShim { + _directory: directory, + executable, + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 5797b860..5cb1e390 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -28,6 +28,9 @@ use gitlawb_core::identity::Keypair; use crate::auth::AuthenticatedDid; use crate::state::AppState; +#[path = "test_git_shim.rs"] +mod git_shim; + /// Build an [`AppState`] over a real, migrated Postgres pool (from `#[sqlx::test]`). /// Runs the schema migrations first, because the per-test database starts empty. /// @@ -430,11 +433,12 @@ mod tests { /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an /// authenticated push — the write pool is a separate budget. Read pool at zero, /// write pool with capacity: the push proceeds PAST admission (it then errors on - /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// the closed DB, with db_unavailable rather than overloaded). Route git-receive-pack /// back to the read pool and this goes red — that is the isolation proof. #[tokio::test] async fn git_receive_pack_not_shed_by_exhausted_read_pool() { let mut state = test_state_lazy(); + state.db.pool().close().await; // Read pool exhausted as if a flood of anonymous clones held every slot. state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); // Write pool keeps its default capacity from test_state_lazy. @@ -456,10 +460,13 @@ mod tests { .await .unwrap(); - assert_ne!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = axum::body::to_bytes(resp.into_body(), 1024).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"], + crate::error::DB_UNAVAILABLE_CODE, + "the push must clear admission and reach the deliberately closed database" ); } @@ -7857,14 +7864,10 @@ mod tests { seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; // A git that takes 300ms per invocation (the read makes two: type, then content). - let slow_git = std::env::temp_dir().join(format!("gl-slow-git-{short}")); - std::fs::write(&slow_git, "#!/bin/sh\nsleep 0.3\nexec git \"$@\"\n").expect("write shim"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&slow_git, std::fs::Permissions::from_mode(0o755)) - .expect("chmod shim"); - } + let slow_git = git_shim::create( + &format!("gl-slow-git-{short}"), + git_shim::Behavior::Delay(300), + ); let ticks = std::sync::Arc::new(AtomicUsize::new(0)); let ticker = { @@ -8640,19 +8643,6 @@ mod tests { ); } - /// Write an executable `git` stand-in and return its path. - fn write_git_shim(name: &str, script: &str) -> std::path::PathBuf { - let path = std::env::temp_dir().join(name); - std::fs::write(&path, script).expect("write the git shim"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) - .expect("chmod the git shim"); - } - path - } - /// F6 scenario 1 (#173 round 13): one hung candidate must not starve the rows behind /// it in the same pass. `DiscoveryCtx` is loaded once per pass, so before the per-row /// slice every source-less row in a pass shared ONE deadline: the first row's wedged @@ -8692,23 +8682,9 @@ mod tests { // The type stage feeds the oid on STDIN (`cat-file --batch-check`) and the // content stage puts it in argv, so the stand-in has to look in both places. - let git_bin = write_git_shim( + let git_bin = git_shim::create( &format!("gl-hung-git-{short}"), - &format!( - "#!/bin/sh\n\ - if [ \"$2\" = \"--batch-check\" ]; then\n\ - \x20 oid=$(cat)\n\ - \x20 case \"$oid\" in\n\ - \x20 {hung_oid}) sleep 30; exit 1 ;;\n\ - \x20 esac\n\ - \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ - \x20 exit $?\n\ - fi\n\ - case \"$*\" in\n\ - \x20 *{hung_oid}*) sleep 30; exit 1 ;;\n\ - esac\n\ - exec git \"$@\"\n" - ), + git_shim::Behavior::HangOid(&hung_oid), ); let stats = tokio::time::timeout( @@ -8793,10 +8769,7 @@ mod tests { // Wedges on every invocation, so no row can ever be repaired and the only // question left is what each one COSTS. - let git_bin = write_git_shim( - &format!("gl-spent-git-{short}"), - "#!/bin/sh\nsleep 30\nexit 1\n", - ); + let git_bin = git_shim::create(&format!("gl-spent-git-{short}"), git_shim::Behavior::Hang); let stats = tokio::time::timeout( std::time::Duration::from_secs(60), @@ -9400,10 +9373,7 @@ mod tests { seed_legacy_pin(&pool, &src, oid, None).await; } - let git_bin = write_git_shim( - &format!("gl-starve-git-{short}"), - "#!/bin/sh\nsleep 30\nexit 1\n", - ); + let git_bin = git_shim::create(&format!("gl-starve-git-{short}"), git_shim::Behavior::Hang); tokio::time::timeout( std::time::Duration::from_secs(120), @@ -9471,9 +9441,9 @@ mod tests { let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; // Wedges only inside the position-nine repo, which the sweep enters by cwd. - let git_bin = write_git_shim( + let git_bin = git_shim::create( &format!("gl-mid-git-{short}"), - "#!/bin/sh\ncase \"$(pwd)\" in\n */midcand9.git) sleep 30; exit 1 ;;\nesac\nexec git \"$@\"\n", + git_shim::Behavior::HangRepo("midcand9.git"), ); let first = tokio::time::timeout( @@ -9635,19 +9605,9 @@ mod tests { let log = std::env::temp_dir().join(format!("gl-ali-log-{short}")); let _ = std::fs::remove_file(&log); - let git_bin = write_git_shim( + let git_bin = git_shim::create( &format!("gl-ali-git-{short}"), - &format!( - "#!/bin/sh\n\ - if [ \"$2\" = \"--batch-check\" ]; then\n\ - \x20 oid=$(cat)\n\ - \x20 printf '%s %s\\n' \"$oid\" \"$(basename $(pwd))\" >> {log}\n\ - \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ - \x20 exit $?\n\ - fi\n\ - exec git \"$@\"\n", - log = log.display() - ), + git_shim::Behavior::LogTypes(&log), ); let mut traversal = crate::ipfs_pin::DiscoveryTraversalState::default(); diff --git a/crates/gitlawb-node/tests/fixtures/git_shim.rs b/crates/gitlawb-node/tests/fixtures/git_shim.rs new file mode 100644 index 00000000..162de46e --- /dev/null +++ b/crates/gitlawb-node/tests/fixtures/git_shim.rs @@ -0,0 +1,59 @@ +use std::io::{Read, Write}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +fn main() -> std::io::Result<()> { + let executable = std::env::current_exe()?; + let config = std::fs::read_to_string(executable.parent().unwrap().join("config"))?; + let (mode, parameter) = config.split_once('\n').unwrap_or((&config, "")); + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if mode == "delay" { + std::thread::sleep(Duration::from_millis(parameter.parse().unwrap())); + } + let batch = args.get(1).is_some_and(|arg| arg == "--batch-check"); + let input = if batch { + let mut bytes = Vec::new(); + std::io::stdin().read_to_end(&mut bytes)?; + Some(bytes) + } else { + None + }; + let cwd = std::env::current_dir()?; + let hang = mode == "hang" + || (mode == "hang-repo" && cwd.file_name().is_some_and(|name| name == parameter)) + || (mode == "hang-oid" + && (args.iter().any(|arg| arg == parameter) + || input + .as_ref() + .is_some_and(|bytes| String::from_utf8_lossy(bytes).trim() == parameter))); + if hang { + std::thread::sleep(Duration::from_secs(30)); + std::process::exit(1); + } + if mode == "log-types" && batch { + let mut log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(parameter)?; + writeln!( + log, + "{} {}", + String::from_utf8_lossy(input.as_ref().unwrap()).trim(), + cwd.file_name().unwrap().to_string_lossy() + )?; + } + let mut child = Command::new("git") + .args(&args) + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::inherit() + }) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(bytes) = input { + child.stdin.take().unwrap().write_all(&bytes)?; + } + std::process::exit(child.wait()?.code().unwrap_or(1)); +} From 5d238cadbc5acaf00d9bb284c3e5e8d2a5fbc50c Mon Sep 17 00:00:00 2001 From: euxaristia Date: Sun, 13 Sep 2026 23:39:30 -0400 Subject: [PATCH 17/17] fix(sync): use a supported Windows promisor blob filter Git for Windows rejects the 10 GiB filter. Use 4 GiB minus one byte on Windows for clone and fetch, retain 10 GiB on other targets, and document that larger Windows blobs remain available through on-demand fetching. Refs #95 Refs #407 --- README.md | 4 ++++ crates/gitlawb-node/src/sync.rs | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9b6f1e0c..a0e0f35d 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,10 @@ Or build from source: cargo build --release -p gl -p git-remote-gitlawb -p gitlawb-node ``` +Node promisor mirrors use a 10 GiB blob filter on Unix. Git for Windows requires +a filter below 4 GiB, so Windows mirrors use 4 GiB minus one byte; larger blobs +remain available through on-demand fetching. + Put these binaries on your `PATH`: ```txt diff --git a/crates/gitlawb-node/src/sync.rs b/crates/gitlawb-node/src/sync.rs index 62fa898a..9111b84c 100644 --- a/crates/gitlawb-node/src/sync.rs +++ b/crates/gitlawb-node/src/sync.rs @@ -691,15 +691,22 @@ async fn existing_promisor_state(repo: &str) -> PromisorProbe { } } -/// Mirror-clone a repo from a remote URL into a local bare repo. -/// `Promisor` mode adds `--filter=blob:limit=10g`, which marks the repo a git -/// promisor (so a pack with origin-omitted withheld blobs is accepted) while -/// the huge size limit means every blob the origin *does* send is kept. +// Git for Windows parses blob limits as a 32-bit unsigned long. Larger +// Windows blobs remain available through promisor on-demand fetching. +#[cfg(windows)] +const PROMISOR_BLOB_FILTER: &str = "blob:limit=4294967295"; +#[cfg(not(windows))] +const PROMISOR_BLOB_FILTER: &str = "blob:limit=10g"; + +/// Mirror-clone a repo, marking filtered mirrors as promisors so packs that +/// omit withheld blobs are accepted. The filter threshold is 10 GiB on Unix +/// and 4 GiB minus one byte on Windows; larger blobs are fetched on demand. async fn clone_repo(remote_url: &str, local_path: &Path, mode: MirrorMode) -> anyhow::Result<()> { let local_str = local_path.to_str().unwrap_or("."); + let filter_arg = format!("--filter={PROMISOR_BLOB_FILTER}"); let mut args = vec!["clone", "--mirror"]; if mode == MirrorMode::Promisor { - args.push("--filter=blob:limit=10g"); + args.push(&filter_arg); } args.push(remote_url); args.push(local_str); @@ -739,7 +746,7 @@ async fn fetch_repo(local_path: &Path, remote_url: &str, mode: MirrorMode) -> an local_str, "config", "remote.origin.partialclonefilter", - "blob:limit=10g", + PROMISOR_BLOB_FILTER, ]) .await?; git_run(&["-C", local_str, "fetch", "--prune", "origin"]).await