diff --git a/.env.example b/.env.example index 81c60824..ba93aa73 100644 --- a/.env.example +++ b/.env.example @@ -125,6 +125,8 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # rather than giving each stage a full budget: a walk that consumes it leaves the # serve nothing and the clone gets a 504. Serving large path-scoped repos may # need a higher value here than when each stage was budgeted separately. +# REST blob size probes and content reads also share this deadline and return 504 on timeout. +# Body delivery gets the same time allowance; expiry aborts the response and releases admission. # Must be 1..=3153600000 (100 years): the node derives deadlines from this value, # and a larger one cannot be represented. Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 @@ -139,8 +141,10 @@ GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 # cost centers. Must be positive; set very large to effectively disable. Default 30. GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 -# Max concurrent git READ ops (upload-pack + the upload-pack info/refs -# advertisement) served at once, a global pool separate from the push pool below. +# Max concurrent git READ ops (upload-pack, its info/refs advertisement, and REST +# blob reads) served at once, a global pool separate from the push pool below. +# REST blobs also have a fixed four-response sub-pool and a 32 MiB per-response +# ceiling; their permits remain held until the response body finishes or disconnects. # The anon receive-pack info/refs advertisement has its OWN pool (see below), not # this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from # here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one @@ -182,6 +186,7 @@ GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 # collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a # high-fanout caller (CI behind one NAT) then needs the operator to raise this. # Default 16. +# REST blob downloads also acquire this per-caller read allowance. GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 # Two further per-source concurrency caps exist on the PUSH side but have NO diff --git a/README.md b/README.md index 3a092bf2..a0e0f35d 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,10 @@ Or build from source: cargo build --release -p gl -p git-remote-gitlawb -p gitlawb-node ``` +Node promisor mirrors use a 10 GiB blob filter on Unix. Git for Windows requires +a filter below 4 GiB, so Windows mirrors use 4 GiB minus one byte; larger blobs +remain available through on-demand fetching. + Put these binaries on your `PATH`: ```txt @@ -340,6 +344,11 @@ GET /{owner}/{repo}/info/refs POST /{owner}/{repo}/git-upload-pack ``` +REST blob reads serve file content only; directory and gitlink paths return 404. +Body delivery has its own allowance equal to `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`. +If delivery exceeds it, the response is interrupted and its admission slots are released. +Blob responses use `Cache-Control: no-store` because they may contain private content. + Signed write routes include: ```txt @@ -396,9 +405,9 @@ Important node settings: | `GITLAWB_ENFORCE_OWNER_PUSH` | Require the authenticated pusher to be the repo owner on `git-receive-pack`. **Defaults to `true`.** A `did:key` signature is authentication, not authorization — anyone can mint a key and sign — so with this off every signed caller may push to every repository, private ones included. Delegated and CI keys count as non-owners: a UCAN `git/push` capability is verified but not yet honored for authorization, so they cannot push while this is on. Set `false` only for a rolling upgrade; see [`docs/RUN-A-NODE.md`](docs/RUN-A-NODE.md). | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, `info/refs` advertisement, or REST blob read may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. On the path-scoped upload-pack path the classification walk and the pack serve share ONE deadline, so this value bounds their combined duration rather than granting each stage a full budget: a walk that consumes it leaves the serve nothing and the clone gets a 504. Serving large path-scoped repos may therefore need a higher value than they did when each stage was budgeted separately. Accepted range is 1 to 3153600000 (100 years), since the node derives deadlines from this value and a larger one cannot be represented. | | `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | -| `GITLAWB_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack and its `info/refs` advertisement) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. | +| `GITLAWB_MAX_CONCURRENT_GIT_OPS` | Max concurrent served git READ ops (upload-pack, its `info/refs` advertisement, and REST blob reads) across all callers; over-cap sheds a 503 + Retry-After. Anonymous reads draw from this pool, so pair it with `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER`. Pushes and the receive-pack advertisement have their own pools, so a read flood cannot shed an authenticated push. Default 128. REST blob responses are limited to 32 MiB each and a dedicated four-request pool holds admission through body delivery, bounding retained blob data to 128 MiB. | | `GITLAWB_MAX_CONCURRENT_GIT_PUSHES` | Max concurrent `git-receive-pack` POST operations, in a pool separate from the read pool. The anon receive-pack `info/refs` advertisement runs in a third pool of the same size, disjoint from both, so an advertisement flood cannot shed a push either. Two per-source push caps are derived from this value (`/8`, floor 1) and have no env var of their own. Over-cap sheds a 503 + Retry-After. Default 32. | | `GITLAWB_MAX_CONCURRENT_READS_PER_CALLER` | Max concurrent read ops a single caller may hold, so one caller cannot monopolize the read pool. Keyed on the resolved source IP, never the DID, and only as granular as `GITLAWB_TRUSTED_PROXY`: left unset, a node behind an edge or NAT keys every caller on the edge IP and this collapses to one global cap. Default 16. | | `GITLAWB_MAX_CONCURRENT_PIN_TASKS` | Max post-push pin loops (IPFS + Pinata) running concurrently across all repos. This caps how many loops RUN at once, not how much object-id list memory the node retains: on the local IPFS path a loop parked waiting for a permit still holds its full list. Do not size memory from this knob alone. A loop over cap waits, never drops a pin. Default 8. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index 92d12980..40139804 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -8338,6 +8338,7 @@ mod tests { #[tokio::test] async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { let mut state = crate::test_support::test_state_lazy(); + state.db.pool().close().await; // Global pool has room; the per-source cap is 1. state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); @@ -8364,16 +8365,23 @@ mod tests { "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" ); + let bytes = axum::body::to_bytes(resp.into_body(), 1024).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "overloaded"); + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and - // proceeds (then errors on the lazy DB, which is not a 503). + // proceeds to the closed DB, which has a distinct db_unavailable error code. let resp = ipfs_router(state) .oneshot(get_cid(&cid, Some(other))) .await .unwrap(); - assert_ne!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "a different source must not be shed by the per-source cap" + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = axum::body::to_bytes(resp.into_body(), 1024).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"], + crate::error::DB_UNAVAILABLE_CODE, + "a different source must clear admission and reach the closed database" ); } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index 4e327c42..f4a0aafc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -2,7 +2,7 @@ use axum::extract::{Extension, Path, Query, State}; use axum::http::StatusCode; use axum::response::Response; use axum::Json; -use bytes::Bytes; +use bytes::{Buf, Bytes}; use std::sync::Arc; use crate::auth::{caller_authorized_to_push, AuthenticatedDid}; @@ -21,6 +21,107 @@ use crate::webhooks; /// The git all-zeros object id — the create/delete sentinel in a ref update. const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; +/// REST blob responses share the same served-object ceiling as `/ipfs/{cid}`. +const MAX_SERVED_BLOB_BYTES: u64 = crate::api::ipfs::MAX_SERVED_OBJECT_BYTES; + +/// A dedicated pool bounds retained REST blob bodies independently of smart-HTTP +/// traffic. At the per-response ceiling, the default caps live blob data at 128 MiB. +pub(crate) const MAX_CONCURRENT_BLOB_READS: usize = 4; + +const BLOB_RESPONSE_CHUNK_BYTES: usize = 64 * 1024; + +struct BlobResponseStream { + content: Bytes, + _git_permit: tokio::sync::OwnedSemaphorePermit, + _blob_permit: tokio::sync::OwnedSemaphorePermit, + _caller_permit: Option, +} + +impl futures::Stream for BlobResponseStream { + type Item = std::result::Result; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let this = self.get_mut(); + if this.content.is_empty() { + return std::task::Poll::Ready(None); + } + let len = this.content.len().min(BLOB_RESPONSE_CHUNK_BYTES); + let chunk = Bytes::copy_from_slice(&this.content[..len]); + this.content.advance(len); + std::task::Poll::Ready(Some(Ok(chunk))) + } +} + +/// The producer owns admission and the full blob; its timer runs even when +/// the HTTP server stops polling the body. Only one copied chunk is queued. +struct BlobDeliveryStream { + receiver: tokio::sync::mpsc::Receiver, + producer: tokio::task::JoinHandle>, + finished: bool, +} + +impl BlobDeliveryStream { + fn new(mut source: BlobResponseStream, timeout: std::time::Duration) -> Self { + let (sender, receiver) = tokio::sync::mpsc::channel(1); + let producer = tokio::spawn(async move { + use futures::StreamExt; + let delivery = async { + while let Some(Ok(chunk)) = source.next().await { + if sender.send(chunk).await.is_err() { + return; + } + } + // Keep admission until the last queued chunk is consumed, even + // for a one-chunk response whose first send never had to wait. + let _drained = sender.reserve().await; + }; + tokio::time::timeout(timeout, delivery).await.map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "blob delivery timed out") + }) + }); + Self { + receiver, + producer, + finished: false, + } + } +} + +impl futures::Stream for BlobDeliveryStream { + type Item = std::io::Result; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + use std::future::Future; + use std::task::Poll; + let this = self.get_mut(); + if this.finished { + return Poll::Ready(None); + } + if let Some(chunk) = std::task::ready!(this.receiver.poll_recv(cx)) { + return Poll::Ready(Some(Ok(chunk))); + } + let result = std::task::ready!(std::pin::Pin::new(&mut this.producer).poll(cx)); + this.finished = true; + match result { + Ok(Ok(())) => Poll::Ready(None), + Ok(Err(error)) => Poll::Ready(Some(Err(error))), + Err(_) => Poll::Ready(Some(Err(std::io::Error::other("blob delivery failed")))), + } + } +} + +impl Drop for BlobDeliveryStream { + fn drop(&mut self) { + self.producer.abort(); + } +} + /// The set of blob OIDs withheld from **anonymous** replication for a repo, or /// `None` when the repo must not replicate at all (private / mode A / /// undetermined — fail closed). This is the anonymous replication gate: @@ -413,48 +514,100 @@ pub async fn list_commits( pub async fn get_blob( State(state): State, Path((owner, name, file_path)): Path<(String, String, String)>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, auth: Option>, ) -> Result { use axum::http::header; - use axum::response::IntoResponse; - // Unnormalized paths ("../..", "./", "//") can't resolve in `git show` + // Unnormalized paths ("../..", "./", "//") are invalid in the ref:path + // input sent to `git cat-file --batch-check`, // and crawlers combinatorially explode them from relative links — that's // a client error, not a 500. let file_path = file_path.trim_matches('/'); if file_path.is_empty() - || file_path - .split('/') - .any(|seg| seg.is_empty() || seg == "." || seg == "..") + || file_path.split('/').any(|seg| { + seg.is_empty() || seg == "." || seg == ".." || seg.chars().any(char::is_control) + }) { return Err(AppError::BadRequest("invalid file path".into())); } + // Charge the source and acquire blob/global read permits before any database work. + // The OwnedSemaphorePermits release on drop if authorization or subsequent work fails. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + &name, + "REST blob", + )?; + let blob_permit = git_permit(&state.git_blob_semaphore)?; + let permit = git_permit(&state.git_read_semaphore)?; + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let gate_path = format!("/{file_path}"); let (record, _rules) = crate::api::authorize_repo_read(&state, &owner, &name, caller, &gate_path).await?; - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; - let head_ref = store::resolve_head(&disk_path, &record.default_branch); - let content = store::read_file(&disk_path, &head_ref, file_path).map_err(|e| { - let msg = e.to_string(); - // `git show ref:path` on a path absent from the tree is a 404, - // not a server error - if msg.contains("does not exist in") - || msg.contains("invalid object name") - || msg.contains("exists on disk, but not in") - { - AppError::NotFound(format!("file not found: {file_path}")) - } else { - AppError::Git(msg) + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding blob request with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(AppError::Internal)?; + + let default_branch = record.default_branch.clone(); + let read_path = file_path.to_string(); + let git_bin = state.git_bin.clone(); + let deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let (read, permit, blob_permit, caller_permit) = tokio::task::spawn_blocking(move || { + let read = store::read_file_bounded( + &git_bin, + &disk_path, + &default_branch, + &read_path, + MAX_SERVED_BLOB_BYTES, + deadline, + ); + (read, permit, blob_permit, caller_permit) + }) + .await + .map_err(|e| AppError::Internal(anyhow::anyhow!("blob read task failed: {e}")))?; + let read = read.map_err(|e| { + if e.downcast_ref::().is_some() { + return AppError::Timeout("git service timed out".into()); + } + if matches!( + e.downcast_ref::(), + Some(store::ProbeError::Transient(_)) + ) { + return AppError::Overloaded( + "object store temporarily unavailable, retry shortly".into(), + ); } + AppError::Internal(e) })?; + let content = match read { + store::BoundedFileRead::Found(content) => content, + store::BoundedFileRead::Missing => { + return Err(AppError::NotFound(format!("file not found: {file_path}"))); + } + store::BoundedFileRead::TooLarge { size, max } => { + tracing::warn!(repo = %name, path = %file_path, size, max, "REST blob exceeds served size limit"); + return Err(AppError::PayloadTooLarge(format!( + "file exceeds the maximum served size of {max} bytes" + ))); + } + }; + // Guess content type let mime = match file_path.rsplit('.').next() { Some("html") => "text/html; charset=utf-8", @@ -467,7 +620,32 @@ pub async fn get_blob( _ => "application/octet-stream", }; - Ok(([(header::CONTENT_TYPE, mime)], content).into_response()) + let content_len = content.len(); + let stream = BlobResponseStream { + content: Bytes::from(content), + _git_permit: permit, + _blob_permit: blob_permit, + _caller_permit: caller_permit, + }; + let stream = BlobDeliveryStream::new( + stream, + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ); + let mut response = Response::new(axum::body::Body::from_stream(stream)); + response.headers_mut().insert( + header::CACHE_CONTROL, + axum::http::HeaderValue::from_static("no-store"), + ); + response.headers_mut().insert( + header::CONTENT_TYPE, + axum::http::HeaderValue::from_static(mime), + ); + response.headers_mut().insert( + header::CONTENT_LENGTH, + axum::http::HeaderValue::from_str(&content_len.to_string()) + .expect("a decimal content length is a valid header value"), + ); + Ok(response) } /// GET /api/v1/repos/:owner/:repo/tree (root listing) @@ -3343,6 +3521,103 @@ mod tests { const OWNER_SHORT: &str = "z6MkpTHR8VNsBxYAAWHut2Geadd9jSwuBV8xRoAnwWsdvktH"; const STRANGER_DID: &str = "did:key:z6Mkffonly5tranger0000000000000000000000000000000"; + #[tokio::test] + async fn blob_response_holds_admission_until_the_body_is_dropped() { + use futures::StreamExt; + + let git = Arc::new(tokio::sync::Semaphore::new(1)); + let blob = Arc::new(tokio::sync::Semaphore::new(1)); + let callers = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let source = vec![b'x'; BLOB_RESPONSE_CHUNK_BYTES * 2]; + let source_start = source.as_ptr() as usize; + let source_end = source_start + source.len(); + let mut stream = BlobResponseStream { + content: Bytes::from(source), + _git_permit: git.clone().try_acquire_owned().unwrap(), + _blob_permit: blob.clone().try_acquire_owned().unwrap(), + _caller_permit: callers.try_acquire("source"), + }; + + let first = stream.next().await.unwrap().unwrap(); + assert_eq!(first.len(), BLOB_RESPONSE_CHUNK_BYTES); + assert!(git.clone().try_acquire_owned().is_err()); + assert!(blob.clone().try_acquire_owned().is_err()); + assert!(callers.try_acquire("source").is_none()); + + let second = stream.next().await.unwrap().unwrap(); + assert!(stream.next().await.is_none()); + for chunk in [&first, &second] { + let chunk_start = chunk.as_ptr() as usize; + assert!( + chunk_start < source_start || chunk_start >= source_end, + "emitted chunks must not retain the full source allocation" + ); + } + + drop(stream); + assert!(git.try_acquire_owned().is_ok()); + assert!(blob.try_acquire_owned().is_ok()); + assert!(callers.try_acquire("source").is_some()); + } + + #[tokio::test(start_paused = true)] + async fn blob_delivery_deadline_releases_admission_without_body_polls() { + use futures::StreamExt; + for (chunks, read_first_chunk) in [(1, false), (8, false), (8, true)] { + let git = Arc::new(tokio::sync::Semaphore::new(1)); + let blob = Arc::new(tokio::sync::Semaphore::new(1)); + let callers = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let source = BlobResponseStream { + content: Bytes::from(vec![b'x'; BLOB_RESPONSE_CHUNK_BYTES * chunks]), + _git_permit: git.clone().try_acquire_owned().unwrap(), + _blob_permit: blob.clone().try_acquire_owned().unwrap(), + _caller_permit: callers.try_acquire("source"), + }; + let mut stream = BlobDeliveryStream::new(source, std::time::Duration::from_secs(10)); + tokio::task::yield_now().await; + if read_first_chunk { + assert_eq!( + stream.next().await.unwrap().unwrap().len(), + BLOB_RESPONSE_CHUNK_BYTES + ); + tokio::task::yield_now().await; + } + assert_eq!(blob.available_permits(), 0); + tokio::time::advance(std::time::Duration::from_secs(11)).await; + tokio::task::yield_now().await; + // The body still exists and has not been polled during the deadline. + assert_eq!(git.available_permits(), 1); + assert_eq!(blob.available_permits(), 1); + assert!(callers.try_acquire("source").is_some()); + assert!(stream.next().await.unwrap().is_ok()); // one queued chunk + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert!(stream.next().await.is_none()); + } + } + + #[tokio::test(start_paused = true)] + async fn blob_delivery_disconnect_releases_admission_before_deadline() { + let git = Arc::new(tokio::sync::Semaphore::new(1)); + let blob = Arc::new(tokio::sync::Semaphore::new(1)); + let callers = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let stream = BlobDeliveryStream::new( + BlobResponseStream { + content: Bytes::from(vec![b'x'; BLOB_RESPONSE_CHUNK_BYTES * 8]), + _git_permit: git.clone().try_acquire_owned().unwrap(), + _blob_permit: blob.clone().try_acquire_owned().unwrap(), + _caller_permit: callers.try_acquire("source"), + }, + std::time::Duration::from_secs(600), + ); + tokio::task::yield_now().await; + drop(stream); + tokio::task::yield_now().await; + assert_eq!(git.available_permits(), 1); + assert_eq!(blob.available_permits(), 1); + assert!(callers.try_acquire("source").is_some()); + } + #[test] fn upload_pack_request_finalizes_only_with_done_pktline() { let want = "0032want 1111111111111111111111111111111111111111\n"; @@ -3488,6 +3763,392 @@ mod tests { ); } + #[cfg(unix)] + #[sqlx::test] + async fn blob_cat_file_failure_is_opaque(pool: sqlx::PgPool) { + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + for (index, failure) in [ + "echo 'private-git-stderr /internal/repo.git' >&2; exit 1", + "echo 'error: private-git-stderr /internal/repo.git' >&2; exit 0", + "if [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 blob 1'; exit 0; fi\necho 'private-git-stderr /internal/repo.git' >&2; exit 1", + ].iter().enumerate() { + let fake = write_fake_git(tmp.path(), &format!( + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\n{failure}\n" + )); + let owner = format!("z6bloberror{index}"); + let state = f4_state_with_repo(pool.clone(), tmp.path(), &fake, &owner, "repo", false).await; + let response = blob_route_request(state, &owner, "203.0.113.31:5000").await; + assert_eq!(response.status(), axum::http::StatusCode::INTERNAL_SERVER_ERROR); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["message"], crate::error::INTERNAL_ERROR_MESSAGE); + let body = body.to_string(); + assert!(!body.contains("private-git-stderr")); + assert!(!body.contains("/internal/repo.git")); + } + } + + async fn blob_route_request_path( + state: AppState, + owner: &str, + path: &str, + peer: &str, + ) -> Response { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::Request; + use tower::ServiceExt; + let router = axum::Router::new() + .route( + "/repos/{owner}/{repo}/blob/{*path}", + axum::routing::get(get_blob), + ) + .with_state(state); + let mut request = Request::builder() + .uri(format!("/repos/{owner}/repo/blob/{path}")) + .body(Body::empty()) + .unwrap(); + request + .extensions_mut() + .insert(ConnectInfo(peer.parse::().unwrap())); + router.oneshot(request).await.unwrap() + } + + async fn blob_route_request(state: AppState, owner: &str, peer: &str) -> Response { + blob_route_request_path(state, owner, "file.txt", peer).await + } + + #[sqlx::test] + async fn blob_acquire_failure_is_opaque(pool: sqlx::PgPool) { + use http_body_util::BodyExt; + let mut state = crate::test_support::test_state(pool.clone()).await; + state + .db + .upsert_mirror_repo("z6blobacquire", "repo", "/unused", None, false) + .await + .unwrap(); + // An invalid configured storage root makes the real acquire() fail before Git. + let tmp = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + tmp.path().join("private-storage").join("..").join("repos"), + pool, + ); + let response = blob_route_request(state, "z6blobacquire", "203.0.113.31:5000").await; + assert_eq!( + response.status(), + axum::http::StatusCode::INTERNAL_SERVER_ERROR + ); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["message"], crate::error::INTERNAL_ERROR_MESSAGE); + assert!(!body.to_string().contains("parent-directory")); + } + + #[tokio::test] + async fn blob_capacity_sheds_before_database_access() { + use axum::http::StatusCode; + use http_body_util::BodyExt; + for capacity in ["caller", "blob", "global"] { + let mut state = crate::test_support::test_state_lazy(); + // A misplaced DB lookup returns db_unavailable immediately, without a timeout. + state.db.pool().close().await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + let caller_slot = (capacity == "caller").then(|| { + state + .git_read_per_caller + .try_acquire("203.0.113.31") + .unwrap() + }); + if capacity == "blob" { + state.git_blob_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(0)); + } + if capacity == "global" { + state.git_read_semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(0)); + } + let response = + blob_route_request(state.clone(), "z6blobcap", "203.0.113.31:5000").await; + assert_eq!( + response.status(), + StatusCode::SERVICE_UNAVAILABLE, + "{capacity}" + ); + assert_eq!(response.headers()[axum::http::header::RETRY_AFTER], "1"); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "overloaded", "{capacity}"); + if capacity == "caller" { + let other = + blob_route_request(state.clone(), "z6blobcap", "203.0.113.32:5000").await; + assert_eq!(other.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = other.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], crate::error::DB_UNAVAILABLE_CODE); + drop(caller_slot); + let released = blob_route_request(state, "z6blobcap", "203.0.113.31:5000").await; + assert_eq!(released.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = released.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], crate::error::DB_UNAVAILABLE_CODE); + } + } + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_maps_oversize_and_timeout(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + for (owner, command, expected) in [ + ( + "z6blobsize", + format!( + "echo '1111111111111111111111111111111111111111 blob {}'", + MAX_SERVED_BLOB_BYTES + 1 + ), + StatusCode::PAYLOAD_TOO_LARGE, + ), + ( + "z6blobtime", + "exec sleep 30".into(), + StatusCode::GATEWAY_TIMEOUT, + ), + ] { + let fake = write_fake_git( + tmp.path(), + &format!("#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\n{command}\n"), + ); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &fake, owner, "repo", false).await; + let mut config = (*state.config).clone(); + config.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(config); + let response = tokio::time::timeout( + std::time::Duration::from_secs(10), + blob_route_request(state.clone(), owner, "203.0.113.31:5000"), + ) + .await + .expect("blob deadline must terminate the Git child"); + assert_eq!(response.status(), expected); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"], + if expected == StatusCode::PAYLOAD_TOO_LARGE { + "payload_too_large" + } else { + "git_timeout" + } + ); + assert_eq!(state.git_read_semaphore.available_permits(), 64); + assert_eq!( + state.git_blob_semaphore.available_permits(), + MAX_CONCURRENT_BLOB_READS + ); + assert_eq!(state.git_read_per_caller.tracked_keys(), 0); + } + } + + #[tokio::test] + async fn blob_route_rejects_invalid_paths() { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let state = crate::test_support::test_state_lazy(); + for invalid_path in [ + "file%01.txt", + "file%1f.txt", + "foo/%2e%2e/bar.txt", + "foo/%2e/bar.txt", + "foo//bar.txt", + ] { + let response = + blob_route_request_path(state.clone(), "z6test", invalid_path, "203.0.113.31:5000") + .await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "path: {invalid_path:?}" + ); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "bad_request"); + } + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_returns_not_found_on_missing_blob(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git( + tmp.path(), + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo 'missing'; exit 0; fi\nexit 1\n", + ); + let state = + f4_state_with_repo(pool, tmp.path(), &fake, "z6blobmissing", "repo", false).await; + let response = + blob_route_request_path(state, "z6blobmissing", "missing.txt", "203.0.113.31:5000") + .await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "not_found"); + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_delivery_uses_configured_timeout(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git(tmp.path(), "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 blob 524288'; exit 0; fi\nhead -c 524288 /dev/zero\n"); + let mut state = + f4_state_with_repo(pool, tmp.path(), &fake, "z6blobdelivery", "repo", false).await; + let mut config = (*state.config).clone(); + config.git_service_timeout_secs = 1; + state.config = Arc::new(config); + let response = blob_route_request_path( + state.clone(), + "z6blobdelivery", + "file.txt", + "203.0.113.31:5000", + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert!(state.git_blob_semaphore.available_permits() < MAX_CONCURRENT_BLOB_READS); + // Keep the body alive without ever polling it; the timer must run anyway. + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while state.git_blob_semaphore.available_permits() != MAX_CONCURRENT_BLOB_READS { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("configured deadline must release blob admission"); + use http_body_util::BodyExt; + assert!(response.into_body().collect().await.is_err()); + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_unreadable_pack_is_opaque_error(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git(tmp.path(), "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nread spec\necho \"$spec missing\"\n"); + let state = + f4_state_with_repo(pool, tmp.path(), &fake, "z6blobunreadable", "repo", false).await; + let record = state + .db + .get_repo("z6blobunreadable", "repo") + .await + .unwrap() + .unwrap(); + let path = state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + .unwrap(); + std::os::unix::fs::symlink("removed-pack", path.join("objects/pack/unreadable.pack")) + .unwrap(); + let response = + blob_route_request_path(state, "z6blobunreadable", "file.txt", "203.0.113.31:5000") + .await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + use http_body_util::BodyExt; + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "overloaded"); + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_returns_not_found_on_non_blob_path(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git( + tmp.path(), + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 tree 1024'; exit 0; fi\nexit 1\n", + ); + let state = f4_state_with_repo(pool, tmp.path(), &fake, "z6blobtree", "repo", false).await; + let response = + blob_route_request_path(state, "z6blobtree", "somedir", "203.0.113.31:5000").await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "not_found"); + } + + #[sqlx::test] + async fn blob_route_acquire_timeout_sheds_503_and_releases_permits(pool: sqlx::PgPool) { + use axum::http::StatusCode; + use http_body_util::BodyExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().to_path_buf(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let endpoint = crate::test_support::silent_http_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + let lock_pool = + crate::git::repo_store::build_lock_pool(&pool, 4, std::time::Duration::from_secs(5)); + state.repo_store = + crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), lock_pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Repo exists in DB but not on disk, so acquire attempts Tigris and times out. + state + .db + .upsert_mirror_repo("z6blobacqtimeout", "repo", "/unused", None, false) + .await + .unwrap(); + + let response = blob_route_request_path( + state.clone(), + "z6blobacqtimeout", + "file.txt", + "203.0.113.31:5000", + ) + .await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], "overloaded"); + assert_eq!(state.git_read_semaphore.available_permits(), 64); + assert_eq!( + state.git_blob_semaphore.available_permits(), + MAX_CONCURRENT_BLOB_READS + ); + assert_eq!(state.git_read_per_caller.tracked_keys(), 0); + } + + #[cfg(unix)] + #[sqlx::test] + async fn blob_route_returns_200_with_expected_headers_and_content(pool: sqlx::PgPool) { + use axum::http::{header, StatusCode}; + use http_body_util::BodyExt; + let tmp = tempfile::TempDir::new().unwrap(); + let fake = write_fake_git( + tmp.path(), + "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nif [ \"$2\" = --batch-check ]; then echo '1111111111111111111111111111111111111111 blob 13'; exit 0; fi\nprintf '{\"hello\":123}'; exit 0\n", + ); + let state = f4_state_with_repo(pool, tmp.path(), &fake, "z6blobhappy", "repo", false).await; + let response = + blob_route_request_path(state, "z6blobhappy", "data.json", "203.0.113.31:5000").await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers()[header::CONTENT_TYPE], + "application/json; charset=utf-8" + ); + assert_eq!(response.headers()[header::CONTENT_LENGTH], "13"); + assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store"); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&bytes[..], b"{\"hello\":123}"); + } + #[test] fn git_service_app_error_classifies_timeout_bad_request_and_git() { // GitServiceTimeout carried through anyhow -> 504 Timeout. @@ -7124,6 +7785,7 @@ mod tests { /// A pkt-line receive-pack body carrying one branch-create ref update, so the /// handler's post-receive tail resolves a non-empty new-tip set (the delta /// scan's git stages run). + #[cfg(unix)] fn ref_update_body(new_sha: &str) -> axum::body::Bytes { let line = format!("{ZERO_SHA} {new_sha} refs/heads/main"); axum::body::Bytes::from(format!("{:04x}{}0000", line.len() + 4, line)) @@ -9581,6 +10243,7 @@ mod tests { ) } + #[cfg(unix)] fn f2a_log(log: &std::path::Path) -> String { std::fs::read_to_string(log).unwrap_or_default() } @@ -9588,6 +10251,7 @@ mod tests { /// Withheld-walk children run so far. `ls-tree` is the walk's signature child /// (`blob_paths` lists every reachable commit's tree); the delta scan and the /// full-scan fallback use `rev-list` / `cat-file` instead. + #[cfg(unix)] fn f2a_walks(log: &std::path::Path) -> usize { f2a_log(log) .lines() @@ -9599,6 +10263,7 @@ mod tests { /// the withheld walk actually runs rather than taking the no-rule shortcut). /// The repo's on-disk path is passed to the tail directly, so no repo_store or /// receive-pack plumbing is involved. + #[cfg(unix)] async fn f2a_state( pool: sqlx::PgPool, git_bin: &str, @@ -9630,6 +10295,7 @@ mod tests { (state, rec) } + #[cfg(unix)] fn f2a_update(ref_name: &str, new_sha: &str) -> Vec { vec![RefUpdate { old_sha: ZERO_SHA.to_string(), @@ -9638,6 +10304,7 @@ mod tests { }] } + #[cfg(unix)] const F2A_PUSHER: &str = "did:key:z6MkF2aPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; /// Scenario 1 (the finding). A second rapid push to the same repo coalesces @@ -9711,6 +10378,7 @@ mod tests { } /// Poll `cond` until it holds, with a bound so a regression fails the test /// rather than hanging the suite. + #[cfg(unix)] async fn f2a_wait_for(mut cond: impl FnMut() -> bool, what: &str) { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); while !cond() { @@ -9725,6 +10393,7 @@ mod tests { /// A `rev-list --objects` line names the tips a DELTA scan was asked to resolve, /// so it attributes that scan to one push's tips. The withheld walk's own /// `rev-list --all` / `ls-tree` lines never carry a tip as an argument this way. + #[cfg(unix)] fn f2a_delta_scanned(log: &std::path::Path, tip: &str) -> bool { f2a_log(log) .lines() @@ -9871,6 +10540,7 @@ mod tests { /// Mount a Pinata upload endpoint that assigns every object the same CID, and /// point the state at it. Returns the server (kept alive by the caller) and CID. + #[cfg(unix)] async fn f2a_pinata(state: &mut AppState) -> (mockito::ServerGuard, String) { let cid = "bafyf2acoalescedmapping".to_string(); let mut server = mockito::Server::new_async().await; @@ -9890,6 +10560,7 @@ mod tests { /// Poll the branch to CID table until the push's mapping lands (the Pinata /// worker is detached), bounded so a regression fails rather than hangs. + #[cfg(unix)] async fn f2a_wait_for_branch_cid( db: &crate::db::Db, slug: &str, @@ -9909,6 +10580,7 @@ mod tests { } } + #[cfg(unix)] fn f2a_slug(rec: &crate::db::RepoRecord) -> String { format!( "{}/{}", @@ -10042,6 +10714,7 @@ mod tests { // and `z6p2fail`), and owner-only push is on by default, so the identity has to // follow the repo each push targets rather than being fixed for both. + #[cfg(unix)] fn p2_push( state: &AppState, owner: &str, @@ -10060,6 +10733,7 @@ mod tests { ) } + #[cfg(unix)] fn p2_logged(log: &std::path::Path, prefix: &str) -> bool { f2a_log(log).lines().any(|l| l.starts_with(prefix)) } @@ -10359,6 +11033,7 @@ mod tests { /// runs exactly one `rev-list --all`, so this counts the walks that were attempted /// (the `ls-tree` counter above cannot: a walk whose enumeration fails never gets /// to `ls-tree`). + #[cfg(unix)] fn f2b_walk_attempts(log: &std::path::Path) -> usize { f2a_log(log) .lines() diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 27b67786..337485df 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -532,6 +532,9 @@ mod tests { peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_blob_semaphore: Arc::new(tokio::sync::Semaphore::new( + crate::api::repos::MAX_CONCURRENT_BLOB_READS, + )), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index 1fbf4376..5c12954a 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -314,8 +314,8 @@ pub struct Config { /// cap is a different axis (500 connections each fan out to git + /// pack-objects + threads). Size below the process budget with headroom. /// - /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK - /// `info/refs` advertisement only. The authenticated push POST draws from a + /// This is the READ pool (`git_read_semaphore`): upload-pack, the UPLOAD-PACK + /// `info/refs` advertisement, and REST blob reads. The authenticated push POST draws from a /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can /// never reach, and each read caller is additionally bounded by /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual @@ -422,9 +422,10 @@ pub struct Config { )] pub max_concurrent_pin_tasks: usize, - /// Maximum concurrent read operations (`upload-pack` and the upload-pack - /// `info/refs` advertisement) a single caller may hold at once, so one caller - /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// Maximum concurrent read operations (`upload-pack`, the upload-pack + /// `info/refs` advertisement, and REST blob reads) a single caller may hold at + /// once, so one caller cannot monopolize the `max_concurrent_git_ops` read pool + /// (#174). Callers are /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index 474408e5..8b99ef0a 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -35,6 +35,9 @@ pub enum AppError { #[error("invalid request: {0}")] BadRequest(String), + #[error("payload too large: {0}")] + PayloadTooLarge(String), + /// A DID was well-formed enough to carry to a resolver but no verifying key /// could be derived from it. Its own code rather than plain `bad_request` /// because the auth middleware already answers `unresolvable_did` for the @@ -162,6 +165,11 @@ impl IntoResponse for AppError { // IcaptchaProofRequired is handled above (it carries extra headers/fields). AppError::IcaptchaProofRequired { .. } => unreachable!("handled before this match"), AppError::BadRequest(msg) => (StatusCode::BAD_REQUEST, "bad_request", msg.clone()), + AppError::PayloadTooLarge(msg) => ( + StatusCode::PAYLOAD_TOO_LARGE, + "payload_too_large", + msg.clone(), + ), AppError::UnresolvableDid(msg) => { (StatusCode::BAD_REQUEST, "unresolvable_did", msg.clone()) } @@ -274,6 +282,16 @@ mod tests { ); } + #[test] + fn payload_too_large_maps_to_413() { + assert_eq!( + AppError::PayloadTooLarge("x".into()) + .into_response() + .status(), + StatusCode::PAYLOAD_TOO_LARGE + ); + } + #[test] fn overloaded_maps_to_503_with_retry_after() { let resp = AppError::Overloaded("x".into()).into_response(); diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index 45820746..2a465c59 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -83,7 +83,7 @@ impl RepoStore { /// Test-only: every guard from this store parks in `release` right before the /// `pg_advisory_unlock` await, until `gate` is notified. Dropping the future /// while it is parked reproduces a client disconnect inside `release`. - #[cfg(test)] + #[cfg(all(test, unix))] pub fn with_pre_unlock_gate(mut self, gate: Arc) -> Self { self.pre_unlock_gate = Some(gate); self @@ -107,7 +107,7 @@ impl RepoStore { /// Test-only: how many write guards from this store have reached the Tigris upload /// site. See [`RepoStore::upload_site_reached`]. - #[cfg(test)] + #[cfg(all(test, unix))] pub fn tigris_upload_site_reached(&self) -> usize { self.upload_site_reached .load(std::sync::atomic::Ordering::SeqCst) diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 5617b419..901fb27d 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -213,21 +213,213 @@ pub fn ls_tree(repo_path: &Path, refname: &str, tree_path: &str) -> Result Result> { +#[derive(Debug, PartialEq, Eq)] +pub enum BoundedFileRead { + Found(Vec), + Missing, + TooLarge { size: u64, max: u64 }, +} + +fn ref_resolves_bounded( + git_bin: &str, + repo_path: &Path, + refname: &str, + deadline: std::time::Instant, +) -> Result { + let commit = format!("{refname}^{{commit}}"); + let (status, _, _) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["rev-parse", "--verify", &commit], + repo_path, + &[], + deadline, + )?; + Ok(status.success()) +} + +/// Resolve the same ref preference as [`resolve_head`] without running an unbounded +/// child on an async worker. Every command shares the caller's deadline. +fn resolve_head_bounded( + git_bin: &str, + repo_path: &Path, + preferred_branch: &str, + deadline: std::time::Instant, +) -> Result { + if ref_resolves_bounded(git_bin, repo_path, "HEAD", deadline)? { + return Ok("HEAD".into()); + } + + let preferred = format!("refs/heads/{preferred_branch}"); + if ref_resolves_bounded(git_bin, repo_path, &preferred, deadline)? { + return Ok(preferred); + } + + for candidate in ["refs/heads/main", "refs/heads/master", "refs/heads/develop"] { + if candidate != preferred.as_str() + && ref_resolves_bounded(git_bin, repo_path, candidate, deadline)? + { + return Ok(candidate.into()); + } + } + + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ + "for-each-ref", + "--count=1", + "--format=%(refname)", + "refs/heads/", + ], + repo_path, + &[], + deadline, + )?; + let first = String::from_utf8(out).context("git returned a non-UTF-8 ref name")?; + let first = first.trim(); + Ok(if first.is_empty() { "HEAD" } else { first }.to_string()) +} + +/// Read one file for the REST blob endpoint without materializing unbounded child +/// output. The mutable ref is resolved to one immutable blob OID, its declared size is +/// checked before content capture, and the stdout drain retains at most `max_bytes`. +/// All Git children share `deadline`; async callers must invoke this in `spawn_blocking`. +pub fn read_file_bounded( + git_bin: &str, + repo_path: &Path, + preferred_branch: &str, + file_path: &str, + max_bytes: u64, + deadline: std::time::Instant, +) -> Result { + if file_path.contains('\r') || file_path.contains('\n') { + bail!("file path contains a line break"); + } + + let refname = resolve_head_bounded(git_bin, repo_path, preferred_branch, deadline)?; let spec = format!("{refname}:{file_path}"); - let output = Command::new("git") - .args(["show", &spec]) - .current_dir(repo_path) - .output() - .context("failed to run git show")?; + let stdin = format!("{spec}\n"); + let output = blob_metadata_bounded(git_bin, repo_path, &spec, stdin.as_bytes(), deadline)?; + let line = output.trim(); + if line.split_whitespace().last() == Some("missing") { + return Ok(BoundedFileRead::Missing); + } - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git show failed: {stderr}"); + let mut parts = line.split_whitespace(); + let oid = parts.next().context("git omitted the blob object ID")?; + let kind = parts.next().context("git omitted the object type")?; + let size = parts + .next() + .context("git omitted the object size")? + .parse::() + .context("git returned an invalid object size")?; + if parts.next().is_some() + || !matches!(oid.len(), 40 | 64) + || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + bail!("git returned invalid object metadata"); + } + if kind != "blob" { + return Ok(BoundedFileRead::Missing); + } + if size > max_bytes { + return Ok(BoundedFileRead::TooLarge { + size, + max: max_bytes, + }); + } + + let max_stdout = usize::try_from(max_bytes).context("blob limit exceeds platform capacity")?; + let (status, content, stderr, exceeded) = + crate::git::visibility_pack::run_bounded_git_raw_capped( + git_bin, + &["cat-file", "blob", oid], + repo_path, + &[], + deadline, + max_stdout, + )?; + if !status.success() { + bail!( + "git cat-file blob failed: {}", + String::from_utf8_lossy(&stderr) + ); + } + if exceeded { + bail!("git emitted blob content beyond the served size limit"); + } + if content.len() as u64 != size { + bail!( + "git blob size changed between metadata and content reads (expected {size}, got {})", + content.len() + ); } + Ok(BoundedFileRead::Found(content)) +} + +fn blob_metadata_bounded( + git_bin: &str, + repo_path: &Path, + spec: &str, + stdin: &[u8], + deadline: std::time::Instant, +) -> Result { + let probe_started = std::time::Instant::now(); + for attempt in 0..2 { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "--batch-check"], + repo_path, + stdin, + deadline, + )?; + if !status.success() { + bail!( + "git cat-file --batch-check failed: {}", + String::from_utf8_lossy(&stderr) + ); + } + + let stderr = String::from_utf8_lossy(&stderr); + if stderr + .lines() + .any(|line| line.starts_with("error:") || line.starts_with("fatal:")) + { + bail!("git cat-file --batch-check reported: {}", stderr.trim()); + } - Ok(output.stdout) + let output = String::from_utf8(stdout).context("git returned non-UTF-8 object metadata")?; + let mut lines = output.lines(); + let line = lines.next().unwrap_or_default().trim(); + if lines.next().is_some() { + bail!("git returned multiple object metadata records"); + } + if line.split_whitespace().last() == Some("missing") { + // A clean missing response also occurs for unreadable packs. Use the + // same out-of-band check as object_type_bounded, then confirm once. + let worktree_git = repo_path.join(".git"); + let git_dir = if worktree_git.is_dir() { + worktree_git.as_path() + } else { + repo_path + }; + if !object_store_readable(git_dir, spec) { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: object store not readable" + )) + .into()); + } + if attempt == 0 { + if deadline.saturating_duration_since(std::time::Instant::now()) + < probe_started.elapsed() + { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + continue; + } + } + return Ok(output); + } + unreachable!("the confirming probe returns its result") } #[derive(Debug, Clone, serde::Serialize)] @@ -882,6 +1074,558 @@ mod tests { use std::path::Path; use std::process::Command; + #[cfg(unix)] + fn write_blob_probe_fixture(dir: &Path, content: &str) -> std::path::PathBuf { + use std::io::Write; + let script = dir.join("fakegit"); + // Write in a child so parallel tests cannot inherit an open writable + // script descriptor when they fork (which would cause ETXTBSY). + let mut writer = Command::new("sh") + .args(["-c", "cat > \"$1\" && chmod 755 \"$1\"", "fixture-writer"]) + .arg(&script) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + writer + .stdin + .take() + .unwrap() + .write_all(content.as_bytes()) + .unwrap(); + assert!(writer.wait().unwrap().success()); + script + } + + #[test] + fn bounded_file_read_rejects_packed_blob_and_preserves_allowed_content() { + let td = tempfile::TempDir::new().unwrap(); + let work = td.path(); + let run = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(work) + .status() + .unwrap() + .success(), + "git {args:?} failed" + ); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + let content = vec![b'a'; 64 * 1024]; + std::fs::write(work.join("large.txt"), &content).unwrap(); + run(&["add", "large.txt"]); + run(&["commit", "-qm", "add blob"]); + run(&["gc", "-q"]); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + assert_eq!( + super::read_file_bounded("git", work, "main", "large.txt", 1024, deadline).unwrap(), + super::BoundedFileRead::TooLarge { + size: content.len() as u64, + max: 1024, + } + ); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + assert_eq!( + super::read_file_bounded( + "git", + work, + "main", + "large.txt", + content.len() as u64, + deadline, + ) + .unwrap(), + super::BoundedFileRead::Found(content) + ); + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_confirms_missing_and_recovers_after_reprobe() { + for recover in [false, true] { + let td = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(td.path().join("objects")).unwrap(); + let oid = "a".repeat(40); + let script = write_blob_probe_fixture( + td.path(), + &format!( + r#"#!/bin/sh +if [ "$1" = rev-parse ]; then echo {oid}; exit 0; fi +if [ "$2" = --batch-check ]; then + read spec + echo probe >> probes + if [ -f probed ] && [ "{recover}" = true ]; then echo '{oid} blob 5'; else echo "$spec missing"; fi + touch probed + exit 0 +fi +if [ "$2" = blob ]; then printf hello; exit 0; fi +exit 1 +"# + ), + ); + let result = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "hello.txt", + 1024, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap(); + assert_eq!( + result, + if recover { + super::BoundedFileRead::Found(b"hello".to_vec()) + } else { + super::BoundedFileRead::Missing + } + ); + assert_eq!( + std::fs::read_to_string(td.path().join("probes")) + .unwrap() + .lines() + .count(), + 2 + ); + } + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_does_not_report_missing_for_unreadable_pack() { + use std::os::unix::fs::symlink; + let td = tempfile::TempDir::new().unwrap(); + let pack = td.path().join("objects/pack"); + std::fs::create_dir_all(&pack).unwrap(); + // A disappearing pack is unopenable even when CI runs as root. + symlink("removed-during-repack", pack.join("unreadable.pack")).unwrap(); + let script = write_blob_probe_fixture(td.path(), "#!/bin/sh\nif [ \"$1\" = rev-parse ]; then exit 0; fi\nread spec\necho \"$spec missing\"\n"); + let error = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "hello.txt", + 1024, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap_err(); + assert!( + error.to_string().contains("object store not readable"), + "{error:#}" + ); + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_does_not_capture_rejected_content() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let marker = td.path().join("content-called"); + let oid = "a".repeat(40); + let script = td.path().join("fakegit"); + std::fs::write( + &script, + format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 4096\"; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"blob\" ]; then touch \"{}\"; printf x; exit 0; fi\n\ + exit 1\n", + marker.display() + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&script, permissions).unwrap(); + + let result = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "large.txt", + 1024, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap(); + assert_eq!( + result, + super::BoundedFileRead::TooLarge { + size: 4096, + max: 1024, + } + ); + assert!( + !marker.exists(), + "content command must not run after an oversized metadata result" + ); + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_caps_content_that_exceeds_preflight_size() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let oid = "b".repeat(40); + let script = td.path().join("fakegit"); + std::fs::write( + &script, + format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 3\"; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"blob\" ] && [ \"$3\" = \"{oid}\" ]; then printf 0123456789abcdef0123456789abcdef; exit 0; fi\n\ + exit 1\n" + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&script, permissions).unwrap(); + + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "large.txt", + 16, + std::time::Instant::now() + std::time::Duration::from_secs(10), + ) + .unwrap_err(); + assert!( + err.to_string().contains("beyond the served size limit"), + "unexpected error: {err:#}" + ); + } + + #[cfg(unix)] + #[test] + fn bounded_file_read_guards_size_mismatch_and_metadata_parse() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let script = td.path().join("fakegit"); + let write_fake = |body: &str| { + std::fs::write(&script, body).unwrap(); + let mut permissions = std::fs::metadata(&script).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&script, permissions).unwrap(); + }; + + let oid = "c".repeat(40); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // 1. Size mismatch (emitted bytes fewer than declared size): + // Declares 20 bytes, emits 5 bytes ("hello"). max_bytes is 64 (so exceeded is false). + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 20\"; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"blob\" ]; then printf hello; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "mismatch.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string() + .contains("size changed between metadata and content reads"), + "expected size mismatch error, got: {err:#}" + ); + + // 2. Metadata parse: missing fields (only 2 fields, no size) + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob\"; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("omitted the object size"), + "expected missing fields error, got: {err:#}" + ); + + // 3. Metadata parse: non-hex OID + write_fake( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo nothex; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"not-a-valid-hex-oid-1234567890123456789012 blob 10\"; exit 0; fi\n\ + exit 1\n" + ); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("invalid object metadata"), + "expected non-hex oid error, got: {err:#}" + ); + + // 4. Metadata parse: trailing fields + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; echo \"{oid} blob 10 trailing-junk\"; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("invalid object metadata"), + "expected trailing fields error, got: {err:#}" + ); + + // 5. Metadata parse: multi-record output + write_fake(&format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"rev-parse\" ]; then echo {oid}; exit 0; fi\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then read spec; printf '%s\\n%s\\n' '{oid} blob 10' '{oid} blob 10'; exit 0; fi\n\ + exit 1\n" + )); + let err = super::read_file_bounded( + script.to_str().unwrap(), + td.path(), + "main", + "bad.txt", + 64, + deadline, + ) + .unwrap_err(); + assert!( + err.to_string().contains("multiple object metadata records"), + "expected multi-record error, got: {err:#}" + ); + } + + #[cfg(unix)] + #[test] + fn blob_metadata_bounded_reprobe_budget_exhaustion_returns_timeout() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::TempDir::new().unwrap(); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(bare.join("objects/pack")).unwrap(); + let log = td.path().join("probe spawns.log"); + let quoted_log = format!("'{}'", log.display().to_string().replace('\'', "'\"'\"'")); + let fake = td.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\n\ + echo call >> {quoted_log}\n\ + if [ \"$1\" = \"cat-file\" ] && [ \"$2\" = \"--batch-check\" ]; then \ + read spec; \ + sleep 2.5; \ + printf '%s\\n' \"$spec missing\"; \ + echo done >> {quoted_log}; \ + exit 0; \ + fi\n\ + exit 1\n" + ), + ) + .unwrap(); + let mut permissions = std::fs::metadata(&fake).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(&fake, permissions).unwrap(); + + // Spend over half the budget while leaving 1.5s for scheduler jitter. + // The completion marker below must still rule out a watchdog kill. + let budget = std::time::Duration::from_secs(4); + let deadline = std::time::Instant::now() + budget; + let err = super::blob_metadata_bounded( + fake.to_str().unwrap(), + &bare, + "spec", + b"spec\n", + deadline, + ) + .unwrap_err(); + assert!( + err.downcast_ref::() + .is_some(), + "exhausted reprobe budget must return GitServiceTimeout, got: {err:#}" + ); + assert_eq!( + std::fs::read_to_string(&log).unwrap(), + "call\ndone\n", + "the first probe must finish, without spawning an unaffordable confirming probe" + ); + } + + #[test] + fn read_file_bounded_denies_non_blob_paths() { + let td = tempfile::TempDir::new().unwrap(); + let work = td.path(); + let run = |args: &[&str]| { + let status = std::process::Command::new("git") + .args(args) + .current_dir(work) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + std::fs::create_dir(work.join("subfolder")).unwrap(); + std::fs::write(work.join("subfolder/file.txt"), b"contents").unwrap(); + run(&["add", "subfolder"]); + run(&["commit", "-qm", "add directory"]); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + + // "subfolder" resolves to a tree object in git, not a blob. + let read = + super::read_file_bounded("git", work, "main", "subfolder", 1024, deadline).unwrap(); + assert_eq!(read, super::BoundedFileRead::Missing); + } + + #[test] + fn resolve_head_bounded_covers_all_fallback_arms() { + let td = tempfile::TempDir::new().unwrap(); + let work = td.path(); + let run = |args: &[&str]| { + let status = std::process::Command::new("git") + .args(args) + .current_dir(work) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + run(&["init", "-q"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + std::fs::write(work.join("hello.txt"), b"hello").unwrap(); + run(&["add", "hello.txt"]); + run(&["commit", "-qm", "initial"]); + run(&["branch", "-M", "custom-feature"]); + run(&["branch", "aaa-earlier"]); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + assert_eq!( + super::resolve_head_bounded("git", work, "aaa-earlier", deadline).unwrap(), + "HEAD" + ); + // Detach or point HEAD to an unborn branch so HEAD itself does not resolve. + run(&["symbolic-ref", "HEAD", "refs/heads/unborn-branch"]); + + // 1. Preferred branch arm: "custom-feature" resolves to refs/heads/custom-feature. + let resolved = + super::resolve_head_bounded("git", work, "custom-feature", deadline).unwrap(); + assert_eq!(resolved, "refs/heads/custom-feature"); + let read = + super::read_file_bounded("git", work, "custom-feature", "hello.txt", 1024, deadline) + .unwrap(); + assert_eq!(read, super::BoundedFileRead::Found(b"hello".to_vec())); + + // 2. Candidate branch arm (main, master, develop): create "master". + run(&["branch", "-M", "custom-feature", "master"]); + // Asking for a nonexistent preferred branch falls back to "master". + let resolved_master = + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(); + assert_eq!(resolved_master, "refs/heads/master"); + let read_master = + super::read_file_bounded("git", work, "nonexistent", "hello.txt", 1024, deadline) + .unwrap(); + assert_eq!( + read_master, + super::BoundedFileRead::Found(b"hello".to_vec()) + ); + + // Main must precede master, which must precede develop; all precede + // the alphabetically earlier branch chosen by for-each-ref. + run(&["branch", "develop", "master"]); + run(&["branch", "main", "master"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/main" + ); + run(&["branch", "-m", "main", "z-retired-main"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/master" + ); + run(&["branch", "-M", "master", "isolated-branch"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/develop" + ); + run(&["branch", "-m", "develop", "z-retired-develop"]); + assert_eq!( + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(), + "refs/heads/aaa-earlier" + ); + run(&["branch", "-m", "aaa-earlier", "z-retired-earlier"]); + // 3. for-each-ref fallback arm: only a nonstandard branch remains. + let resolved_isolated = + super::resolve_head_bounded("git", work, "nonexistent", deadline).unwrap(); + assert_eq!(resolved_isolated, "refs/heads/isolated-branch"); + let read_isolated = + super::read_file_bounded("git", work, "nonexistent", "hello.txt", 1024, deadline) + .unwrap(); + assert_eq!( + read_isolated, + super::BoundedFileRead::Found(b"hello".to_vec()) + ); + + // 4. Empty refs fallback: a completely empty repo with unborn HEAD. + let td_empty = tempfile::TempDir::new().unwrap(); + let run_empty = |args: &[&str]| { + let status = std::process::Command::new("git") + .args(args) + .current_dir(td_empty.path()) + .status() + .unwrap(); + assert!(status.success(), "git {args:?} failed"); + }; + run_empty(&["init", "-q"]); + let resolved_empty = + super::resolve_head_bounded("git", td_empty.path(), "nonexistent", deadline).unwrap(); + assert_eq!(resolved_empty, "HEAD"); + let read_empty = super::read_file_bounded( + "git", + td_empty.path(), + "nonexistent", + "hello.txt", + 1024, + deadline, + ) + .unwrap(); + assert_eq!(read_empty, super::BoundedFileRead::Missing); + } + #[test] fn branch_diff_names_lists_changed_paths() { let td = tempfile::TempDir::new().unwrap(); diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index 08666994..985184e9 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -29,6 +29,29 @@ const WALK_TIMEOUT: Duration = Duration::from_secs(600); #[cfg(unix)] const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); +/// Drain stdout completely while retaining no more than `limit` bytes. Continuing to +/// drain after the retained buffer is full prevents a child from deadlocking on its +/// stdout pipe without allowing its output to keep growing resident memory. +fn drain_stdout( + reader: &mut impl std::io::Read, + limit: Option, +) -> std::io::Result<(Vec, bool)> { + let mut out = Vec::new(); + let mut exceeded = false; + let mut chunk = [0_u8; 8192]; + loop { + let read = reader.read(&mut chunk)?; + if read == 0 { + return Ok((out, exceeded)); + } + let retained = limit + .map(|limit| read.min(limit.saturating_sub(out.len()))) + .unwrap_or(read); + out.extend_from_slice(&chunk[..retained]); + exceeded |= retained < read; + } +} + /// Run one git child under a shared `deadline` with process-group teardown, /// BLOCKING, and return its stdout. The child runs in its own process group; a /// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, @@ -67,13 +90,14 @@ fn child_terminated_without_reaping(pid: i32) -> bool { } #[cfg(unix)] -pub(crate) fn run_bounded_git_raw( +fn run_bounded_git_raw_with_limit( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result<(std::process::ExitStatus, Vec, Vec)> { + stdout_limit: Option, +) -> Result<(std::process::ExitStatus, Vec, Vec, bool)> { use std::io::{Read, Write}; use std::os::unix::process::CommandExt; use std::sync::mpsc::RecvTimeoutError; @@ -149,14 +173,13 @@ pub(crate) fn run_bounded_git_raw( err }); let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut out = Vec::new(); + let read_result = drain_stdout(&mut stdout, stdout_limit); // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block // until the kernel returns, pinning the walk thread and its permit. That residual // is unreachable in userspace (no signal reaps a D-state process) and matches the // async `reap_group_on_timeout`, which likewise only warns and gives up there. - let read_result = stdout.read_to_end(&mut out); // The drain has returned, but that only means all stdout write ends are closed — // NOT that the child has exited. A group member, or the leader itself, can close // stdout and keep running; standing the watchdog down on the drain alone (as the @@ -183,7 +206,7 @@ pub(crate) fn run_bounded_git_raw( let status = child.wait().context("git wait failed")?; let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - read_result.context("failed to read git stdout")?; + let (out, stdout_exceeded) = read_result.context("failed to read git stdout")?; // The watchdog runs off a wall clock that can race a child finishing right at the // deadline. A child that exited on its own (success) is not a timeout even if the // watchdog fired late; only a child that did not exit successfully is a genuine @@ -191,9 +214,46 @@ pub(crate) fn run_bounded_git_raw( if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err, stdout_exceeded)) +} + +#[cfg(unix)] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + let (status, out, err, _) = + run_bounded_git_raw_with_limit(git_bin, args, repo_path, stdin_bytes, deadline, None)?; Ok((status, out, err)) } +/// Run a bounded git child while retaining at most `max_stdout_bytes` from stdout. +/// The pipe is still drained after the limit so the child cannot block on a full pipe; +/// `true` in the fourth tuple field reports that bytes were discarded. This is the +/// output-side companion to the process deadline for callers serving attacker-chosen +/// objects: a child that emits more than its preflight size cannot grow memory past the +/// caller's ceiling. +pub(crate) fn run_bounded_git_raw_capped( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, + max_stdout_bytes: usize, +) -> Result<(std::process::ExitStatus, Vec, Vec, bool)> { + run_bounded_git_raw_with_limit( + git_bin, + args, + repo_path, + stdin_bytes, + deadline, + Some(max_stdout_bytes), + ) +} + /// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin /// wrapper the walk callers use. Probes that must distinguish exit classes — /// `git cat-file` absence vs an object-store access failure — call @@ -227,13 +287,14 @@ pub(crate) fn run_bounded_git( /// the Unix version's signature and result semantics so every caller compiles on all /// targets (#174). #[cfg(not(unix))] -pub(crate) fn run_bounded_git_raw( +fn run_bounded_git_raw_with_limit( git_bin: &str, args: &[&str], repo_path: &Path, stdin_bytes: &[u8], deadline: Instant, -) -> Result<(std::process::ExitStatus, Vec, Vec)> { + stdout_limit: Option, +) -> Result<(std::process::ExitStatus, Vec, Vec, bool)> { use std::io::{Read, Write}; use std::sync::mpsc::RecvTimeoutError; @@ -283,8 +344,7 @@ pub(crate) fn run_bounded_git_raw( }) }; - let mut out = Vec::new(); - let read_result = stdout.read_to_end(&mut out); + let read_result = drain_stdout(&mut stdout, stdout_limit); // The drain has returned (child exited or was killed), so taking the lock here // cannot deadlock against the watchdog. let status = child @@ -296,10 +356,23 @@ pub(crate) fn run_bounded_git_raw( let killed = watchdog.join().unwrap_or(false); let err = err_reader.join().unwrap_or_default(); let _ = writer.join(); - read_result.context("failed to read git stdout")?; + let (out, stdout_exceeded) = read_result.context("failed to read git stdout")?; if killed && !status.success() { return Err(crate::git::smart_http::GitServiceTimeout.into()); } + Ok((status, out, err, stdout_exceeded)) +} + +#[cfg(not(unix))] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + let (status, out, err, _) = + run_bounded_git_raw_with_limit(git_bin, args, repo_path, stdin_bytes, deadline, None)?; Ok((status, out, err)) } @@ -1225,6 +1298,15 @@ pub fn withheld_blob_recipients_bounded( mod tests { use super::*; + #[test] + fn stdout_drain_discards_bytes_past_the_retention_limit() { + let mut input = std::io::Cursor::new(b"0123456789".to_vec()); + let (out, exceeded) = drain_stdout(&mut input, Some(4)).unwrap(); + assert_eq!(out, b"0123"); + assert!(exceeded); + assert_eq!(input.position(), 10, "the pipe must still be fully drained"); + } + /// Write an executable fake `git` shell script into `dir` and return its path, /// so a test can drive the walk's process-group teardown without a real git and /// without mutating the process-global PATH (the crate's only injection seam). diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 5d4579a3..9b0e929d 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -2534,6 +2534,7 @@ mod tests { /// type probe carries its oid on stdin rather than in argv, so an oid appears in /// the log only once an object has already got past its probe, and a healthy /// object costs two invocations to a faulting one's one. + #[cfg(unix)] fn objects_attempted(log: &std::path::Path) -> usize { std::fs::read_to_string(log) .unwrap_or_default() diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index 66bfa096..bf85679e 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -444,6 +444,9 @@ async fn main() -> Result<()> { peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_blob_semaphore: Arc::new(tokio::sync::Semaphore::new( + crate::api::repos::MAX_CONCURRENT_BLOB_READS, + )), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( config.max_concurrent_git_pushes, )), diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 14f1d582..d3e29217 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -911,6 +911,7 @@ mod tests { /// probe carries its oid on stdin rather than in argv, so an oid appears in the log only /// once an object has already got past its probe, and a healthy object costs two /// invocations to a faulting one's one. + #[cfg(unix)] fn objects_attempted(log: &std::path::Path) -> usize { std::fs::read_to_string(log) .unwrap_or_default() diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 24607e5a..482f3dc5 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -194,14 +194,18 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, - /// Bounds concurrent served git READ operations (upload-pack and its own - /// `info/refs` advertisement ONLY — the receive-pack advertisement draws from - /// `git_push_advert_semaphore`, so sizing this pool for both would undercount - /// the read capacity an operator gets). A read handler acquires a permit before - /// spawning git and holds it for the op; when none are free the request is shed - /// with a 503. Writes draw from `git_write_semaphore` so a read flood cannot + /// Bounds concurrent served git READ operations (upload-pack, its own + /// `info/refs` advertisement, and REST blob reads — the receive-pack advertisement + /// draws from `git_push_advert_semaphore`, so sizing this pool for both would + /// undercount the read capacity an operator gets). A read handler acquires a permit + /// before spawning git and holds it for the op; when none are free the request is + /// shed with a 503. Writes draw from `git_write_semaphore` so a read flood cannot /// shed an authenticated push at admission (#174). pub git_read_semaphore: Arc, + /// Bounds REST blob bodies that are being read or delivered. The response stream + /// owns this permit until EOF or disconnect, so slow clients cannot recycle a Git + /// execution slot while retaining another full blob in memory. + pub git_blob_semaphore: Arc, /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate /// from `git_read_semaphore` so an anonymous READ flood can never shed an /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from @@ -279,8 +283,8 @@ pub struct AppState { /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` - /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` - /// advertisement. + /// (#174). Applied by `git_upload_pack`, the upload-pack `info/refs` + /// advertisement, and REST blob reads. pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` /// advertisement: each source IP may hold at most a small share of the DEDICATED diff --git a/crates/gitlawb-node/src/sync.rs b/crates/gitlawb-node/src/sync.rs index 0ed4a9f9..9111b84c 100644 --- a/crates/gitlawb-node/src/sync.rs +++ b/crates/gitlawb-node/src/sync.rs @@ -691,15 +691,22 @@ async fn existing_promisor_state(repo: &str) -> PromisorProbe { } } -/// Mirror-clone a repo from a remote URL into a local bare repo. -/// `Promisor` mode adds `--filter=blob:limit=10g`, which marks the repo a git -/// promisor (so a pack with origin-omitted withheld blobs is accepted) while -/// the huge size limit means every blob the origin *does* send is kept. +// Git for Windows parses blob limits as a 32-bit unsigned long. Larger +// Windows blobs remain available through promisor on-demand fetching. +#[cfg(windows)] +const PROMISOR_BLOB_FILTER: &str = "blob:limit=4294967295"; +#[cfg(not(windows))] +const PROMISOR_BLOB_FILTER: &str = "blob:limit=10g"; + +/// Mirror-clone a repo, marking filtered mirrors as promisors so packs that +/// omit withheld blobs are accepted. The filter threshold is 10 GiB on Unix +/// and 4 GiB minus one byte on Windows; larger blobs are fetched on demand. async fn clone_repo(remote_url: &str, local_path: &Path, mode: MirrorMode) -> anyhow::Result<()> { let local_str = local_path.to_str().unwrap_or("."); + let filter_arg = format!("--filter={PROMISOR_BLOB_FILTER}"); let mut args = vec!["clone", "--mirror"]; if mode == MirrorMode::Promisor { - args.push("--filter=blob:limit=10g"); + args.push(&filter_arg); } args.push(remote_url); args.push(local_str); @@ -739,7 +746,7 @@ async fn fetch_repo(local_path: &Path, remote_url: &str, mode: MirrorMode) -> an local_str, "config", "remote.origin.partialclonefilter", - "blob:limit=10g", + PROMISOR_BLOB_FILTER, ]) .await?; git_run(&["-C", local_str, "fetch", "--prune", "origin"]).await @@ -1250,6 +1257,18 @@ mod tests { .await; } + /// Use a rooted path without a Windows drive prefix in the remote fixture. + /// The destination still resolves on the temp directory's current drive, + /// while the remote's relative path contains no illegal colon component. + fn absolute_slug_path(path: &Path) -> String { + assert!(path.is_absolute()); + path.components() + .filter(|part| !matches!(part, std::path::Component::Prefix(_))) + .collect::() + .to_string_lossy() + .replace('\\', "/") + } + #[sqlx::test] async fn process_batch_rejects_slug_escaping_repos_dir(pool: PgPool) { // The verified escape from #272: `PathBuf::join` discards everything @@ -1265,12 +1284,13 @@ mod tests { // fails on but leaves the parent it created, so assert on both. let outside = TempDir::new().unwrap(); let escape_dir = outside.path().join("nest"); - let slug = format!("a/{}/escape", escape_dir.display()); + let escape_path = absolute_slug_path(&escape_dir); + let slug = format!("a/{escape_path}/escape"); let escape_target = escape_dir.join("escape.git"); // Serve the composed URL for real, so a run without the guard genuinely // clones outside the root rather than merely failing at git. - let rel = format!("a{}/escape", escape_dir.display()); + let rel = format!("a{escape_path}/escape"); let (_remote, peer_url) = rooted_remote(&[&rel]); let did = "did:key:z6MkAttacker"; @@ -1390,6 +1410,7 @@ mod tests { // ── canonical containment before the git call (issue #272) ─────────────── /// Every ref in `repo`, as one string, for a before/after comparison. + #[cfg(unix)] fn refs_of(repo: &Path) -> String { let out = Command::new("git") .args(["-C", repo.to_str().unwrap(), "for-each-ref"]) @@ -1722,6 +1743,7 @@ mod tests { /// starvation tests is fixed rather than dependent on how fast the loop /// runs. Two rows enqueued in the same microsecond would otherwise order /// arbitrarily. + #[cfg(unix)] async fn enqueue_at(db: &Db, pool: &PgPool, repo: &str, did: &str, enqueued_at: &str) { enqueue(db, repo, did).await; sqlx::query("UPDATE sync_queue SET enqueued_at = $1 WHERE repo = $2") @@ -1915,7 +1937,8 @@ mod tests { let outside = TempDir::new().unwrap(); let escape_dir = outside.path().join("nest"); - let slug = format!("a/{}/gitlawb-probe", escape_dir.display()); + let escape_path = absolute_slug_path(&escape_dir); + let slug = format!("a/{escape_path}/gitlawb-probe"); let escape_target = escape_dir.join("gitlawb-probe.git"); // Two things this fixture must get right or the test is green for the @@ -1926,7 +1949,7 @@ mod tests { // run without the guard genuinely clones outside the root instead of // just failing at git. Db::upsert_peer cannot seed this row: it gates on // is_public_http_url, which rejects file://. - let rel = format!("a{}/gitlawb-probe", escape_dir.display()); + let rel = format!("a{escape_path}/gitlawb-probe"); let (_remote, peer_url) = rooted_remote(&[&rel]); let did = "did:key:z6MkAttacker"; seed_local_peer(&pool, did, &peer_url).await; diff --git a/crates/gitlawb-node/src/test_git_shim.rs b/crates/gitlawb-node/src/test_git_shim.rs new file mode 100644 index 00000000..a53b05d6 --- /dev/null +++ b/crates/gitlawb-node/src/test_git_shim.rs @@ -0,0 +1,75 @@ +//! Native Git fixtures shared by timing tests on Unix and Windows. + +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +/// The fixture delays or hangs in its own process, so the production watchdog +/// can terminate it without depending on a shell or descendant process support. +pub(super) enum Behavior<'a> { + Delay(u64), + Hang, + HangOid(&'a str), + HangRepo(&'a str), + LogTypes(&'a Path), +} + +pub(super) struct GitShim { + _directory: tempfile::TempDir, + executable: PathBuf, +} + +impl Deref for GitShim { + type Target = Path; + + fn deref(&self) -> &Path { + &self.executable + } +} + +pub(super) fn create(name: &str, behavior: Behavior<'_>) -> GitShim { + static COMPILED: OnceLock = OnceLock::new(); + let compiled = COMPILED.get_or_init(|| { + let directory = tempfile::tempdir().expect("native Git fixture directory"); + let source = directory.path().join("git_shim.rs"); + std::fs::write(&source, include_str!("../tests/fixtures/git_shim.rs")) + .expect("write native Git fixture source"); + let executable = directory + .path() + .join(format!("git-shim{}", std::env::consts::EXE_SUFFIX)); + let output = + std::process::Command::new(std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into())) + .arg("--edition=2021") + .arg("-Dwarnings") + .arg(&source) + .arg("-o") + .arg(executable) + .output() + .expect("compile native Git fixture with the installed Rust toolchain"); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + directory + }); + let directory = tempfile::Builder::new() + .prefix(name) + .tempdir() + .expect("Git fixture instance"); + let file_name = format!("git-shim{}", std::env::consts::EXE_SUFFIX); + let executable = directory.path().join(&file_name); + std::fs::copy(compiled.path().join(file_name), &executable).expect("copy native Git fixture"); + let config = match behavior { + Behavior::Delay(milliseconds) => format!("delay\n{milliseconds}"), + Behavior::Hang => "hang\n".to_owned(), + Behavior::HangOid(oid) => format!("hang-oid\n{oid}"), + Behavior::HangRepo(repo) => format!("hang-repo\n{repo}"), + Behavior::LogTypes(path) => format!("log-types\n{}", path.display()), + }; + std::fs::write(directory.path().join("config"), config).expect("configure native Git fixture"); + GitShim { + _directory: directory, + executable, + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 430c0600..5cb1e390 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -28,6 +28,9 @@ use gitlawb_core::identity::Keypair; use crate::auth::AuthenticatedDid; use crate::state::AppState; +#[path = "test_git_shim.rs"] +mod git_shim; + /// Build an [`AppState`] over a real, migrated Postgres pool (from `#[sqlx::test]`). /// Runs the schema migrations first, because the per-test database starts empty. /// @@ -118,6 +121,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { shutdown_tx: tokio::sync::watch::channel(false).0, // Generous — no test drives the handler-level shed (git_permit is unit-tested). git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_blob_semaphore: Arc::new(tokio::sync::Semaphore::new( + crate::api::repos::MAX_CONCURRENT_BLOB_READS, + )), git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), @@ -427,11 +433,12 @@ mod tests { /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an /// authenticated push — the write pool is a separate budget. Read pool at zero, /// write pool with capacity: the push proceeds PAST admission (it then errors on - /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// the closed DB, with db_unavailable rather than overloaded). Route git-receive-pack /// back to the read pool and this goes red — that is the isolation proof. #[tokio::test] async fn git_receive_pack_not_shed_by_exhausted_read_pool() { let mut state = test_state_lazy(); + state.db.pool().close().await; // Read pool exhausted as if a flood of anonymous clones held every slot. state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); // Write pool keeps its default capacity from test_state_lazy. @@ -453,10 +460,13 @@ mod tests { .await .unwrap(); - assert_ne!( - resp.status(), - StatusCode::SERVICE_UNAVAILABLE, - "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + let bytes = axum::body::to_bytes(resp.into_body(), 1024).await.unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"], + crate::error::DB_UNAVAILABLE_CODE, + "the push must clear admission and reach the deliberately closed database" ); } @@ -652,6 +662,71 @@ mod tests { ); } + /// A signed non-reader and an anonymous caller get the same opaque 404 for + /// a withheld blob path. The gate runs before repository acquisition, so + /// the test needs no on-disk repository and any leaked record detail is a + /// direct authorization regression. + #[sqlx::test] + async fn get_blob_denies_withheld_path_without_leaking_details(pool: PgPool) { + use crate::db::VisibilityMode; + + let owner = "did:key:zBLOBOWNERAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let stranger = "did:key:zBLOBSTRANGERBBBBBBBBBBBBBBBBBBBBBBBBBBB"; + let state = test_state(pool).await; + let mut repo = seed_repo(owner, "blob-repo"); + repo.description = Some("protected-description-marker".into()); + repo.default_branch = "protected-branch-marker".into(); + state.db.create_repo(&repo).await.expect("seed repo"); + state + .db + .set_visibility_rule(&repo.id, "/secret/**", VisibilityMode::B, &[], owner) + .await + .expect("set rule"); + + let router = || { + Router::new() + .route( + "/api/v1/repos/{owner}/{repo}/blob/{*path}", + axum::routing::get(crate::api::repos::get_blob), + ) + .with_state(state.clone()) + }; + let uri = format!("/api/v1/repos/{owner}/blob-repo/blob/secret/protected-file-marker.txt"); + + for (caller, request) in [ + ( + "authenticated non-reader", + signed_request_as(stranger, Method::GET, &uri, Body::empty()), + ), + ("anonymous caller", anon_get(&uri)), + ] { + let response = router().oneshot(request).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "{caller} must be denied as if the repository did not exist" + ); + let body = json_body(response).await; + assert_eq!(body["error"], "repo_not_found", "wrong denial for {caller}"); + assert_eq!( + body["message"], + format!("repository '{owner}/blob-repo' not found"), + "{caller} must receive the opaque repository denial" + ); + let body = body.to_string(); + for protected_detail in [ + "protected-file-marker", + "protected-description-marker", + "protected-branch-marker", + ] { + assert!( + !body.contains(protected_detail), + "{caller} denial leaked protected detail: {protected_detail}" + ); + } + } + } + fn seed_task(id: &str, delegator: &str) -> AgentTask { let now = Utc::now().to_rfc3339(); AgentTask { @@ -7789,14 +7864,10 @@ mod tests { seed_legacy_pin(&pool, &bare, &fx.public_oid, Some(&repo.id)).await; // A git that takes 300ms per invocation (the read makes two: type, then content). - let slow_git = std::env::temp_dir().join(format!("gl-slow-git-{short}")); - std::fs::write(&slow_git, "#!/bin/sh\nsleep 0.3\nexec git \"$@\"\n").expect("write shim"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&slow_git, std::fs::Permissions::from_mode(0o755)) - .expect("chmod shim"); - } + let slow_git = git_shim::create( + &format!("gl-slow-git-{short}"), + git_shim::Behavior::Delay(300), + ); let ticks = std::sync::Arc::new(AtomicUsize::new(0)); let ticker = { @@ -8572,19 +8643,6 @@ mod tests { ); } - /// Write an executable `git` stand-in and return its path. - fn write_git_shim(name: &str, script: &str) -> std::path::PathBuf { - let path = std::env::temp_dir().join(name); - std::fs::write(&path, script).expect("write the git shim"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) - .expect("chmod the git shim"); - } - path - } - /// F6 scenario 1 (#173 round 13): one hung candidate must not starve the rows behind /// it in the same pass. `DiscoveryCtx` is loaded once per pass, so before the per-row /// slice every source-less row in a pass shared ONE deadline: the first row's wedged @@ -8624,23 +8682,9 @@ mod tests { // The type stage feeds the oid on STDIN (`cat-file --batch-check`) and the // content stage puts it in argv, so the stand-in has to look in both places. - let git_bin = write_git_shim( + let git_bin = git_shim::create( &format!("gl-hung-git-{short}"), - &format!( - "#!/bin/sh\n\ - if [ \"$2\" = \"--batch-check\" ]; then\n\ - \x20 oid=$(cat)\n\ - \x20 case \"$oid\" in\n\ - \x20 {hung_oid}) sleep 30; exit 1 ;;\n\ - \x20 esac\n\ - \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ - \x20 exit $?\n\ - fi\n\ - case \"$*\" in\n\ - \x20 *{hung_oid}*) sleep 30; exit 1 ;;\n\ - esac\n\ - exec git \"$@\"\n" - ), + git_shim::Behavior::HangOid(&hung_oid), ); let stats = tokio::time::timeout( @@ -8725,10 +8769,7 @@ mod tests { // Wedges on every invocation, so no row can ever be repaired and the only // question left is what each one COSTS. - let git_bin = write_git_shim( - &format!("gl-spent-git-{short}"), - "#!/bin/sh\nsleep 30\nexit 1\n", - ); + let git_bin = git_shim::create(&format!("gl-spent-git-{short}"), git_shim::Behavior::Hang); let stats = tokio::time::timeout( std::time::Duration::from_secs(60), @@ -9332,10 +9373,7 @@ mod tests { seed_legacy_pin(&pool, &src, oid, None).await; } - let git_bin = write_git_shim( - &format!("gl-starve-git-{short}"), - "#!/bin/sh\nsleep 30\nexit 1\n", - ); + let git_bin = git_shim::create(&format!("gl-starve-git-{short}"), git_shim::Behavior::Hang); tokio::time::timeout( std::time::Duration::from_secs(120), @@ -9403,9 +9441,9 @@ mod tests { let (raw_cid, provider_cid) = seed_legacy_pin(&pool, &src, &fx.public_oid, None).await; // Wedges only inside the position-nine repo, which the sweep enters by cwd. - let git_bin = write_git_shim( + let git_bin = git_shim::create( &format!("gl-mid-git-{short}"), - "#!/bin/sh\ncase \"$(pwd)\" in\n */midcand9.git) sleep 30; exit 1 ;;\nesac\nexec git \"$@\"\n", + git_shim::Behavior::HangRepo("midcand9.git"), ); let first = tokio::time::timeout( @@ -9567,19 +9605,9 @@ mod tests { let log = std::env::temp_dir().join(format!("gl-ali-log-{short}")); let _ = std::fs::remove_file(&log); - let git_bin = write_git_shim( + let git_bin = git_shim::create( &format!("gl-ali-git-{short}"), - &format!( - "#!/bin/sh\n\ - if [ \"$2\" = \"--batch-check\" ]; then\n\ - \x20 oid=$(cat)\n\ - \x20 printf '%s %s\\n' \"$oid\" \"$(basename $(pwd))\" >> {log}\n\ - \x20 printf '%s\\n' \"$oid\" | git \"$@\"\n\ - \x20 exit $?\n\ - fi\n\ - exec git \"$@\"\n", - log = log.display() - ), + git_shim::Behavior::LogTypes(&log), ); let mut traversal = crate::ipfs_pin::DiscoveryTraversalState::default(); diff --git a/crates/gitlawb-node/tests/fixtures/git_shim.rs b/crates/gitlawb-node/tests/fixtures/git_shim.rs new file mode 100644 index 00000000..162de46e --- /dev/null +++ b/crates/gitlawb-node/tests/fixtures/git_shim.rs @@ -0,0 +1,59 @@ +use std::io::{Read, Write}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +fn main() -> std::io::Result<()> { + let executable = std::env::current_exe()?; + let config = std::fs::read_to_string(executable.parent().unwrap().join("config"))?; + let (mode, parameter) = config.split_once('\n').unwrap_or((&config, "")); + let args: Vec<_> = std::env::args_os().skip(1).collect(); + if mode == "delay" { + std::thread::sleep(Duration::from_millis(parameter.parse().unwrap())); + } + let batch = args.get(1).is_some_and(|arg| arg == "--batch-check"); + let input = if batch { + let mut bytes = Vec::new(); + std::io::stdin().read_to_end(&mut bytes)?; + Some(bytes) + } else { + None + }; + let cwd = std::env::current_dir()?; + let hang = mode == "hang" + || (mode == "hang-repo" && cwd.file_name().is_some_and(|name| name == parameter)) + || (mode == "hang-oid" + && (args.iter().any(|arg| arg == parameter) + || input + .as_ref() + .is_some_and(|bytes| String::from_utf8_lossy(bytes).trim() == parameter))); + if hang { + std::thread::sleep(Duration::from_secs(30)); + std::process::exit(1); + } + if mode == "log-types" && batch { + let mut log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(parameter)?; + writeln!( + log, + "{} {}", + String::from_utf8_lossy(input.as_ref().unwrap()).trim(), + cwd.file_name().unwrap().to_string_lossy() + )?; + } + let mut child = Command::new("git") + .args(&args) + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::inherit() + }) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + if let Some(bytes) = input { + child.stdin.take().unwrap().write_all(&bytes)?; + } + std::process::exit(child.wait()?.code().unwrap_or(1)); +}