diff --git a/src/ops/temporal.c b/src/ops/temporal.c index 00bbbcb61..9ea07c88a 100644 --- a/src/ops/temporal.c +++ b/src/ops/temporal.c @@ -101,16 +101,67 @@ static int64_t rte_extract_one(int64_t us, int field) { * rest of the runtime). The previous version of this helper treated * TIMESTAMP as µs, which made (yyyy ts) decode to absurd years (26204 * on 2024-03-15) — a 1000× unit mismatch. */ -static inline int64_t rte_to_us(int8_t type, int64_t raw) { - if (type == RAY_DATE || type == -RAY_DATE) return raw * RTE_USEC_PER_DAY; - if (type == RAY_TIME || type == -RAY_TIME) return raw * 1000LL; - /* RAY_TIMESTAMP / -RAY_TIMESTAMP: ns → µs (floor toward -inf). */ - return raw >= 0 ? raw / 1000LL - : -(((-raw) + 999LL) / 1000LL); +static inline bool rte_to_us_ck(int8_t type, int64_t raw, int64_t* out) { + if (type == RAY_DATE || type == -RAY_DATE) { + /* DATE is int32 days, so `raw` can be as large as INT32_MAX — a date + * hundreds of millennia out. day * µs-per-day then overflows int64 + * (UBSan). Such a value is simply not representable in the µs domain + * the extract/truncate math uses; report it so the caller emits a + * null, consistent with how a null input is handled. */ + if (raw > INT64_MAX / RTE_USEC_PER_DAY || raw < INT64_MIN / RTE_USEC_PER_DAY) + return false; + *out = raw * RTE_USEC_PER_DAY; + return true; + } + if (type == RAY_TIME || type == -RAY_TIME) { *out = raw * 1000LL; return true; } + /* RAY_TIMESTAMP / -RAY_TIMESTAMP: ns → µs, floor toward -inf. Done with + * truncate-then-adjust rather than negating: the old `-((-raw)+999)/1000` + * overflowed for `raw` within 999 of INT64_MIN (the low edge of the + * representable range). */ + int64_t q = raw / 1000LL; + if (raw % 1000LL != 0 && raw < 0) q--; + *out = q; + return true; +} + +/* Extract one field from a raw temporal slot. Returns false when the source + * value is not representable in the µs extract domain (caller emits null). */ +static inline bool rte_extract_elem(int8_t t, int64_t raw, int field, int64_t* out) { + int64_t us; + if (!rte_to_us_ck(t, raw, &us)) return false; + *out = rte_extract_one(us, field); + return true; } -/* Inverse of rte_to_us for TIMESTAMP output paths (truncate). */ -static inline int64_t rte_us_to_ts_raw(int64_t us) { return us * 1000LL; } +/* Truncate a raw temporal slot to a TIMESTAMP-ns bucket. Returns false when + * the source, or the bucketed result, is outside the int64 nanosecond range + * (caller emits null): a value hundreds of millennia out cannot be a + * TIMESTAMP. The floor uses overflow-safe arithmetic and the ACTUAL bucketed + * result is range-checked (mirroring exec_date_trunc), so a value whose + * floored bucket is still representable — e.g. the low day boundary + * 1707.09.23 — round-trips instead of being rejected by a pre-floor headroom + * that disagreed with the DAG path. */ +static inline bool rte_trunc_elem(int8_t t, int64_t raw, int64_t bucket, int64_t* out_ns) { + int64_t us; + if (!rte_to_us_ck(t, raw, &us)) return false; + /* Floor `us` to the bucket boundary (toward -inf). `us - r` truncates + * toward zero (magnitude only shrinks, never overflows); a negative + * remainder needs one more bucket subtracted to floor toward -inf — the + * one step that can underflow (a DATE reaches `us` across the full µs + * range), so guard it. */ + int64_t r = us % bucket; + int64_t out_us = us - r; + if (r < 0) { + if (out_us < INT64_MIN + bucket) return false; + out_us -= bucket; + } + /* Range-check the ACTUAL floored result against the int64 nanosecond + * domain (out_us × 1000), matching exec_date_trunc — not a conservative + * pre-floor headroom, which rejected representable boundary values. */ + if (out_us > INT64_MAX / 1000LL || out_us < INT64_MIN / 1000LL) return false; + *out_ns = out_us * 1000LL; + return true; +} ray_t* ray_temporal_extract(ray_t* input, int field) { if (!input || RAY_IS_ERR(input)) return input; @@ -124,9 +175,9 @@ ray_t* ray_temporal_extract(ray_t* input, int field) { if (t != -RAY_DATE && t != -RAY_TIME && t != -RAY_TIMESTAMP) return ray_error("type", "extract: expected date/time/timestamp, got %s", ray_type_name(t)); if (RAY_ATOM_IS_NULL(input)) return ray_typed_null(-RAY_I64); - int64_t raw = input->i64; - int64_t us = rte_to_us(t, raw); - return ray_i64(rte_extract_one(us, field)); + int64_t ev; + if (!rte_extract_elem(t, input->i64, field, &ev)) return ray_typed_null(-RAY_I64); + return ray_i64(ev); } /* Vector input. */ @@ -158,31 +209,35 @@ ray_t* ray_temporal_extract(ray_t* input, int field) { const int32_t* d32 = (const int32_t*)base; if (src_has_nulls) { for (int64_t i = 0; i < len; i++) { - if (ray_vec_is_null(input, i)) { + if (ray_vec_is_null(input, i) || + !rte_extract_elem(t, (int64_t)d32[i], field, &out[i])) { out[i] = NULL_I64; ray_vec_set_null(result, i, true); - continue; } - out[i] = rte_extract_one(rte_to_us(t, (int64_t)d32[i]), field); } } else { for (int64_t i = 0; i < len; i++) - out[i] = rte_extract_one(rte_to_us(t, (int64_t)d32[i]), field); + if (!rte_extract_elem(t, (int64_t)d32[i], field, &out[i])) { + out[i] = NULL_I64; + ray_vec_set_null(result, i, true); + } } } else { const int64_t* d64 = (const int64_t*)base; if (src_has_nulls) { for (int64_t i = 0; i < len; i++) { - if (ray_vec_is_null(input, i)) { + if (ray_vec_is_null(input, i) || + !rte_extract_elem(t, d64[i], field, &out[i])) { out[i] = NULL_I64; ray_vec_set_null(result, i, true); - continue; } - out[i] = rte_extract_one(rte_to_us(t, d64[i]), field); } } else { for (int64_t i = 0; i < len; i++) - out[i] = rte_extract_one(rte_to_us(t, d64[i]), field); + if (!rte_extract_elem(t, d64[i], field, &out[i])) { + out[i] = NULL_I64; + ray_vec_set_null(result, i, true); + } } } return result; @@ -252,13 +307,13 @@ ray_t* ray_temporal_truncate(ray_t* input, int kind) { if (t != -RAY_DATE && t != -RAY_TIME && t != -RAY_TIMESTAMP) return ray_error("type", "truncate: expected date/time/timestamp, got %s", ray_type_name(t)); if (RAY_ATOM_IS_NULL(input)) return ray_typed_null(-RAY_TIMESTAMP); - int64_t us = rte_to_us(t, input->i64); int64_t bucket = (kind == RAY_EXTRACT_DAY) ? RTE_USEC_PER_DAY : RTE_USEC_PER_SEC; - int64_t r = us % bucket; - int64_t out_us = us - r - (r < 0 ? bucket : 0); - return ray_timestamp(rte_us_to_ts_raw(out_us)); + int64_t out_ns; + if (!rte_trunc_elem(t, input->i64, bucket, &out_ns)) + return ray_typed_null(-RAY_TIMESTAMP); + return ray_timestamp(out_ns); } /* Vector input. */ @@ -287,38 +342,30 @@ ray_t* ray_temporal_truncate(ray_t* input, int kind) { if (t == RAY_DATE || t == RAY_TIME) { const int32_t* d32 = (const int32_t*)base; if (src_has_nulls) { - for (int64_t i = 0; i < len; i++) { - if (ray_vec_is_null(input, i)) { - out[i] = NULL_I64; ray_vec_set_null(result, i, true); continue; + for (int64_t i = 0; i < len; i++) + if (ray_vec_is_null(input, i) || + !rte_trunc_elem(t, (int64_t)d32[i], bucket, &out[i])) { + out[i] = NULL_I64; ray_vec_set_null(result, i, true); } - int64_t us = rte_to_us(t, (int64_t)d32[i]); - int64_t r = us % bucket; - out[i] = rte_us_to_ts_raw(us - r - (r < 0 ? bucket : 0)); - } } else { - for (int64_t i = 0; i < len; i++) { - int64_t us = rte_to_us(t, (int64_t)d32[i]); - int64_t r = us % bucket; - out[i] = rte_us_to_ts_raw(us - r - (r < 0 ? bucket : 0)); - } + for (int64_t i = 0; i < len; i++) + if (!rte_trunc_elem(t, (int64_t)d32[i], bucket, &out[i])) { + out[i] = NULL_I64; ray_vec_set_null(result, i, true); + } } } else { const int64_t* d64 = (const int64_t*)base; if (src_has_nulls) { - for (int64_t i = 0; i < len; i++) { - if (ray_vec_is_null(input, i)) { - out[i] = NULL_I64; ray_vec_set_null(result, i, true); continue; + for (int64_t i = 0; i < len; i++) + if (ray_vec_is_null(input, i) || + !rte_trunc_elem(t, d64[i], bucket, &out[i])) { + out[i] = NULL_I64; ray_vec_set_null(result, i, true); } - int64_t us = rte_to_us(t, d64[i]); - int64_t r = us % bucket; - out[i] = rte_us_to_ts_raw(us - r - (r < 0 ? bucket : 0)); - } } else { - for (int64_t i = 0; i < len; i++) { - int64_t us = rte_to_us(t, d64[i]); - int64_t r = us % bucket; - out[i] = rte_us_to_ts_raw(us - r - (r < 0 ? bucket : 0)); - } + for (int64_t i = 0; i < len; i++) + if (!rte_trunc_elem(t, d64[i], bucket, &out[i])) { + out[i] = NULL_I64; ray_vec_set_null(result, i, true); + } } } return result; @@ -383,14 +430,26 @@ ray_t* exec_extract(ray_graph_t* g, ray_op_t* op) { if (IN32) { \ /* RAY_DATE: int32 days → µs; RAY_TIME: int32 ms → µs */ \ int32_t raw32 = ((const int32_t*)m.morsel_ptr)[i]; \ + /* A DATE hundreds of millennia out (int32 days) would \ + * overflow days×µs-per-day; it is not representable in \ + * the µs domain, so emit a null like a null input. */ \ + if (in_type == RAY_DATE && \ + ((int64_t)raw32 > INT64_MAX / USEC_PER_DAY || \ + (int64_t)raw32 < INT64_MIN / USEC_PER_DAY)) { \ + out[off + i] = NULL_I64; \ + ray_vec_set_null(result, off + i, true); \ + continue; \ + } \ us = (in_type == RAY_DATE) \ ? (int64_t)raw32 * USEC_PER_DAY \ : (int64_t)raw32 * 1000LL; \ } else { \ - /* RAY_TIMESTAMP: int64 nanoseconds → µs */ \ + /* RAY_TIMESTAMP: int64 nanoseconds → µs, floor toward \ + * -inf. Truncate-then-adjust instead of negating: the \ + * old -((-ns)+999)/1000 overflowed near INT64_MIN. */ \ int64_t ns = ((const int64_t*)m.morsel_ptr)[i]; \ - us = ns >= 0 ? ns / 1000LL \ - : -(((-ns) + 999LL) / 1000LL); \ + us = ns / 1000LL; \ + if (ns % 1000LL != 0 && ns < 0) us--; \ } \ if (field == RAY_EXTRACT_EPOCH) { \ out[off + i] = us; \ @@ -534,13 +593,29 @@ ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { int64_t us; \ if (IN32) { \ int32_t raw32 = ((const int32_t*)m.morsel_ptr)[i]; \ + /* Bound the DATE by the int64-NANOSECOND representable \ + * day range, not just the µs one: the YEAR/MONTH arms \ + * below re-multiply days_from_civil(...) — a day count \ + * floored DOWN to the period start, up to a year beyond \ + * `raw32` — by DT_USEC_PER_DAY, and the result is then \ + * ×1000 to nanoseconds. Matches rte_trunc_elem; a DATE \ + * outside this range is not a TIMESTAMP, so null it. */ \ + if (in_type == RAY_DATE && \ + ((int64_t)raw32 > INT64_MAX / 1000 / DT_USEC_PER_DAY || \ + (int64_t)raw32 < INT64_MIN / 1000 / DT_USEC_PER_DAY)) { \ + out[off + i] = NULL_I64; \ + ray_vec_set_null(result, off + i, true); \ + continue; \ + } \ us = (in_type == RAY_DATE) \ ? (int64_t)raw32 * DT_USEC_PER_DAY \ : (int64_t)raw32 * 1000LL; \ } else { \ + /* ns → µs, floor toward -inf, overflow-free (the old \ + * -((-ns)+999)/1000 overflowed near INT64_MIN). */ \ int64_t ns = ((const int64_t*)m.morsel_ptr)[i]; \ - us = ns >= 0 ? ns / 1000LL \ - : -(((-ns) + 999LL) / 1000LL); \ + us = ns / 1000LL; \ + if (ns % 1000LL != 0 && ns < 0) us--; \ } \ int64_t out_us; \ switch (field) { \ @@ -598,6 +673,12 @@ ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { out_us = us; \ break; \ } \ + /* Result must fit int64 nanoseconds (out_us × 1000). */ \ + if (out_us > INT64_MAX / 1000LL || out_us < INT64_MIN / 1000LL) { \ + out[off + i] = NULL_I64; \ + ray_vec_set_null(result, off + i, true); \ + continue; \ + } \ out[off + i] = out_us * 1000LL; /* µs → ns for RAY_TIMESTAMP */ \ } \ off += n; \ diff --git a/test/rfl/temporal/extract_trunc_overflow.rfl b/test/rfl/temporal/extract_trunc_overflow.rfl new file mode 100644 index 000000000..b5d948457 --- /dev/null +++ b/test/rfl/temporal/extract_trunc_overflow.rfl @@ -0,0 +1,94 @@ +;; extract_trunc_overflow.rfl — regression for signed-overflow UB in the +;; DATE/TIMESTAMP → microseconds conversion shared by the four temporal +;; decomposition kernels (standalone + DAG, extract + truncate). +;; +;; Bugs (all UBSan signed-overflow in src/ops/temporal.c): +;; * a DATE is int32 days, so an extreme value multiplied by µs-per-day +;; overflowed int64 — (dd (as 'DATE 2147483647)) tripped `raw * 86400000000`; +;; * the ns→µs floor for TIMESTAMP negated the input, so a value within +;; 999 of INT64_MIN overflowed `(-raw) + 999`; +;; * the DAG date_trunc YEAR/MONTH arms re-multiply days_from_civil(...) — a +;; day count floored down to the period start — by µs-per-day, so a DATE +;; that cleared the µs bound still overflowed the ns range (d.year of +;; (as 'DATE -106751991)). +;; +;; Fix: reject the un-representable DATE (emit null, as for a null input) and +;; do the TIMESTAMP floor with truncate-then-adjust so it never negates. The +;; truncate DATE bound is the int64-NANOSECOND representable day range so the +;; re-multiply cannot overflow. Covers both the standalone kernels +;; (yyyy/dd/mm/hh and (date …)) and the DAG kernels (dotted col.field). + +;; ── standalone extract atom path ──────────────────────────────────── +;; Extreme DATE (large magnitude, both signs) → null instead of overflow. +;; NB: -2147483648 == NULL_I32, i.e. a null DATE, so it would exit at the +;; null-input early-out and never exercise the guard; use a non-null value. +(nil? (yyyy (as 'DATE 2147483647))) -- true +(nil? (mm (as 'DATE 2147483647))) -- true +(nil? (dd (as 'DATE -200000000))) -- true +;; Minimum-edge TIMESTAMP no longer overflows the ns→µs floor. +(mm (as 'TIMESTAMP -9223372036854775807)) -- 9 +(hh (as 'TIMESTAMP -9223372036854775807)) -- 0 +;; Ordinary values are unaffected. +(yyyy 2024.03.15D14:27:31.123456789) -- 2024 +(mm 2024.03.15D14:27:31.123456789) -- 3 +(dd 2024.03.15D14:27:31.123456789) -- 15 +(hh 2024.03.15D14:27:31.123456789) -- 14 +(yyyy 2024.03.15) -- 2024 +(dd 2000.01.01) -- 1 + +;; ── standalone extract vector path ────────────────────────────────── +;; A mix of ordinary, null and extreme rows: only the extreme row nulls. +(set Ex (yyyy (as 'DATE [0 366 2147483647]))) +(at Ex 0) -- 2000 +(at Ex 1) -- 2001 +(nil? (at Ex 2)) -- true + +;; ── DAG extract kernel (dotted-path in select) ────────────────────── +(set T (table [d ts] (list (as 'DATE [0 366 2147483647]) (as 'TIMESTAMP [0 86400000000000 -9223372036854775807])))) +(set Ey (at (select {v: d.yyyy from: T}) 'v)) +(at Ey 0) -- 2000 +(at Ey 1) -- 2001 +(nil? (at Ey 2)) -- true +;; TIMESTAMP min edge decodes without overflow through the DAG kernel too. +(set Em (at (select {v: ts.mm from: T}) 'v)) +(at Em 0) -- 1 +(at Em 2) -- 9 + +;; ── DAG date_trunc kernel (dotted-path .date) ─────────────────────── +;; Extreme DATE truncates to null (not representable as a TIMESTAMP); +;; ordinary rows are unchanged. +(set Td (at (select {v: d.date from: T}) 'v)) +(at Td 0) -- 2000.01.01D00:00:00.000000000 +(at Td 1) -- 2001.01.01D00:00:00.000000000 +(nil? (at Td 2)) -- true + +;; ── DAG date_trunc YEAR / MONTH arms (the re-multiply path) ────────── +;; d.year of a large-magnitude negative DATE used to overflow the second +;; days_from_civil(...) * µs-per-day multiply; it must null, not UB. +(set Tn (table [d] (list (as 'DATE [-106751991 8840 200000000])))) +(set Ty (at (select {v: d.year from: Tn}) 'v)) +(nil? (at Ty 0)) -- true +(at Ty 1) -- 2024.01.01D00:00:00.000000000 +(nil? (at Ty 2)) -- true +(set Tmo (at (select {v: d.month from: Tn}) 'v)) +(nil? (at Tmo 0)) -- true +(at Tmo 1) -- 2024.03.01D00:00:00.000000000 +(nil? (at Tmo 2)) -- true + +;; ── standalone truncate kernel via (date ) ──────────────── +;; Ordinary values truncate to the day; values outside the int64-ns range +;; null instead of wrapping. +(date (as 'TIMESTAMP 86400000000000)) -- 2000.01.02D00:00:00.000000000 +(nil? (date (as 'TIMESTAMP -9223372036854775807))) -- true +(date (as 'DATE 8766)) -- 2024.01.01D00:00:00.000000000 +(nil? (date (as 'DATE 2000000000))) -- true + +;; ── low day boundary: floor result is representable (PR #386 review) ─ +;; A TIMESTAMP whose day-floored value is still inside the int64-ns range +;; must round-trip, not null. The old pre-floor headroom rejected it — so +;; the standalone (date …) path disagreed with the DAG ts.date path. The +;; floor now uses overflow-safe arithmetic and range-checks the actual +;; result, so both paths agree at the boundary. +(date (as 'TIMESTAMP -9223286400000000000)) -- 1707.09.23D00:00:00.000000000 +(set Tbnd (table [ts] (list (as 'TIMESTAMP [-9223286400000000000])))) +(at (at (select {v: ts.date from: Tbnd}) 'v) 0) -- 1707.09.23D00:00:00.000000000