diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ea4efe16..19773e88 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,5 +1,5 @@ name: "Bug report" -description: Report a problem with the InQL package, tests, docs, or CI. +description: Report a problem with the IncQL package, tests, docs, or CI. title: "bug - " type: Bug labels: ["bug"] @@ -8,7 +8,7 @@ body: attributes: value: | Thanks for reporting — a **minimal repro** (snippet, commands, or RFC section) helps a lot. - Toolchain problems **outside this repo** (generic Incan, not InQL\’s package or specs) belong in the [Incan compiler](https://github.com/encero-systems/incan/issues) tracker. + Toolchain problems **outside this repo** (generic Incan, not IncQL\’s package or specs) belong in the [Incan compiler](https://github.com/encero-systems/incan/issues) tracker. - type: dropdown id: area attributes: @@ -54,10 +54,10 @@ body: id: env attributes: label: Environment - description: OS, InQL commit or version, and `incan` version/commit if you hit it while building or testing. + description: OS, IncQL commit or version, and `incan` version/commit if you hit it while building or testing. placeholder: | OS: - InQL (commit or version): + IncQL (commit or version): incan (version or commit): Command: validations: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 7f0bc9be..750b2148 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: true contact_links: - name: Documentation - url: https://github.com/encero-systems/InQL#readme - about: Start here for InQL overview, RFC index, and project links. + url: https://github.com/encero-systems/IncQL#readme + about: Start here for IncQL overview, RFC index, and project links. diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml index c1a831c6..95546674 100644 --- a/.github/ISSUE_TEMPLATE/documentation.yml +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -52,7 +52,7 @@ body: attributes: label: Audience options: - - InQL user (library, queries, datasets) + - IncQL user (library, queries, datasets) - Contributor (RFCs, package layout, CI) - Incan / compiler integrator - Not sure diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 15ff3ef7..539bf294 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,5 +1,5 @@ name: "Feature request" -description: Propose an enhancement to the InQL package, specs, or contributor workflow. +description: Propose an enhancement to the IncQL package, specs, or contributor workflow. title: "feature - " type: Feature labels: ["feature"] diff --git a/.github/ISSUE_TEMPLATE/rfc_proposal.yml b/.github/ISSUE_TEMPLATE/rfc_proposal.yml index 8fd29385..e6a26331 100644 --- a/.github/ISSUE_TEMPLATE/rfc_proposal.yml +++ b/.github/ISSUE_TEMPLATE/rfc_proposal.yml @@ -1,15 +1,15 @@ name: "RFC proposal" -description: Propose a significant InQL specification or surface change that should go through the RFC process. +description: Propose a significant IncQL specification or surface change that should go through the RFC process. labels: ["RFC"] body: - type: markdown attributes: value: | Use this issue to propose an RFC topic and get early alignment. - If accepted, we typically ask for a PR adding an RFC under [`docs/rfcs/`](https://github.com/encero-systems/InQL/tree/main/docs/rfcs): copy [**TEMPLATE.md**](https://github.com/encero-systems/InQL/blob/main/docs/rfcs/TEMPLATE.md), name it `NNN_short_slug.md`, and follow that structure. For workflow and conventions, see [**Writing InQL RFCs**](https://github.com/encero-systems/InQL/blob/main/docs/contributing/writing_rfcs.md). + If accepted, we typically ask for a PR adding an RFC under [`docs/rfcs/`](https://github.com/encero-systems/IncQL/tree/main/docs/rfcs): copy [**TEMPLATE.md**](https://github.com/encero-systems/IncQL/blob/main/docs/rfcs/TEMPLATE.md), name it `NNN_short_slug.md`, and follow that structure. For workflow and conventions, see [**Writing IncQL RFCs**](https://github.com/encero-systems/IncQL/blob/main/docs/contributing/writing_rfcs.md). - **InQL RFCs** cover the relational layer, dataset carriers, Substrait contract, `query {}` syntax, execution context, and related docs. - Pure Incan language/compiler changes without InQL impact usually belong in the **Incan** repository instead. + **IncQL RFCs** cover the relational layer, dataset carriers, Substrait contract, `query {}` syntax, execution context, and related docs. + Pure Incan language/compiler changes without IncQL impact usually belong in the **Incan** repository instead. - type: dropdown id: area attributes: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ca40aec..7f061b1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ -# InQL CI — Incan library package +# IncQL CI — Incan library package # # Checks out the pinned Incan compiler source, installs it through Incan's -# downstream install action, then runs the InQL package checks against that +# downstream install action, then runs the IncQL package checks against that # local binary. name: CI @@ -27,8 +27,8 @@ env: INCAN_TEST_SHARED_TARGET_DIR: ${{ github.workspace }}/.incan-generated-cargo-target jobs: - inql: - name: InQL (build + test) + incql: + name: IncQL (build + test) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -36,7 +36,7 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - - name: Check out InQL + - name: Check out IncQL uses: actions/checkout@v4 - name: Check out Incan @@ -50,17 +50,17 @@ jobs: uses: ./incan/.github/actions/install-incan with: profile: debug - cache-shared-key: inql-incan-${{ runner.os }}-${{ env.EXPECTED_INCAN_VERSION }} + cache-shared-key: incql-incan-${{ runner.os }}-${{ env.EXPECTED_INCAN_VERSION }} - - name: Cache generated InQL Cargo artifacts + - name: Cache generated IncQL Cargo artifacts uses: actions/cache@v4 with: path: .incan-generated-cargo-target - key: inql-generated-cargo-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}-${{ hashFiles('incan.lock', 'incan.toml') }} + key: incql-generated-cargo-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}-${{ hashFiles('incan.lock', 'incan.toml') }} restore-keys: | - inql-generated-cargo-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}- + incql-generated-cargo-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}- - - name: Cache generated InQL Rust metadata + - name: Cache generated IncQL Rust metadata uses: actions/cache@v4 with: path: | @@ -69,10 +69,10 @@ jobs: target/incan_lock/rust_inspect/**/src/main.rs target/incan_lock/rust_inspect/**/.incan_rust_inspect_cache.json target/incan_lock/rust_inspect/**/.incan_rust_inspect_fingerprint - key: inql-rust-inspect-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}-${{ hashFiles('incan.lock', 'incan.toml') }}-${{ hashFiles('src/**/*.incn', 'tests/**/*.incn', 'scripts/**/*.incn') }} + key: incql-rust-inspect-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}-${{ hashFiles('incan.lock', 'incan.toml') }}-${{ hashFiles('src/**/*.incn', 'tests/**/*.incn', 'scripts/**/*.incn') }} restore-keys: | - inql-rust-inspect-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}-${{ hashFiles('incan.lock', 'incan.toml') }}- - inql-rust-inspect-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}- + incql-rust-inspect-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}-${{ hashFiles('incan.lock', 'incan.toml') }}- + incql-rust-inspect-${{ runner.os }}-incan-${{ env.EXPECTED_INCAN_VERSION }}- - name: Show toolchain run: | diff --git a/AGENTS.md b/AGENTS.md index c003e5a1..f555742c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ -# Agent Instructions for InQL +# Agent Instructions for IncQL @@ -21,7 +21,7 @@ [incan-docsite-loop]: https://github.com/encero-systems/incan/blob/main/workspaces/docs-site/docs/contributing/tutorials/book/08_docsite_contributor_loop.md [incan-agents-docs-workflow]: https://github.com/encero-systems/incan/blob/main/AGENTS.md#docs-site-workflow-mkdocs-material -**InQL** is the typed **data logic plane** for [Incan][incan-repo]: relational queries, schema-aware table transformations, and streaming-shaped relational work, with a clear split from orchestration and engine-specific runtime in the authoring model. **This repository** holds the InQL **Incan library package** (`.incn` sources) and **normative RFCs** under `docs/rfcs/`. The **Incan compiler** (Rust) that implements parsing, checking, and lowering for InQL surfaces lives in the **Incan** repository. +**IncQL** is the typed **data logic plane** for [Incan][incan-repo]: relational queries, schema-aware table transformations, and streaming-shaped relational work, with a clear split from orchestration and engine-specific runtime in the authoring model. **This repository** holds the IncQL **Incan library package** (`.incn` sources) and **normative RFCs** under `docs/rfcs/`. The **Incan compiler** (Rust) that implements parsing, checking, and lowering for IncQL surfaces lives in the **Incan** repository. This document guides AI agents and contributors working **in this repo**. For compiler implementation, Rust conventions, and the full toolchain pipeline, use **[Incan `AGENTS.md`][incan-agents]** and **[Incan `CONTRIBUTING.md`][incan-contributing]**. @@ -36,9 +36,9 @@ This document guides AI agents and contributors working **in this repo**. For co | Project overview | [README.md][readme] | | Contributing (human workflow) | [CONTRIBUTING.md][contributing] | | Repo vs compiler placement | [docs/architecture.md][architecture] | -| Normative InQL design | [docs/rfcs/][rfcs-index] | -| InQL RFC file template | [docs/rfcs/TEMPLATE.md][rfc-template] | -| Writing InQL RFCs (how-to) | [docs/contributing/writing_rfcs.md][writing-rfcs] | +| Normative IncQL design | [docs/rfcs/][rfcs-index] | +| IncQL RFC file template | [docs/rfcs/TEMPLATE.md][rfc-template] | +| Writing IncQL RFCs (how-to) | [docs/contributing/writing_rfcs.md][writing-rfcs] | | GitHub issue templates | [.github/ISSUE_TEMPLATE/][issue-templates] | | Incan agent rules (Rust, compiler pipeline, skills) | [Incan `AGENTS.md`][incan-agents] | | Incan docs-site conventions | [Contributor loop][incan-docsite-loop] · [Markdown / MkDocs in AGENTS][incan-agents-docs-workflow] | @@ -47,9 +47,9 @@ This document guides AI agents and contributors working **in this repo**. For co | Change | Primary repo | | --------------------------------------------------------------------------------------------- | ----------------------- | -| InQL **RFC** text, `README`, `docs/*` (except normative rules in `__research__/`) | **This repo** | -| InQL **library** API and tests in `.incn` | **This repo** | -| Lexer/parser/typechecker/lowering/**Rust** for InQL syntax, `query {}`, `DataSet` integration | [**Incan**][incan-repo] | +| IncQL **RFC** text, `README`, `docs/*` (except normative rules in `__research__/`) | **This repo** | +| IncQL **library** API and tests in `.incn` | **This repo** | +| Lexer/parser/typechecker/lowering/**Rust** for IncQL syntax, `query {}`, `DataSet` integration | [**Incan**][incan-repo] | Normative behavior is defined in **`docs/rfcs/`**. If package code and an RFC disagree, treat it as a bug unless the RFC is explicitly superseded. @@ -71,7 +71,7 @@ Normative behavior is defined in **`docs/rfcs/`**. If package code and an RFC di 1. **Branch from `main`**: Prefer `/-` (e.g. `feature/8-rfc-table-automation`, `docs/9-mkdocs-ci`), matching team practice. 2. **Follow RFCs**: Behavior changes should be reflected in the right `docs/rfcs/*.md` (or a new RFC) before or alongside code in the appropriate repository. 3. **Run the local gate**: `make ci` (or at least `make fmt-check`, `make build`, `make test`) before considering work done for **this** repo. -4. **Version sync**: If you bump the package version, update **both** [incan.toml][incan-toml] (`[project] version`) and [src/metadata.incn][metadata-incn] (`inql_version()`) in the same commit (see [CONTRIBUTING.md][contributing]). +4. **Version sync**: If you bump the package version, update **both** [incan.toml][incan-toml] (`[project] version`) and [src/metadata.incn][metadata-incn] (`incql_version()`) in the same commit (see [CONTRIBUTING.md][contributing]). 5. **Documentation**: User-facing or spec changes should update `README.md`, relevant `docs/*`, or RFCs as appropriate. Keep prose markdown **without hard wrapping** (natural paragraphs). - Use RFCs for normative design and design history. - Use `docs/language/reference/` for current API/contracts. @@ -85,7 +85,7 @@ The function-catalog RFC series is cumulative. For RFC 016 onward, review each P Function construction is a first-class review axis. Check that new helpers are constructed through the declaration-side pattern: public helper metadata lives in decorators/partials, examples live in docstrings, names/signatures/parameter and return types derive from checked helper metadata where possible, and registry entries are loaded from the helper declaration rather than a central hardcoded list. Keep aggregate measures, scalar applications, window calls, and generator applications as distinct semantic shapes even when they share registry metadata; do not collapse them into ad hoc per-function models or backend-specific shortcuts. -DataFusion is the first adapter, not InQL’s semantic owner. Do not encode DataFusion-only behavior in Substrait IR, do not model core-function “unsupported” as a normal Substrait state, and do not use SQL/string-script generation when DataFusion exposes a typed API. Invalid context belongs in authoring/Prism/lowering validation; backend inability belongs in adapter capability or error handling. +DataFusion is the first adapter, not IncQL’s semantic owner. Do not encode DataFusion-only behavior in Substrait IR, do not model core-function “unsupported” as a normal Substrait state, and do not use SQL/string-script generation when DataFusion exposes a typed API. Invalid context belongs in authoring/Prism/lowering validation; backend inability belongs in adapter capability or error handling. Lessons from the RFC 016–023 closeout: @@ -123,7 +123,7 @@ Requires `incan` on `PATH`, or `make build INCAN=/path/to/incan`. CI builds Inca ### Comments and explanatory prose in code - Treat comments in this repo as part of the readability surface, not just implementation debris. -- This matters more here than in a mature mainstream language codebase because Incan/InQL syntax, planning boundaries, and lowering shapes are still unfamiliar to many readers. +- This matters more here than in a mature mainstream language codebase because Incan/IncQL syntax, planning boundaries, and lowering shapes are still unfamiliar to many readers. - Do **not** apply a simplistic "remove comments that restate the code" rule. Comments that orient the reader, explain data-shape assumptions, or clarify which phase/boundary a block belongs to are valuable even when they partially restate the code. - Be especially willing to keep or add short explanatory comments in: - public API modules @@ -160,11 +160,11 @@ Requires `incan` on `PATH`, or `make build INCAN=/path/to/incan`. CI builds Inca This repository does **not** host the compiler Rust codebase. If your task includes changes under the Incan workspace: - Treat **[Incan `AGENTS.md`][incan-agents]** as authoritative for **no `.unwrap()` / `.expect()`**, clippy, rustdoc, and pipeline boundaries. -- Use `cargo test`, `cargo clippy`, and snapshot workflows **there**, not as substitutes for `make ci` **here** when you only touch InQL. +- Use `cargo test`, `cargo clippy`, and snapshot workflows **there**, not as substitutes for `make ci` **here** when you only touch IncQL. ## Cursor skills (optional) -Reusable Incan workflows live in the Incan repo under `.cursor/skills/` (e.g. `/write-rfc`, `/review-rfc`, `/bump-rfc`). Use them when the task spans Incan or when drafting/reviewing InQL RFCs and you want the same structural discipline — adapted to **`docs/rfcs/`** and InQL’s numbering. +Reusable Incan workflows live in the Incan repo under `.cursor/skills/` (e.g. `/write-rfc`, `/review-rfc`, `/bump-rfc`). Use them when the task spans Incan or when drafting/reviewing IncQL RFCs and you want the same structural discipline — adapted to **`docs/rfcs/`** and IncQL’s numbering. ## PR checklist (this repo) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80514673..aab03c0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,21 +1,21 @@ -# Contributing to InQL +# Contributing to IncQL -Thank you for your interest in InQL — the typed relational layer for [Incan][incan-repo]. This document is the entry point for contributing to **this repository** (the InQL package and its design RFCs). +Thank you for your interest in IncQL — the typed relational layer for [Incan][incan-repo]. This document is the entry point for contributing to **this repository** (the IncQL package and its design RFCs). ## Start here | Resource | Purpose | | -------- | ------- | -| [README.md][readme] | What InQL is and how to build the library | +| [README.md][readme] | What IncQL is and how to build the library | | [AGENTS.md][agents] | AI/agent and maintainer rules; repo vs Incan boundaries | | [docs/architecture.md][architecture] | How this repo fits next to the Incan compiler | | [docs/rfcs/][rfcs-index] | Normative design proposals (behavior changes start here) | -| [Writing InQL RFCs][writing-rfcs] | How to draft RFCs, workflow, and tips | +| [Writing IncQL RFCs][writing-rfcs] | How to draft RFCs, workflow, and tips | | [Incan `CONTRIBUTING.md`][incan-contributing] | Compiler, tooling, and Rust workspace workflow | -| [Incan docs-site contributor loop][incan-docsite-loop] | Divio layout, snippets, checklist for Incan’s MkDocs site — mirror these patterns when shaping InQL `docs/` | +| [Incan docs-site contributor loop][incan-docsite-loop] | Divio layout, snippets, checklist for Incan’s MkDocs site — mirror these patterns when shaping IncQL `docs/` | | [Incan AGENTS — Docs-site workflow][incan-agents-docs-workflow] | Prose **without hard wrapping**, `mkdocs build --strict`, Material-friendly Markdown | -**Compiler and language implementation** (lexer, parser, typechecker, lowering, codegen for `query {}` and related surfaces) lives in the **Incan** repository. Use that project’s docs and gates when you change the toolchain. Use **this** repo for the InQL **library source** (`.incn`) and **InQL RFCs** that specify the relational surface. +**Compiler and language implementation** (lexer, parser, typechecker, lowering, codegen for `query {}` and related surfaces) lives in the **Incan** repository. Use that project’s docs and gates when you change the toolchain. Use **this** repo for the IncQL **library source** (`.incn`) and **IncQL RFCs** that specify the relational surface. ## Getting started @@ -25,8 +25,8 @@ Thank you for your interest in InQL — the typed relational layer for [Incan][i 2. **Clone this repository** ```bash - git clone https://github.com/encero-systems/InQL - cd InQL + git clone https://github.com/encero-systems/IncQL + cd IncQL ``` 3. **Build and verify** @@ -59,7 +59,7 @@ See [docs/architecture.md][architecture] for a concise map. In short: ## Changing behavior -1. **Specify the change in an InQL RFC** under `docs/rfcs/` (or amend an existing RFC). New documents should start from [RFC template][rfc-template]; follow [Writing InQL RFCs][writing-rfcs] for workflow so naming, typing, and lowering rules stay coherent across `query {}`, carriers, and optional pipe-forward. +1. **Specify the change in an IncQL RFC** under `docs/rfcs/` (or amend an existing RFC). New documents should start from [RFC template][rfc-template]; follow [Writing IncQL RFCs][writing-rfcs] for workflow so naming, typing, and lowering rules stay coherent across `query {}`, carriers, and optional pipe-forward. 2. **Implement** in the right place: library APIs here when they are ordinary Incan code; compiler or stdlib changes in the Incan repo as needed. 3. **Keep the [README][readme] and docs accurate** for anything a new user or contributor would notice. @@ -72,7 +72,7 @@ See [docs/architecture.md][architecture] for a concise map. In short: ### Comments and readability - In this repository, comments are part of the contributor-facing readability contract. -- Do not assume the usual "remove comments that restate the code" heuristic applies cleanly here. Incan/InQL is still a new language surface for most readers, so short explanatory comments often pull real weight even when they partially echo the code. +- Do not assume the usual "remove comments that restate the code" heuristic applies cleanly here. Incan/IncQL is still a new language surface for most readers, so short explanatory comments often pull real weight even when they partially echo the code. - Keep or add concise comments that explain: - what phase or boundary a block belongs to - what shape of data is being parsed or transformed @@ -103,10 +103,10 @@ See [docs/architecture.md][architecture] for a concise map. In short: ## Version bumps -InQL carries its version in two places that **must stay in sync**: +IncQL carries its version in two places that **must stay in sync**: 1. `incan.toml` — `[project] version = "…"` -2. `src/metadata.incn` — the string returned by `inql_version()` +2. `src/metadata.incn` — the string returned by `incql_version()` Bump both in the same commit. @@ -132,7 +132,7 @@ The auto-label workflow uses a shared **organization-level GitHub App** installa | `TRIAGE_APP_INSTALLATION_ID` | Installation ID for the **organization-level** app installation | | `TRIAGE_APP_PRIVATE_KEY` | App private key (PEM) | -Install the app on the `encero-systems` organization and grant it access to `InQL` (and any future repositories that should share triage automation). Without these secrets the workflow fails at the token step. +Install the app on the `encero-systems` organization and grant it access to `IncQL` (and any future repositories that should share triage automation). Without these secrets the workflow fails at the token step. ## Pull request guidelines @@ -145,7 +145,7 @@ Install the app on the `encero-systems` organization and grant it access to `InQ When you open an issue, GitHub offers **templates** under [`.github/ISSUE_TEMPLATE/`][issue-templates] (bug report, feature request, chore, documentation, RFC proposal), aligned with the Incan project’s shape. -Open an issue on this repository for InQL-specific design or package questions; use the [Incan repository][incan-repo] for compiler and language issues that are not InQL-scoped. +Open an issue on this repository for IncQL-specific design or package questions; use the [Incan repository][incan-repo] for compiler and language issues that are not IncQL-scoped. diff --git a/Makefile b/Makefile index 8bccca83..495bd542 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# InQL — Incan library package +# IncQL — Incan library package # ============================= # # Requires `incan` on your PATH (build/install from the Incan compiler repository). @@ -13,7 +13,7 @@ export INCAN_NO_BANNER ?= 1 .PHONY: help help: ## Show Make targets - @echo "\033[1mInQL\033[0m — typed relational layer (Incan library)." + @echo "\033[1mIncQL\033[0m — typed relational layer (Incan library)." @echo "Requires \033[36m$(INCAN)\033[0m on PATH, or set \033[36mINCAN=\033[0m/path/to/incan." @echo "" @grep -E '^[a-zA-Z0-9_.-]+:.*?##' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' @@ -25,19 +25,19 @@ help: ## Show Make targets # Test discovery defaults to `.` and walks the whole tree. CI checks out the # Incan compiler under `./incan/`; running `incan test` without a path would # pick up compiler integration tests (e.g. `incan/tests/test_*.incn`), which -# are not InQL package tests. Scope to `tests/` only (see INQL_FMT_DIRS). +# are not IncQL package tests. Scope to `tests/` only (see INCQL_FMT_DIRS). -INQL_TEST_DIR := tests +INCQL_TEST_DIR := tests .PHONY: build build: ## Build the library (`incan build --lib`) - @echo "\033[1mBuilding InQL library...\033[0m" + @echo "\033[1mBuilding IncQL library...\033[0m" @$(INCAN) build --lib .PHONY: test test: ## Run package tests (`incan test tests`) - @echo "\033[1mRunning InQL tests...\033[0m" - @$(INCAN) test $(INQL_TEST_DIR) + @echo "\033[1mRunning IncQL tests...\033[0m" + @$(INCAN) test $(INCQL_TEST_DIR) .PHONY: vocab-companion-test vocab-companion-test: ## Run Rust tests for the query-block vocabulary companion @@ -58,47 +58,47 @@ registry-metadata: ## Validate RFC 014 function registry checked API metadata .PHONY: build-locked build-locked: ## Build with `--locked` (stricter; requires current `incan.lock`) - @echo "\033[1mBuilding InQL library (locked)...\033[0m" + @echo "\033[1mBuilding IncQL library (locked)...\033[0m" @$(INCAN) build --lib --locked .PHONY: test-locked test-locked: ## Run tests with `--locked` - @echo "\033[1mRunning InQL tests (locked)...\033[0m" - @$(INCAN) test $(INQL_TEST_DIR) --locked + @echo "\033[1mRunning IncQL tests (locked)...\033[0m" + @$(INCAN) test $(INCQL_TEST_DIR) --locked # ============================================================================= # Formatting (Incan source) # ============================================================================= # -# Scope to InQL-owned source paths. CI checks out the Incan compiler under +# Scope to IncQL-owned source paths. CI checks out the Incan compiler under # `./incan/`; formatting `.` would walk that tree and fail on stdlib snapshots # and test fixtures that are not meant for `incan fmt`. Standalone example # packages are listed by source directory so generated `target/` output stays # outside the formatting walk. -INQL_FMT_DIRS := src tests examples/advanced_retail_query_blocks/src -INQL_FMT_FILES := examples/*.incn +INCQL_FMT_DIRS := src tests examples/advanced_retail_query_blocks/src +INCQL_FMT_FILES := examples/*.incn .PHONY: fmt fmt: ## Format package `.incn` sources (`incan fmt` per directory) @echo "\033[1mFormatting Incan sources (package dirs)...\033[0m" - @for d in $(INQL_FMT_DIRS); do \ + @for d in $(INCQL_FMT_DIRS); do \ if [ -d "$$d" ]; then $(INCAN) fmt "$$d"; fi; \ done - @for f in $(INQL_FMT_FILES); do \ + @for f in $(INCQL_FMT_FILES); do \ if [ -f "$$f" ]; then $(INCAN) fmt "$$f"; fi; \ done .PHONY: fmt-check fmt-check: ## Check formatting without writing (`incan fmt --check` per directory) @echo "\033[1mChecking Incan source formatting (package dirs)...\033[0m" - @for d in $(INQL_FMT_DIRS); do \ + @for d in $(INCQL_FMT_DIRS); do \ if [ -d "$$d" ]; then \ echo "\033[1m -> $$d/\033[0m"; \ $(INCAN) fmt --check "$$d" || exit $$?; \ fi; \ done - @for f in $(INQL_FMT_FILES); do \ + @for f in $(INCQL_FMT_FILES); do \ if [ -f "$$f" ]; then \ echo "\033[1m -> $$f\033[0m"; \ $(INCAN) fmt --check "$$f" || exit $$?; \ @@ -118,14 +118,14 @@ pre-commit: fmt-check test-style vocab-companion-test registry-metadata build te @echo "\033[32m✓ pre-commit gate passed\033[0m" .PHONY: ci -ci: fmt-check test-style vocab-companion-test registry-metadata build test smoke-consumer ## Same steps as GitHub Actions `inql` job +ci: fmt-check test-style vocab-companion-test registry-metadata build test smoke-consumer ## Same steps as GitHub Actions `incql` job @echo "\033[32m✓ ci gate passed\033[0m" .PHONY: verify verify: ci ## Alias for `ci` .PHONY: smoke-consumer -smoke-consumer: ## Verify InQL works as a pub dependency from a fresh consumer project +smoke-consumer: ## Verify IncQL works as a pub dependency from a fresh consumer project @echo "\033[1mRunning pub-consumer smoke check...\033[0m" @bash scripts/smoke_pub_consumer.sh diff --git a/README.md b/README.md index 06088ec7..35be75c5 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ -# InQL +# IncQL -**InQL** is the **typed data logic plane** for [Incan](https://github.com/encero-systems/incan): the place where you express **relational queries**, **schema-aware table transformations**, and **streaming-shaped relational work** with compile-time checks, without folding orchestration, catalogs, or engine-specific runtime into the authoring model. Row shapes come from Incan `model` types; column, join, and alias rules are part of one semantic core whether you use `query { }` blocks, method chains on `DataSet[T]` carriers, or (later) optional pipe-forward (`|>`). +**IncQL** is the **typed data logic plane** for [Incan](https://github.com/encero-systems/incan): the place where you express **relational queries**, **schema-aware table transformations**, and **streaming-shaped relational work** with compile-time checks, without folding orchestration, catalogs, or engine-specific runtime into the authoring model. Row shapes come from Incan `model` types; column, join, and alias rules are part of one semantic core whether you use `query { }` blocks, method chains on `DataSet[T]` carriers, or (later) optional pipe-forward (`|>`). -**What InQL is not:** It is not a pipeline or workflow framework, not a semantic catalog, and not a catch-all that swallows execution concerns. It owns **data logic**: query authoring, relational plan shape, resolution and schema flow, typed carrier semantics, and **backend-neutral logical intent**. Execution, binding, and operational semantics live in the layer below (session, adapters, runners). +**What IncQL is not:** It is not a pipeline or workflow framework, not a semantic catalog, and not a catch-all that swallows execution concerns. It owns **data logic**: query authoring, relational plan shape, resolution and schema flow, typed carrier semantics, and **backend-neutral logical intent**. Execution, binding, and operational semantics live in the layer below (session, adapters, runners). -**Why it matters:** Raw SQL strings and untyped rows defer mistakes to runtime. InQL keeps relational work **in the language**: schemas are ordinary models, invalid references and many aggregation mistakes are caught where Incan promises checking, and plans are intended to lower to **Apache Substrait** so logical intent stays portable while credentials and physical reads are resolved outside the normative plan story. +**Why it matters:** Raw SQL strings and untyped rows defer mistakes to runtime. IncQL keeps relational work **in the language**: schemas are ordinary models, invalid references and many aggregation mistakes are caught where Incan promises checking, and plans are intended to lower to **Apache Substrait** so logical intent stays portable while credentials and physical reads are resolved outside the normative plan story. -**Bottom line:** InQL is where you write **checked relational logic** in Incan: the same naming and schema rules apply to `query { }` blocks and `DataSet[T]` APIs, and **execution stays downstream** (sessions, adapters, runners) and consistent. +**Bottom line:** IncQL is where you write **checked relational logic** in Incan: the same naming and schema rules apply to `query { }` blocks and `DataSet[T]` APIs, and **execution stays downstream** (sessions, adapters, runners) and consistent. ## What you get @@ -25,7 +25,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for workflow and [architecture.md](docs/a ## Design (RFCs) -Normative proposals live under **[docs/rfcs/](docs/rfcs/README.md)**. InQL’s RFC series is separate from [Incan’s RFC index](https://github.com/encero-systems/incan/tree/main/workspaces/docs-site/docs/RFCs). +Normative proposals live under **[docs/rfcs/](docs/rfcs/README.md)**. IncQL’s RFC series is separate from [Incan’s RFC index](https://github.com/encero-systems/incan/tree/main/workspaces/docs-site/docs/RFCs). | RFC | Topic | | ------- | ------------------------------------------------------------------------------------------------ | @@ -46,7 +46,7 @@ Normative proposals live under **[docs/rfcs/](docs/rfcs/README.md)**. InQL’s R - `src/lib.incn` — public exports - `src/` — library modules - `tests/` — tests -- `.github/workflows/` — CI (builds the pinned Incan release branch, then runs InQL checks) +- `.github/workflows/` — CI (builds the pinned Incan release branch, then runs IncQL checks) Build and test from this repo root (with `incan` on your `PATH`): diff --git a/docs/README.md b/docs/README.md index e1155177..93eac12f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ -# InQL documentation +# IncQL documentation -This directory holds the public documentation for the InQL project. +This directory holds the public documentation for the IncQL project. Use the docs tree like this: @@ -35,7 +35,7 @@ Use the docs tree like this: ### Understand the system design -1. [InQL architecture][architecture] +1. [IncQL architecture][architecture] 2. [RFC index][rfcs-index] 3. Relevant RFCs for deeper normative context diff --git a/docs/architecture.md b/docs/architecture.md index ebb70284..4c3c717b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,19 +1,19 @@ -# InQL architecture +# IncQL architecture -This document describes the architectural model of **InQL**. It is scoped to the InQL repository and its relationship to the Incan compiler, not to product orchestration or engine-specific operational concerns. +This document describes the architectural model of **IncQL**. It is scoped to the IncQL repository and its relationship to the Incan compiler, not to product orchestration or engine-specific operational concerns. -## What InQL is +## What IncQL is -InQL is two things that evolve together: +IncQL is two things that evolve together: -1. **A specification** under [docs/rfcs/][inql-rfcs]: naming and core semantics, dataset carriers, Substrait emission, query authoring, the execution boundary, and the internal planning substrate. +1. **A specification** under [docs/rfcs/][incql-rfcs]: naming and core semantics, dataset carriers, Substrait emission, query authoring, the execution boundary, and the internal planning substrate. 2. **An Incan library package**: `.incn` modules built with `incan build --lib`, consumed by Incan programs as a typed relational package. -The Incan compiler remains responsible for parsing, typechecking, lowering, and Rust/code generation. The InQL repo holds the author-facing package, its documentation, and the RFCs that define what that package is supposed to mean. +The Incan compiler remains responsible for parsing, typechecking, lowering, and Rust/code generation. The IncQL repo holds the author-facing package, its documentation, and the RFCs that define what that package is supposed to mean. ## Architectural model -InQL is organized around three layers: +IncQL is organized around three layers: - **Prism internally** — the immutable planning and optimization engine over persistent authored plan state and derived optimized views - **Substrait at the boundary** — the normative emitted logical interchange contract @@ -35,7 +35,7 @@ This separation keeps internal planning concerns, portable interchange semantics ## Conceptual pipeline -InQL follows this shape: +IncQL follows this shape: ```text Incan models / model-derived schema @@ -87,7 +87,7 @@ For current package behavior, see [Dataset carriers (Reference)][dataset-ref] an ### Prism -Per [RFC 007][rfc-007], Prism is InQL’s internal logical planning and optimization engine. +Per [RFC 007][rfc-007], Prism is IncQL’s internal logical planning and optimization engine. Prism is responsible for: @@ -100,7 +100,7 @@ Prism is not the normative interchange format and not the execution engine. ### Substrait -Per [RFC 002][rfc-002], Apache Substrait is the normative logical interchange boundary for InQL. +Per [RFC 002][rfc-002], Apache Substrait is the normative logical interchange boundary for IncQL. That means: @@ -168,11 +168,11 @@ Normative behavior lives in the RFC series first. Current package behavior and u ## Repository vs compiler -The InQL repository and the Incan compiler have different responsibilities. +The IncQL repository and the Incan compiler have different responsibilities. ```text ┌─────────────────────────────────────────────────────────────────────────────┐ -│ InQL repo │ +│ IncQL repo │ ├─────────────────────────────────────────────────────────────────────────────┤ │ RFCs, package modules, tests, docs, architecture, conformance corpus │ │ Defines the relational package surface and its normative contracts │ @@ -184,7 +184,7 @@ The InQL repository and the Incan compiler have different responsibilities. │ Incan compiler │ ├─────────────────────────────────────────────────────────────────────────────┤ │ Parsing, typechecking, lowering, Rust emission, LSP, test runner, builds │ -│ Makes InQL package code executable and supports language surfaces │ +│ Makes IncQL package code executable and supports language surfaces │ └─────────────────────────────────────────────────────────────────────────────┘ ``` @@ -201,10 +201,10 @@ incan test tests In practice: -- `incan build --lib` parses, typechecks, lowers, and emits a Rust crate for the InQL library +- `incan build --lib` parses, typechecks, lowers, and emits a Rust crate for the IncQL library - `incan test tests` discovers and runs package tests under `tests/` -CI builds `incan` first, then runs the InQL package checks against that compiler. +CI builds `incan` first, then runs the IncQL package checks against that compiler. ## Reading order @@ -227,22 +227,22 @@ If you want the clearest current story, read in this order: | Execution context | [Reference][execution-ref] · [Explanation][execution-expl] | | Substrait integration | [Reference docs][substrait-read-root] · [RFC 002][rfc-002] | | Prism planning engine | [RFC 007][rfc-007] | -| InQL RFC index | [docs/rfcs/README.md][inql-rfcs] | +| IncQL RFC index | [docs/rfcs/README.md][incql-rfcs] | | Incan compiler architecture | [Incan architecture docs][incan-architecture] | -| Contributing | [CONTRIBUTING.md][inql-contributing] | +| Contributing | [CONTRIBUTING.md][incql-contributing] | [incan-architecture]: https://github.com/encero-systems/incan/blob/main/workspaces/docs-site/docs/contributing/explanation/architecture.md [docs-root]: README.md [language-root]: language/README.md -[inql-rfcs]: rfcs/README.md -[inql-contributing]: ../CONTRIBUTING.md +[incql-rfcs]: rfcs/README.md +[incql-contributing]: ../CONTRIBUTING.md [dataset-ref]: language/reference/dataset_carriers.md [dataset-expl]: language/explanation/dataset_carriers.md [execution-ref]: language/reference/execution_context.md [execution-expl]: language/explanation/execution_context.md [substrait-read-root]: language/reference/substrait/read_root_binding_contract.md [rfc-002]: rfcs/002_apache_substrait_integration.md -[rfc-004]: rfcs/004_inql_execution_context.md +[rfc-004]: rfcs/004_incql_execution_context.md [rfc-007]: rfcs/007_prism_planning_engine.md [rfc-008]: rfcs/008_optimizer_boundary_stats_cbo_aqe.md diff --git a/docs/contributing/writing_rfcs.md b/docs/contributing/writing_rfcs.md index 1358a84d..d668ca61 100644 --- a/docs/contributing/writing_rfcs.md +++ b/docs/contributing/writing_rfcs.md @@ -1,31 +1,31 @@ -# Writing InQL RFCs (how-to) +# Writing IncQL RFCs (how-to) -This guide is for contributors writing an RFC (design record) in the **InQL** repository. +This guide is for contributors writing an RFC (design record) in the **IncQL** repository. RFC means “Request for Comments”: a normative design document under [`docs/rfcs/`](../rfcs/README.md), numbered separately from [Incan language RFCs](https://github.com/encero-systems/incan/tree/main/workspaces/docs-site/docs/RFCs). !!! warning "Before you start" - Always check whether there is already an InQL RFC or an **RFC proposal** issue covering what you want to propose. Use the [RFC index](../rfcs/README.md) and repository search. + Always check whether there is already an IncQL RFC or an **RFC proposal** issue covering what you want to propose. Use the [RFC index](../rfcs/README.md) and repository search. Start here: - [RFC index](../rfcs/README.md) - [RFC file template](../rfcs/TEMPLATE.md) -- [RFC proposal issue](https://github.com/encero-systems/InQL/issues/new?template=rfc_proposal.yml) (optional) +- [RFC proposal issue](https://github.com/encero-systems/IncQL/issues/new?template=rfc_proposal.yml) (optional) ## When you should write an RFC -Write an InQL RFC when **behavior or contracts** of the relational layer change in a way authors or tools would rely on, for example: +Write an IncQL RFC when **behavior or contracts** of the relational layer change in a way authors or tools would rely on, for example: - Language surface: naming, query schema, resolution rules, or new relational syntax - `DataSet[T]` / carrier types, bounded vs unbounded rules, or public library API contracts - Substrait (or other) logical plan shape, extension policy, or binding boundaries - `query {}` grammar, typing, or lowering expectations - Execution context: session, I/O boundaries, or how plans meet runners -- Any change that must stay consistent across `query {}`, method chains, and future surfaces (see InQL RFC 000) +- Any change that must stay consistent across `query {}`, method chains, and future surfaces (see IncQL RFC 000) -Purely internal Incan compiler refactors with **no** InQL-visible meaning usually belong in the **Incan** repository, not as an InQL RFC. +Purely internal Incan compiler refactors with **no** IncQL-visible meaning usually belong in the **Incan** repository, not as an IncQL RFC. ## Workflow @@ -54,7 +54,7 @@ Purely internal Incan compiler refactors with **no** InQL-visible meaning usuall ## After acceptance -- **Implemented:** Set **Shipped in** to the first InQL **package** release that contains the change. When you adopt a `closed/implemented/` layout (optional), move the file there and keep the index accurate. +- **Implemented:** Set **Shipped in** to the first IncQL **package** release that contains the change. When you adopt a `closed/implemented/` layout (optional), move the file there and keep the index accurate. - **Deferred:** Update **Status** to `Deferred` and record why. ## Closed RFCs (optional layout) @@ -63,10 +63,10 @@ If you introduce subfolders under `docs/rfcs/`, a common pattern is: - `docs/rfcs/` — active RFCs (Draft, Planned, In Progress, …) - `docs/rfcs/closed/implemented/` — shipped -- `docs/rfcs/closed/superseded/` — replaced by a newer InQL RFC +- `docs/rfcs/closed/superseded/` — replaced by a newer IncQL RFC - `docs/rfcs/closed/rejected/` — withdrawn or rejected -When superseding or rejecting, update the status line (for example `Superseded by InQL RFC NNN`) and move the file if you use `closed/`. +When superseding or rejecting, update the status line (for example `Superseded by IncQL RFC NNN`) and move the file if you use `closed/`. ## Tips for a good RFC @@ -78,7 +78,7 @@ When superseding or rejecting, update the status line (for example `Superseded b ## Compiler and tooling work -InQL RFCs often imply changes in the **Incan** compiler (parse, check, lower). When you implement there, follow that repository’s contributor guide, **AGENTS.md**, and CI gates. The InQL RFC remains the spec; the Incan repo carries the toolchain implementation. +IncQL RFCs often imply changes in the **Incan** compiler (parse, check, lower). When you implement there, follow that repository’s contributor guide, **AGENTS.md**, and CI gates. The IncQL RFC remains the spec; the Incan repo carries the toolchain implementation. ## Further reading (Incan documentation norms) diff --git a/docs/language/README.md b/docs/language/README.md index c865d582..6a0bd6a0 100644 --- a/docs/language/README.md +++ b/docs/language/README.md @@ -1,6 +1,6 @@ -# InQL language docs +# IncQL language docs -This section documents the current InQL package surface. +This section documents the current IncQL package surface. - Use [reference/][reference] for API shape, signatures, and current behavior contracts. - Use [how-to/][how-to] for concrete task workflows. diff --git a/docs/language/explanation/dataset_carriers.md b/docs/language/explanation/dataset_carriers.md index b36db92a..df2b07f8 100644 --- a/docs/language/explanation/dataset_carriers.md +++ b/docs/language/explanation/dataset_carriers.md @@ -1,6 +1,6 @@ # Dataset carriers (Explanation) -This page explains how to think about and use InQL's dataset carriers. It is intentionally conceptual. Exact method and builder signatures live in the reference pages. +This page explains how to think about and use IncQL's dataset carriers. It is intentionally conceptual. Exact method and builder signatures live in the reference pages. ## Why dataset carriers? @@ -32,7 +32,7 @@ This enables **static capability gating**: operations that require unbounded sta Use `DataFrame[T]` when you have data in hand and want to inspect or manipulate it directly: ```incan -from pub::inql import DataFrame +from pub::incql import DataFrame from models import Order def inspect_orders(orders: DataFrame[Order]) -> None: @@ -55,8 +55,8 @@ Collected `DataFrame[T]` values currently expose structured materialization meta Use `LazyFrame[T]` when you want to compose operations before execution: ```incan -from pub::inql import LazyFrame -from pub::inql.functions import col, gt, lit +from pub::incql import LazyFrame +from pub::incql.functions import col, gt, lit from models import Order def high_value_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -68,8 +68,8 @@ def high_value_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: Use `DataStream[T]` for streaming/unbounded data: ```incan -from pub::inql import DataStream -from pub::inql.functions import col, eq +from pub::incql import DataStream +from pub::incql.functions import col, eq from models import Event def important_events(events: DataStream[Event]) -> DataStream[Event]: @@ -83,7 +83,7 @@ def important_events(events: DataStream[Event]) -> DataStream[Event]: The trait hierarchy gives you three levels of specificity: ```incan -from pub::inql import DataSet, BoundedDataSet, UnboundedDataSet +from pub::incql import DataSet, BoundedDataSet, UnboundedDataSet from models import Order, Event # Accepts any carrier — generic utilities @@ -102,7 +102,7 @@ def write_to_kafka(events: UnboundedDataSet[Event]) -> None: And two levels of concrete-type specificity: ```incan -from pub::inql import DataFrame, LazyFrame, DataStream +from pub::incql import DataFrame, LazyFrame, DataStream from models import Order, Event # Materialized data in hand @@ -128,13 +128,13 @@ Today there are three concrete builder families: ### Aggregate helpers -`.agg(...)` uses **imported** symbols from `pub::inql.functions` through explicit builders such as `col(...)`, `sum(...)`, and `count()`. +`.agg(...)` uses **imported** symbols from `pub::incql.functions` through explicit builders such as `col(...)`, `sum(...)`, and `count()`. Concrete builder example: ```incan -from pub::inql.functions import col, count, sum -from pub::inql import LazyFrame +from pub::incql.functions import col, count, sum +from pub::incql import LazyFrame from models import Order def orders_by_customer(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -148,8 +148,8 @@ That is the current semantic target for future sugar such as `.customer_id` or ` Computed columns now have one real entrypoint: `with_column(name, expr)`. ```incan -from pub::inql.functions import add, col, lit, mul -from pub::inql import LazyFrame +from pub::incql.functions import add, col, lit, mul +from pub::incql import LazyFrame from models import Order def enrich_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -174,11 +174,11 @@ The most useful way to read the current surface is to separate: ### Concrete method-chain example -This is real current InQL, not aspirational pseudocode: +This is real current IncQL, not aspirational pseudocode: ```incan -from pub::inql.functions import add, col, count, lit, sum -from pub::inql import LazyFrame +from pub::incql.functions import add, col, count, lit, sum +from pub::incql import LazyFrame from models import Order def summarize_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: diff --git a/docs/language/explanation/execution_context.md b/docs/language/explanation/execution_context.md index de3735ea..f45f35e1 100644 --- a/docs/language/explanation/execution_context.md +++ b/docs/language/explanation/execution_context.md @@ -1,6 +1,6 @@ # Execution context (Explanation) -This page explains how to think about InQL's execution model as it works today. +This page explains how to think about IncQL's execution model as it works today. ## The mental model @@ -72,7 +72,7 @@ This keeps materialization convenient while leaving sink ownership explicit at t ## Runtime evidence is separate from plan evidence -Plan inspection explains the relational work InQL has authored. Execution observations explain a concrete runtime attempt to run that work through a Session and backend adapter. +Plan inspection explains the relational work IncQL has authored. Execution observations explain a concrete runtime attempt to run that work through a Session and backend adapter. That split matters because the same plan can be attempted more than once, with different backends, bindings, diagnostics, timings, or trace IDs. The plan target remains the semantic anchor. The execution attempt target records what happened in one runtime lifecycle event. @@ -90,13 +90,13 @@ Adapter coverage answers a different question from execution success. Execution Coverage can come from two places. Tools can pass explicit `AdapterRequirement` records to `session.check_coverage(...)` when the requirement comes from policy, governance, or another caller-owned contract. For requirements that are visible in local plan evidence, `inspect_plan(...)` records inferred requirements and `session.check_inspection_coverage(...)` or `session.check_plan_coverage(...)` evaluates them against the selected adapter. -Inferred requirements are intentionally evidence-backed. InQL infers concrete plan needs such as row filters, ordered execution, extension functions, variant semantics, baseline null semantics, and lineage-preservation evidence; it does not fabricate policy requirements such as masking, audit emission, region binding, or cryptographic proofs when no inspected evidence asks for them. Unknown coverage is therefore not a soft success; it means InQL does not have evidence that the adapter enforces that capability. +Inferred requirements are intentionally evidence-backed. IncQL infers concrete plan needs such as row filters, ordered execution, extension functions, variant semantics, baseline null semantics, and lineage-preservation evidence; it does not fabricate policy requirements such as masking, audit emission, region binding, or cryptographic proofs when no inspected evidence asks for them. Unknown coverage is therefore not a soft success; it means IncQL does not have evidence that the adapter enforces that capability. ## Typical flow ```incan -from pub::inql import LazyFrame, Session -from pub::inql.functions import col, gt, lit, mul +from pub::incql import LazyFrame, Session +from pub::incql.functions import col, gt, lit, mul from models import Order session = Session.default() diff --git a/docs/language/how-to/README.md b/docs/language/how-to/README.md index 4d8d87a7..a61dc184 100644 --- a/docs/language/how-to/README.md +++ b/docs/language/how-to/README.md @@ -1,6 +1,6 @@ -# InQL language how-to guides +# IncQL language how-to guides -How-to guides show concrete task workflows for the current InQL package surface. They complement the reference docs, which define API shape and behavior contracts. +How-to guides show concrete task workflows for the current IncQL package surface. They complement the reference docs, which define API shape and behavior contracts. - [Add window columns][window-columns] - [Build typed HyperLogLog sketches][typed-hll-sketches] diff --git a/docs/language/how-to/approximate_metrics.md b/docs/language/how-to/approximate_metrics.md index 740c97d4..6bf18a1d 100644 --- a/docs/language/how-to/approximate_metrics.md +++ b/docs/language/how-to/approximate_metrics.md @@ -2,14 +2,14 @@ This how-to shows how to opt in to approximate aggregate helpers when exact results are not required. -Use approximate helpers explicitly. InQL does not silently replace exact aggregates with approximate implementations because a backend can do so. +Use approximate helpers explicitly. IncQL does not silently replace exact aggregates with approximate implementations because a backend can do so. ## Estimate distinct counts and percentiles Group the relation normally, then use approximate aggregate measures inside `agg(...)`. ```incan -from pub::inql.functions import approx_count_distinct, approx_percentile, col +from pub::incql.functions import approx_count_distinct, approx_percentile, col summary = ( events diff --git a/docs/language/how-to/dataset_transformations.md b/docs/language/how-to/dataset_transformations.md index 99b2b598..f9aef955 100644 --- a/docs/language/how-to/dataset_transformations.md +++ b/docs/language/how-to/dataset_transformations.md @@ -7,8 +7,8 @@ This how-to shows how to combine common carrier methods while keeping work defer Use `with_column(...)` to append a new computed column or replace an existing column by name. ```incan -from pub::inql import LazyFrame -from pub::inql.functions import add, col, mul +from pub::incql import LazyFrame +from pub::incql.functions import add, col, mul from models import Order def enrich(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -24,8 +24,8 @@ def enrich(orders: LazyFrame[Order]) -> LazyFrame[Order]: Use scalar helpers for row predicates and aggregate helpers for grouped measures. ```incan -from pub::inql import LazyFrame -from pub::inql.functions import avg, col, count, eq, sum +from pub::incql import LazyFrame +from pub::incql.functions import avg, col, count, eq, sum from models import Order def paid_spend_by_customer(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -46,7 +46,7 @@ def paid_spend_by_customer(orders: LazyFrame[Order]) -> LazyFrame[Order]: Use ordering helpers inside `order_by(...)`, then cap rows with `limit(...)`. ```incan -from pub::inql.functions import col, desc +from pub::incql.functions import col, desc top_orders = ( orders diff --git a/docs/language/how-to/execution_observations.md b/docs/language/how-to/execution_observations.md index bb7f2094..db3a2fc9 100644 --- a/docs/language/how-to/execution_observations.md +++ b/docs/language/how-to/execution_observations.md @@ -9,7 +9,7 @@ Use the observed Session methods when you need an auditable execution attempt re Use `collect_observed(...)` when you need materialized data and execution evidence from the same attempt. ```incan -from pub::inql import ExecutionObservationStatus, LazyFrame, Session +from pub::incql import ExecutionObservationStatus, LazyFrame, Session from models import Order session = Session.default() @@ -48,7 +48,7 @@ match observed.error: Use `write_observed(...)` when the write itself is the operation you want to audit. ```incan -from pub::inql import csv_sink +from pub::incql import csv_sink write_attempt = session.write_observed(orders, csv_sink("target/orders.csv")) @@ -61,11 +61,11 @@ The write result has no `data` field. The output artifact is the sink side effec ## Check inferred adapter requirements -Use `check_plan_coverage(...)` when you want InQL to inspect a lazy plan and evaluate the adapter requirements that are visible in that plan evidence. +Use `check_plan_coverage(...)` when you want IncQL to inspect a lazy plan and evaluate the adapter requirements that are visible in that plan evidence. ```incan -from pub::inql import AdapterCoverageState -from pub::inql.functions import col, desc, eq +from pub::incql import AdapterCoverageState +from pub::incql.functions import col, desc, eq review = orders .filter(eq(col("status"), "paid")) @@ -86,7 +86,7 @@ for record in coverage: Use `check_coverage(...)` when the requirement comes from a policy, workflow, or review step rather than directly from the inspected plan shape. Build the requirements that matter, then ask the selected adapter for coverage records. ```incan -from pub::inql import ( +from pub::incql import ( AdapterCoverageState, AdapterRequirement, AdapterRequirementCapability, @@ -112,7 +112,7 @@ match coverage[0].state: AdapterCoverageState.Unknown => println(coverage[0].diagnostics[0].message) ``` -Treat `Unknown` as non-enforcing. It means InQL has not classified that adapter capability; it does not mean the adapter has proven support. +Treat `Unknown` as non-enforcing. It means IncQL has not classified that adapter capability; it does not mean the adapter has proven support. ## Choose the right observed method diff --git a/docs/language/how-to/generator_rows.md b/docs/language/how-to/generator_rows.md index d41bb078..c7348512 100644 --- a/docs/language/how-to/generator_rows.md +++ b/docs/language/how-to/generator_rows.md @@ -9,8 +9,8 @@ Generators return `GeneratorApplication` values. Apply them through `generate(.. Use `explode(...)` when each array element should become a generated row. ```incan -from pub::inql import LazyFrame -from pub::inql.functions import col, explode +from pub::incql import LazyFrame +from pub::incql.functions import col, explode from models import Order def order_lines(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -22,8 +22,8 @@ def order_lines(orders: LazyFrame[Order]) -> LazyFrame[Order]: Use `inline(...)` when the generated rows should expose one output column per struct field. ```incan -from pub::inql import LazyFrame -from pub::inql.functions import array, inline, lit, named_struct +from pub::incql import LazyFrame +from pub::incql.functions import array, inline, lit, named_struct from models import Order def fixed_items(orders: LazyFrame[Order]) -> LazyFrame[Order]: diff --git a/docs/language/how-to/governed_evidence.md b/docs/language/how-to/governed_evidence.md index d1eb5a5b..d4bd5b9d 100644 --- a/docs/language/how-to/governed_evidence.md +++ b/docs/language/how-to/governed_evidence.md @@ -1,13 +1,13 @@ # Inspect governed evidence -Use governed attributes and policy checkpoints when you need local evidence about sensitive, classified, reviewed, or policy-relevant relational targets without making InQL responsible for your organization’s policy engine. +Use governed attributes and policy checkpoints when you need local evidence about sensitive, classified, reviewed, or policy-relevant relational targets without making IncQL responsible for your organization’s policy engine. ## Inspect inferred governed attributes `inspect_plan(...)` carries governed evidence next to lineage, metadata attachments, adapter requirements, and unsupported-evidence markers. ```incan -from pub::inql import inspect_plan +from pub::incql import inspect_plan inspection = inspect_plan(summary) @@ -23,7 +23,7 @@ Inspection infers field schema attributes when local plan evidence includes a fi Use `governed_attribute(...)` when another local step already knows a governed fact and wants to attach it to a semantic target. ```incan -from pub::inql import ( +from pub::incql import ( GovernedAttributeConfidence, GovernedAttributeScope, GovernedAttributeSource, @@ -55,7 +55,7 @@ This only creates evidence. It does not mask the field, filter rows, block execu Use `policy_checkpoint(...)` to record a policy result or observation at a known boundary. ```incan -from pub::inql import ( +from pub::incql import ( MetadataVisibility, PolicyCheckpointAction, PolicyCheckpointKind, @@ -73,7 +73,7 @@ checkpoint = policy_checkpoint( ) ``` -The checkpoint says that a warning decision was recorded. A CI job, notebook, orchestration layer, or governance tool can decide what to do with that warning. InQL keeps the decision evidence explicit and local. +The checkpoint says that a warning decision was recorded. A CI job, notebook, orchestration layer, or governance tool can decide what to do with that warning. IncQL keeps the decision evidence explicit and local. ## Read policy observation from inspection diff --git a/docs/language/how-to/inspect_plan_lineage.md b/docs/language/how-to/inspect_plan_lineage.md index 4037e090..cc7e9206 100644 --- a/docs/language/how-to/inspect_plan_lineage.md +++ b/docs/language/how-to/inspect_plan_lineage.md @@ -7,8 +7,8 @@ Use `inspect_plan(...)` when you need the full inspection record. Use `inspect_l ## Build a lazy plan ```incan -from pub::inql import LazyFrame -from pub::inql.functions import col, eq, str_lit, sum +from pub::incql import LazyFrame +from pub::incql.functions import col, eq, str_lit, sum from models import Order def paid_spend_summary(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -23,7 +23,7 @@ def paid_spend_summary(orders: LazyFrame[Order]) -> LazyFrame[Order]: ## Inspect the plan ```incan -from pub::inql import inspect_plan +from pub::incql import inspect_plan summary = paid_spend_summary(orders) inspection = inspect_plan(summary) @@ -37,7 +37,7 @@ println(inspection.output_fields[0].name) ## Read lineage directly ```incan -from pub::inql import inspect_lineage +from pub::incql import inspect_lineage lineage = inspect_lineage(summary) diff --git a/docs/language/how-to/nested_row_values.md b/docs/language/how-to/nested_row_values.md index 94e49d24..6b31d456 100644 --- a/docs/language/how-to/nested_row_values.md +++ b/docs/language/how-to/nested_row_values.md @@ -9,7 +9,7 @@ Use nested scalar helpers when each input row should remain one output row. Use Build arrays with `array(...)`, then inspect them with row-level helpers such as `cardinality(...)`, `array_contains(...)`, and `element_at(...)`. ```incan -from pub::inql.functions import array, array_contains, cardinality, col, element_at, lit +from pub::incql.functions import array, array_contains, cardinality, col, element_at, lit projected = ( events diff --git a/docs/language/how-to/normalize_semistructured_fields.md b/docs/language/how-to/normalize_semistructured_fields.md index 0d8596aa..721e0364 100644 --- a/docs/language/how-to/normalize_semistructured_fields.md +++ b/docs/language/how-to/normalize_semistructured_fields.md @@ -9,7 +9,7 @@ Use format helpers when the payload should stay a scalar expression in the curre Hash identifiers, extract URL and JSON fields, and validate schema-bearing payloads with model type parameters. ```incan -from pub::inql.functions import col, from_csv, from_json, get_json_object, parse_url, sha2, to_json +from pub::incql.functions import col, from_csv, from_json, get_json_object, parse_url, sha2, to_json model EventPayload: type_ as "type": str diff --git a/docs/language/how-to/quality_observations.md b/docs/language/how-to/quality_observations.md index 8004d530..4ad10cc6 100644 --- a/docs/language/how-to/quality_observations.md +++ b/docs/language/how-to/quality_observations.md @@ -7,7 +7,7 @@ This how-to shows how to declare quality assertions and evaluate them through a Use assertion helpers to describe the checks you want to observe. The first quality surface covers relation row counts, field null rates, field uniqueness, per-group row counts, and explicit cross-relation row-count equality. ```incan -from pub::inql import QualityAssertionSeverity, col, group_row_count, null_rate, row_count, unique +from pub::incql import QualityAssertionSeverity, col, group_row_count, null_rate, row_count, unique checks = [ row_count(min_count=Some(1), max_count=Some(1000000)).require(), @@ -27,7 +27,7 @@ The helper options are deliberately small: Use `.require()`, `.quarantine()`, `.with_mode(...)`, or `.with_severity(...)` when a caller needs to record intended handling. Session observation still reports evidence rather than enforcing policy. For example, a failed required check returns a `Failed` observation; it does not throw merely because the assertion mode is `Require`. -You can also declare the same checks with the `quality` vocab surface after importing `pub::inql`. A `quality` block returns a `list[QualityAssertion]`; it does not execute checks by itself. +You can also declare the same checks with the `quality` vocab surface after importing `pub::incql`. A `quality` block returns a `list[QualityAssertion]`; it does not execute checks by itself. ```incan max_customer_null_rate = 0.0 @@ -47,7 +47,7 @@ Use leading-dot field references inside `quality` blocks when the assertion shou Quality syntax composes with query syntax. Use `query { ... }` to shape the relation you want to check, then pass that relation and the `quality { ... }` assertions to `session.observe_quality(...)`. ```incan -from pub::inql import LazyFrame, Session +from pub::incql import LazyFrame, Session from models import Order session = Session.default() @@ -73,14 +73,14 @@ checks = quality: observations = session.observe_quality(paid_orders, checks) ``` -Inside `query { ... }`, leading-dot field references such as `.status` and `.order_id` use query-block resolution. Inside `quality { ... }` or `quality:`, leading-dot field references use quality assertion resolution and lower to `col("...")`. Outside those vocab blocks, use ordinary InQL expressions such as `col("customer_id")`. +Inside `query { ... }`, leading-dot field references such as `.status` and `.order_id` use query-block resolution. Inside `quality { ... }` or `quality:`, leading-dot field references use quality assertion resolution and lower to `col("...")`. Outside those vocab blocks, use ordinary IncQL expressions such as `col("customer_id")`. ## Read the output `session.observe_quality(...)` returns one `QualityObservation` per assertion. The important fields for most callers are the assertion name, status, metrics, diagnostics, and execution observation IDs. ```incan -from pub::inql import QualityObservationStatus +from pub::incql import QualityObservationStatus for observation in observations: println(f"{observation.assertion.name}: {observation.status.value()}") @@ -99,7 +99,7 @@ for observation in observations: QualityObservationStatus.Unsupported => println(observation.diagnostics[0].message) ``` -A compact rendering of a mixed result set might look like this. This is an example presentation, not a fixed InQL CLI output format. +A compact rendering of a mixed result set might look like this. This is an example presentation, not a fixed IncQL CLI output format. ```text row_count: passed @@ -130,7 +130,7 @@ group_row_count:region: passed Use `session.observe_quality_pair(...)` for cross-relation assertions. The first release includes row-count equality. ```incan -from pub::inql import cross_relation_row_count_equal +from pub::incql import cross_relation_row_count_equal from models import SourceOrder, TargetOrder source_orders: LazyFrame[SourceOrder] = session.read_csv("source_orders", "source_orders.csv")? diff --git a/docs/language/how-to/typed_hll_sketches.md b/docs/language/how-to/typed_hll_sketches.md index 8508e924..858123c1 100644 --- a/docs/language/how-to/typed_hll_sketches.md +++ b/docs/language/how-to/typed_hll_sketches.md @@ -9,7 +9,7 @@ Use sketch helpers when approximate state itself needs to flow through a plan. U Aggregate source values into typed sketch state with `hll_sketch(...)`. ```incan -from pub::inql.functions import col, hll_sketch +from pub::incql.functions import col, hll_sketch daily = events.group_by([col("event_date")]).agg([ hll_sketch(col("user_id"), precision=14), @@ -25,7 +25,7 @@ literal_seed = events.group_by([col("event_date")]).agg([ Reference sketch columns with matching logical type metadata, then merge and estimate them. ```incan -from pub::inql.sketches import hll_estimate, hll_merge, hll_type, sketch_col +from pub::incql.sketches import hll_estimate, hll_merge, hll_type, sketch_col monthly = daily.group_by([col("month")]).agg([ hll_merge(sketch_col("hll_sketch_user_id", hll_type(precision=14))), diff --git a/docs/language/how-to/variant_payloads.md b/docs/language/how-to/variant_payloads.md index c1ed2c3a..9b882643 100644 --- a/docs/language/how-to/variant_payloads.md +++ b/docs/language/how-to/variant_payloads.md @@ -9,7 +9,7 @@ Use variant helpers when the plan needs kind-aware semi-structured inspection. U Parse once, then apply `typeof(...)`, `variant_get(...)`, and variant predicates to the typed value. ```incan -from pub::inql.functions import col, is_array, is_null_value, parse_variant_json, typeof, variant_get +from pub::incql.functions import col, is_array, is_null_value, parse_variant_json, typeof, variant_get payload = parse_variant_json(col("payload")) literal_payload = parse_variant_json("{\"status\":\"paid\"}") diff --git a/docs/language/how-to/window_columns.md b/docs/language/how-to/window_columns.md index 8a82983a..0b737152 100644 --- a/docs/language/how-to/window_columns.md +++ b/docs/language/how-to/window_columns.md @@ -9,8 +9,8 @@ Window helpers produce one output value per input row while reading related rows Build a window spec, call `.over(spec)` on each window helper, and attach the resulting applications as named columns. ```incan -from pub::inql import LazyFrame -from pub::inql.functions import col, current_row, desc, lag, rank, sum, unbounded_preceding, window +from pub::incql import LazyFrame +from pub::incql.functions import col, current_row, desc, lag, rank, sum, unbounded_preceding, window from models import Order def ranked_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: diff --git a/docs/language/reference/execution_context.md b/docs/language/reference/execution_context.md index ce32fb6e..57c63085 100644 --- a/docs/language/reference/execution_context.md +++ b/docs/language/reference/execution_context.md @@ -1,6 +1,6 @@ # Execution context (Reference) -This page documents the public execution surface in the InQL package. Normative design intent lives in [RFC 004][rfc-004]. +This page documents the public execution surface in the IncQL package. Normative design intent lives in [RFC 004][rfc-004]. ## Core types @@ -162,7 +162,7 @@ Plan inference is evidence-backed rather than policy-complete. The current imple | `AdapterCoverageState` | `Covered`, `PartiallyCovered`, `Uncovered`, `Unknown` | | `AdapterRequirementCapability` | `ExtensionFunction`, `VariantSemantics`, `DecimalSemantics`, `NullSemantics`, `LineagePreservation`, `AuditEmission`, `RowFilter`, `ColumnMask`, `AggregateThreshold`, `RegionBinding`, `OrderedExecution`, `SnapshotCapture`, `CanonicalDigest`, `CrossRelationReconciliation`, `IncrementalWatermark`, `VerificationEventStream`, `WaiverRecording`, `CryptographicQueryProof` | -Coverage states are conservative. `Covered` means the selected adapter is known to cover that requirement family. `PartiallyCovered` means support depends on the concrete function, plan shape, or restriction. `Uncovered` means the selected adapter is known not to provide that guarantee. `Unknown` means InQL has not classified coverage; consumers must not treat it as enforced behavior. +Coverage states are conservative. `Covered` means the selected adapter is known to cover that requirement family. `PartiallyCovered` means support depends on the concrete function, plan shape, or restriction. `Uncovered` means the selected adapter is known not to provide that guarantee. `Unknown` means IncQL has not classified coverage; consumers must not treat it as enforced behavior. ### Current DataFusion coverage classification @@ -225,7 +225,7 @@ DataFusion is the implemented execution backend. `Session` stores a backend kind - For adapter requirement and coverage design, see [RFC 033][rfc-033] - For quality assertion and observation design, see [RFC 034][rfc-034] -[rfc-004]: ../../rfcs/004_inql_execution_context.md +[rfc-004]: ../../rfcs/004_incql_execution_context.md [rfc-032]: ../../rfcs/032_execution_observations.md [rfc-033]: ../../rfcs/033_adapter_requirements_coverage.md [rfc-034]: ../../rfcs/034_quality_assertions_observations.md diff --git a/docs/language/reference/functions/approximate.md b/docs/language/reference/functions/approximate.md index 3a33bb8b..69b363d2 100644 --- a/docs/language/reference/functions/approximate.md +++ b/docs/language/reference/functions/approximate.md @@ -1,6 +1,6 @@ # Approximate Functions (Reference) -Approximate helpers are explicit opt-in functions. InQL does not silently replace exact aggregates with approximate execution because a backend can do so. +Approximate helpers are explicit opt-in functions. IncQL does not silently replace exact aggregates with approximate execution because a backend can do so. The portable RFC 023 aggregate surface is: @@ -9,11 +9,11 @@ The portable RFC 023 aggregate surface is: | `approx_count_distinct(expr)` | Estimate the number of distinct non-null values produced by one expression. | | `approx_percentile(expr, percentile, accuracy=10000)` | Estimate one percentile over numeric non-null values. | -`approx_count_distinct` is registered as an approximate aggregate with HyperLogLog-family metadata. The portable author contract is an approximate non-null distinct-count estimate. It does not expose a user-tunable relative-error parameter because the registered InQL Substrait extension mapping for this function is unary. Backend adapters must keep this approximation visible in capability/error handling rather than redefining exact `count_distinct` semantics. +`approx_count_distinct` is registered as an approximate aggregate with HyperLogLog-family metadata. The portable author contract is an approximate non-null distinct-count estimate. It does not expose a user-tunable relative-error parameter because the registered IncQL Substrait extension mapping for this function is unary. Backend adapters must keep this approximation visible in capability/error handling rather than redefining exact `count_distinct` semantics. `approx_percentile` is registered as an approximate aggregate with t-digest-family metadata. `percentile` must be between `0.0` and `1.0` inclusive. `accuracy` must be positive and is carried as an explicit aggregate argument so backend capability handling can accept, emulate, or reject the requested approximation instead of silently changing semantics. Generated aggregate output names include the percentile and accuracy arguments. -Both helpers lower through registered InQL Substrait aggregate extension names. The DataFusion adapter maps `approx_count_distinct` to DataFusion's `approx_distinct` implementation and maps `approx_percentile` to `approx_percentile_cont` at the backend boundary. +Both helpers lower through registered IncQL Substrait aggregate extension names. The DataFusion adapter maps `approx_count_distinct` to DataFusion's `approx_distinct` implementation and maps `approx_percentile` to `approx_percentile_cont` at the backend boundary. Sketch-state construction, merge, estimate, serialization, and deserialization are implemented by [Sketch functions](sketches.md). Those helpers use typed sketch logical values with sketch family, value domain, merge compatibility, and serialized format identity. Exposing sketch state as strings or binary payloads would violate the RFC 023 type-safety requirement. diff --git a/docs/language/reference/functions/index.md b/docs/language/reference/functions/index.md index 9b076bab..95adc523 100644 --- a/docs/language/reference/functions/index.md +++ b/docs/language/reference/functions/index.md @@ -1,6 +1,6 @@ # Functions (Reference) -This section is the landing page for InQL's registered function families. +This section is the landing page for IncQL's registered function families. Today the concrete shipped surfaces are documented here: @@ -17,7 +17,7 @@ Today the concrete shipped surfaces are documented here: The canonical scalar literal helper is `lit(...)`. Typed literal helpers construct the same scalar-expression representation. Public helper signatures use literal-or-column aliases such as `IntValueOrColumn`, `StrValueOrColumn`, `FloatValueOrColumn`, `BoolValueOrColumn`, and `ScalarValueOrColumn` when a parameter naturally accepts either a primitive literal or an existing scalar expression. These aliases normalize literals into the same scalar expression nodes, so typed helpers can be written as `make_date(2026, 5, 30)` or `substring(col("sku"), 1, 3)` without wrapping constants in `lit(...)`. Referenced columns are validated during Prism/Substrait lowering when the current query schema has concrete primitive kind facts. -The current registry-backed helper surface covers references, literals, casts, operators, predicates, conditionals, math, ordering, aggregates, generators, nested data, windows, format helpers, sketch helpers, and variant helpers. Each runtime entry exposes a stable function reference such as `inql.functions.col`, namespace, canonical name, typed lifecycle metadata (`since`, versioned changes, and optional deprecation), function policy category, function class, null behavior, alias policy, aggregate modifier policy, and Substrait mapping metadata. Checked public helpers provide the callable signature, source-level argument and return types, and, by default, the canonical name; metadata may override the canonical name only for source spelling constraints such as the reserved-word `mod` case. +The current registry-backed helper surface covers references, literals, casts, operators, predicates, conditionals, math, ordering, aggregates, generators, nested data, windows, format helpers, sketch helpers, and variant helpers. Each runtime entry exposes a stable function reference such as `incql.functions.col`, namespace, canonical name, typed lifecycle metadata (`since`, versioned changes, and optional deprecation), function policy category, function class, null behavior, alias policy, aggregate modifier policy, and Substrait mapping metadata. Checked public helpers provide the callable signature, source-level argument and return types, and, by default, the canonical name; metadata may override the canonical name only for source spelling constraints such as the reserved-word `mod` case. The registry is the source for non-derivable machine facts such as policy, lifecycle, aliases, aggregate modifiers, and Substrait mappings. Public helper declarations remain the source for checked callable signatures, argument names, and typed expression return shape. Docstrings remain human-facing explanation, examples, and parameter intent. The `registry-metadata` check validates the checked API metadata projections produced from public facade aliases, registry decorators, and decorated callable signatures. Runtime registry entries are lazy and process-local: they support helper execution and lowering for loaded helpers, while the complete public catalog comes from checked metadata. This matters for generated docs, diagnostics, Prism lowering, and backend capability checks as the catalog grows. diff --git a/docs/language/reference/functions/nested.md b/docs/language/reference/functions/nested.md index d9394ffb..00beef39 100644 --- a/docs/language/reference/functions/nested.md +++ b/docs/language/reference/functions/nested.md @@ -40,7 +40,7 @@ Generator or table-valued operations such as row-expanding `explode(...)` are se ## Semantics - Array indexing is one-based for `element_at(...)`, `array_position(...)`, and `array_slice(...)`. -- `element_at(...)` currently maps to the portable array-element adapter path. Out-of-range behavior follows the current backend adapter's recoverable result until InQL has a richer static/runtime error-policy split for strict versus try-style element access. +- `element_at(...)` currently maps to the portable array-element adapter path. Out-of-range behavior follows the current backend adapter's recoverable result until IncQL has a richer static/runtime error-policy split for strict versus try-style element access. - `array_flatten(...)` is intentionally named to stay distinct from the relation-shaping generator `flatten(...)`. - Grouping or ordering by nested values is not documented as portable until equality and ordering semantics for arrays, maps, and structs are specified. - For task-oriented usage, see [Work with nested row values](../../how-to/nested_row_values.md). diff --git a/docs/language/reference/functions/sketches.md b/docs/language/reference/functions/sketches.md index 55d2c081..e609fef9 100644 --- a/docs/language/reference/functions/sketches.md +++ b/docs/language/reference/functions/sketches.md @@ -17,6 +17,6 @@ Sketch compatibility is structural. HyperLogLog sketches can merge only when fam The public helper surface follows the typed value-or-column conventions used by the rest of the function catalog: `hll_sketch(...)` accepts primitive values or scalar expressions, while `hll_deserialize(...)` accepts string payload values or scalar expressions. -RFC 025 helpers lower through InQL-owned Substrait extension mappings and carry sketch metadata in function options. The DataFusion adapter reports a backend planning diagnostic for typed sketch execution because it has no sketch runtime implementation. That rejection is an adapter capability boundary; the InQL plan remains typed and backend-neutral. +RFC 025 helpers lower through IncQL-owned Substrait extension mappings and carry sketch metadata in function options. The DataFusion adapter reports a backend planning diagnostic for typed sketch execution because it has no sketch runtime implementation. That rejection is an adapter capability boundary; the IncQL plan remains typed and backend-neutral. For task-oriented usage, see [Build typed HyperLogLog sketches](../../how-to/typed_hll_sketches.md). diff --git a/docs/language/reference/functions/variants.md b/docs/language/reference/functions/variants.md index d88525b4..890bc17e 100644 --- a/docs/language/reference/functions/variants.md +++ b/docs/language/reference/functions/variants.md @@ -22,6 +22,6 @@ Variant helpers model semi-structured payloads as typed logical values, not as o `typeof(...)` accepts a `VariantExpr` value and returns a `StringColumnExpr`. Variant predicates accept `VariantExpr` values and return `BoolColumnExpr` values. They do not parse strings directly. Parse helpers accept `StrValueOrColumn` inputs; that keeps parsing, variant inspection, and RFC 022 JSON text helpers separate without forcing authors to wrap literal payloads in `lit(...)`. -RFC 026 helpers lower through InQL-owned Substrait extension mappings and carry variant metadata in function options. The DataFusion adapter currently reports a backend planning diagnostic for typed variant execution because it has no variant runtime implementation. That rejection is an adapter capability boundary; the InQL plan remains typed and backend-neutral. +RFC 026 helpers lower through IncQL-owned Substrait extension mappings and carry variant metadata in function options. The DataFusion adapter currently reports a backend planning diagnostic for typed variant execution because it has no variant runtime implementation. That rejection is an adapter capability boundary; the IncQL plan remains typed and backend-neutral. For task-oriented usage, see [Inspect typed variant payloads](../../how-to/variant_payloads.md). diff --git a/docs/language/reference/governance.md b/docs/language/reference/governance.md index 719e9515..9aee4558 100644 --- a/docs/language/reference/governance.md +++ b/docs/language/reference/governance.md @@ -1,20 +1,20 @@ # Governed attributes and policy checkpoints (Reference) -This page documents the current governed-evidence surface in the InQL package. Normative design intent lives in [RFC 035][rfc-035]. +This page documents the current governed-evidence surface in the IncQL package. Normative design intent lives in [RFC 035][rfc-035]. -Governed attributes are typed evidence records attached to semantic targets. They can describe facts such as classification, origin, purpose, jurisdiction, masking state, or schema-derived field facts. A governed attribute records where the fact came from, how confident InQL or a caller is in it, whether it is asserted or inferred, which authority supplied or reviewed it, and which evidence references support it. +Governed attributes are typed evidence records attached to semantic targets. They can describe facts such as classification, origin, purpose, jurisdiction, masking state, or schema-derived field facts. A governed attribute records where the fact came from, how confident IncQL or a caller is in it, whether it is asserted or inferred, which authority supplied or reviewed it, and which evidence references support it. Policy checkpoints are decision records attached to semantic targets at authoring, planning, binding, or execution boundaries. They record that a policy result was observed, warned, denied, required approval, requested masking, or required another check. They do not define a policy language and they do not enforce behavior by themselves. ## Entry points ```incan -from pub::inql import governed_attribute, policy_checkpoint +from pub::incql import governed_attribute, policy_checkpoint ``` | Helper | Signature | Purpose | | ------ | --------- | ------- | -| `governed_attribute` | `def governed_attribute(target, key, value, value_schema = "inql.governance.string.v0.1", scope = GovernedAttributeScope.Field, source = GovernedAttributeSource.Inferred, confidence = GovernedAttributeConfidence.Unknown, status = GovernedAttributeStatus.Inferred, authority = None, observed_at_unix_nanoseconds = None, expires_at_unix_nanoseconds = None, evidence_refs = []) -> GovernedAttribute` | Build one governed fact attached to a semantic target. | +| `governed_attribute` | `def governed_attribute(target, key, value, value_schema = "incql.governance.string.v0.1", scope = GovernedAttributeScope.Field, source = GovernedAttributeSource.Inferred, confidence = GovernedAttributeConfidence.Unknown, status = GovernedAttributeStatus.Inferred, authority = None, observed_at_unix_nanoseconds = None, expires_at_unix_nanoseconds = None, evidence_refs = []) -> GovernedAttribute` | Build one governed fact attached to a semantic target. | | `policy_checkpoint` | `def policy_checkpoint(target, checkpoint, action, policy_ref, reason_code, evidence_refs = [], visibility = MetadataVisibility.Public, diagnostics = []) -> PolicyCheckpoint` | Build one policy decision or observation record attached to a semantic target. | ## Records @@ -24,7 +24,7 @@ from pub::inql import governed_attribute, policy_checkpoint | `GovernedAttribute` | One governed fact attached to a semantic target with provenance, confidence, status, authority, lifetime, and evidence references. | | `PolicyCheckpoint` | One policy decision or observation attached to a semantic target with checkpoint phase, action, policy reference, reason code, visibility, diagnostics, and evidence references. | -`GovernedAttribute.value` is a `MetadataPayload` with a schema and compact string value. The default helper shape is intentionally compact: inspection-derived primitive-kind attributes use `inql.governance.schema-primitive-kind.v0.1` to distinguish governed schema evidence from generic metadata strings. Catalog, contract, exchange, or policy-ingress steps can normalize richer external formats before they become governed attributes. +`GovernedAttribute.value` is a `MetadataPayload` with a schema and compact string value. The default helper shape is intentionally compact: inspection-derived primitive-kind attributes use `incql.governance.schema-primitive-kind.v0.1` to distinguish governed schema evidence from generic metadata strings. Catalog, contract, exchange, or policy-ingress steps can normalize richer external formats before they become governed attributes. `GovernedAttribute` exposes convenience methods: @@ -63,7 +63,7 @@ Inspection-derived governed attributes use `GovernedAttributeSource.Lineage`, `G ## Boundary -InQL carries governed evidence. It does not decide legal obligations, own an organization policy language, approve changes, or enforce masking/row-filter behavior by itself. A caller may read `PolicyCheckpoint` and `GovernedAttribute` records and choose how to act, but that enforcement decision is outside the record semantics. +IncQL carries governed evidence. It does not decide legal obligations, own an organization policy language, approve changes, or enforce masking/row-filter behavior by itself. A caller may read `PolicyCheckpoint` and `GovernedAttribute` records and choose how to act, but that enforcement decision is outside the record semantics. diff --git a/docs/language/reference/inspection.md b/docs/language/reference/inspection.md index b1acf920..c391e8d2 100644 --- a/docs/language/reference/inspection.md +++ b/docs/language/reference/inspection.md @@ -5,7 +5,7 @@ Local inspection exposes structured evidence records for Prism-backed lazy plans ## Entry points ```incan -from pub::inql import inspect_plan, inspect_lineage +from pub::incql import inspect_plan, inspect_lineage inspection = inspect_plan(summary) lineage = inspect_lineage(summary) @@ -20,7 +20,7 @@ The inspection surface consumes shared evidence record families and adds the loc | Record | Purpose | | ------ | ------- | | `SemanticTarget` | Stable local anchor for a plan, Prism node, relation output, field, read root, or future evidence family. | -| `PlanInspection` | Top-level inspection result with schema version, InQL version, plan target, output schema, output fields, Prism nodes, lineage, artifacts, diagnostics, and unsupported-evidence markers. | +| `PlanInspection` | Top-level inspection result with schema version, IncQL version, plan target, output schema, output fields, Prism nodes, lineage, artifacts, diagnostics, and unsupported-evidence markers. | | `InspectionNodeKind` | Typed public node-kind vocabulary for authored and rewritten Prism node inspection records. | | `LineageGraph` | Plan-local lineage graph with a rule version and typed lineage edges. | | `LineageEdge` | Source-to-destination edge with relationship kind, transformation kind, confidence, expression reference, and evidence references. | diff --git a/docs/language/reference/quality.md b/docs/language/reference/quality.md index 56233f45..112d6b67 100644 --- a/docs/language/reference/quality.md +++ b/docs/language/reference/quality.md @@ -1,6 +1,6 @@ # Quality assertions and observations (Reference) -This page documents the public quality assertion and observation surface in the InQL package. Normative design intent lives in [RFC 034][rfc-034]. +This page documents the public quality assertion and observation surface in the IncQL package. Normative design intent lives in [RFC 034][rfc-034]. Quality assertions are declarations. They do not filter, quarantine, or mutate the checked relation by themselves. Quality observations are runtime evidence produced when a `Session` evaluates assertions against explicit relation inputs. @@ -14,11 +14,11 @@ Quality assertions are declarations. They do not filter, quarantine, or mutate t | `group_row_count(group_by, min_count=1, max_count=None)` | group | Assert that every group has a row count within inclusive thresholds | | `cross_relation_row_count_equal()` | cross relation | Assert that two explicit relation inputs have the same row count | -The first implementation evaluates these helpers through ordinary InQL plans and `Session.collect_observed(...)`. Row-count checks use structured materialization row counts. Field and group checks build filter and aggregate plans and evaluate the resulting row counts. They do not scrape rendered preview text. +The first implementation evaluates these helpers through ordinary IncQL plans and `Session.collect_observed(...)`. Row-count checks use structured materialization row counts. Field and group checks build filter and aggregate plans and evaluate the resulting row counts. They do not scrape rendered preview text. ## Quality syntax -Importing `pub::inql` activates `quality { ... }` and expression-position `quality:` syntax. Both forms return `list[QualityAssertion]`. +Importing `pub::incql` activates `quality { ... }` and expression-position `quality:` syntax. Both forms return `list[QualityAssertion]`. ```incan checks = quality { diff --git a/docs/language/reference/query_blocks.md b/docs/language/reference/query_blocks.md index 0d4d97c9..1f12677d 100644 --- a/docs/language/reference/query_blocks.md +++ b/docs/language/reference/query_blocks.md @@ -1,9 +1,9 @@ # Query blocks (Reference) -Query blocks are dependency-activated InQL expressions. Import `pub::inql` to make the vocabulary and helper surface available in a downstream Incan package. +Query blocks are dependency-activated IncQL expressions. Import `pub::incql` to make the vocabulary and helper surface available in a downstream Incan package. ```incan -from pub::inql import DataFrame, count, desc, sum +from pub::incql import DataFrame, count, desc, sum from models import Order, OrderSummary def summarize_orders(orders: DataFrame[Order]) -> DataFrame[OrderSummary]: @@ -21,7 +21,7 @@ def summarize_orders(orders: DataFrame[Order]) -> DataFrame[OrderSummary]: The `OrderSummary` type parameter documents the intended output row model. The v0.1 implementation checks query schema evolution and selected aliases through the carrier planning surface; full field/type compatibility validation against annotated output models is tracked as schema-validation follow-up work. -InQL also accepts the colon spelling in expression position: +IncQL also accepts the colon spelling in expression position: ```incan selected = query: @@ -47,11 +47,11 @@ The implemented v0.1 query-block surface supports: - `EXPLODE as ` - `WINDOW BY = ` -`ORDER BY` uses InQL ordering helpers such as `asc(...)` and `desc(...)`; postfix SQL spellings such as `.amount DESC` are not part of the v0.1 query-block grammar. +`ORDER BY` uses IncQL ordering helpers such as `asc(...)` and `desc(...)`; postfix SQL spellings such as `.amount DESC` are not part of the v0.1 query-block grammar. ## Expressions -Query-block expressions use Incan expression operators and desugar to the same InQL helper calls available in ordinary method-chain code: +Query-block expressions use Incan expression operators and desugar to the same IncQL helper calls available in ordinary method-chain code: | Query expression | Helper equivalent | | ---------------- | ----------------- | @@ -71,4 +71,4 @@ The comparison helper names use `lte` and `gte` for inclusive bounds; `le` and ` - `SELECT` aliases become the output schema for later clauses. - A `SELECT` alias may be reused by later expressions in the same `SELECT` list. -Query blocks desugar into the same carrier method calls available to ordinary InQL code before lowering through the current carrier planning path. `LazyFrame` flows are Prism-backed; concrete `DataFrame` and `DataStream` flows still use their documented carrier paths before converging at the Substrait boundary. +Query blocks desugar into the same carrier method calls available to ordinary IncQL code before lowering through the current carrier planning path. `LazyFrame` flows are Prism-backed; concrete `DataFrame` and `DataStream` flows still use their documented carrier paths before converging at the Substrait boundary. diff --git a/docs/language/reference/substrait/conformance.md b/docs/language/reference/substrait/conformance.md index 0ce47da2..08c5745d 100644 --- a/docs/language/reference/substrait/conformance.md +++ b/docs/language/reference/substrait/conformance.md @@ -1,12 +1,12 @@ # Substrait conformance corpus (Reference) -This page documents where InQL's Substrait conformance scenarios live and how they are represented. It is the current reference for the conformance corpus shape used by package code and tests. +This page documents where IncQL's Substrait conformance scenarios live and how they are represented. It is the current reference for the conformance corpus shape used by package code and tests. -The corpus is the machine-readable validation layer for the current InQL package implementation profile. For operator-level mappings, see the [Substrait operator catalog][ref-operator-catalog]. +The corpus is the machine-readable validation layer for the current IncQL package implementation profile. For operator-level mappings, see the [Substrait operator catalog][ref-operator-catalog]. ## Source of truth -The canonical conformance corpus is implemented in InQL package code: +The canonical conformance corpus is implemented in IncQL package code: - `src/substrait/conformance.incn` - `src/substrait/conformance_catalog.incn` @@ -35,7 +35,7 @@ Each scenario is selected by `CoreScenarioKey` and materialized via `core_scenar `scenario_id` values must be stable and use this convention: ```text -inql.substrait... +incql.substrait... ``` The numeric suffix is immutable after publication. If requirements change incompatibly, add a new scenario ID instead of mutating semantics under an existing ID. @@ -46,18 +46,18 @@ Core scenarios currently implemented in `src/substrait/conformance_catalog.incn` | Scenario ID | Selector | Primary core `Rel` coverage | | ------------------------------------------------------ | ---------------------------------------------------------- | --------------------------- | -| `inql.substrait.core.read_named_table.001` | `core_scenario(CoreScenarioKey.ReadNamedTable)` | `ReadRel` (`NamedTable`) | -| `inql.substrait.core.read_local_files.001` | `core_scenario(CoreScenarioKey.ReadLocalFiles)` | `ReadRel` (`LocalFiles`) | -| `inql.substrait.core.read_virtual_table.001` | `core_scenario(CoreScenarioKey.ReadVirtualTable)` | `ReadRel` (`VirtualTable`) | -| `inql.substrait.core.filter_rows.001` | `core_scenario(CoreScenarioKey.FilterRows)` | `FilterRel` | -| `inql.substrait.core.project_computed_columns.001` | `core_scenario(CoreScenarioKey.ProjectComputedColumns)` | `ProjectRel` | -| `inql.substrait.core.join_rel_variants.001` | `core_scenario(CoreScenarioKey.JoinRelVariants)` | `JoinRel` | -| `inql.substrait.core.cross_rel_cartesian.001` | `core_scenario(CoreScenarioKey.CrossRelCartesian)` | `CrossRel` | -| `inql.substrait.core.aggregate_grouping_sets.001` | `core_scenario(CoreScenarioKey.AggregateGroupingSets)` | `AggregateRel` | -| `inql.substrait.core.sort_rel_ordering.001` | `core_scenario(CoreScenarioKey.SortRelOrdering)` | `SortRel` | -| `inql.substrait.core.fetch_rel_limit_offset.001` | `core_scenario(CoreScenarioKey.FetchRelLimitOffset)` | `FetchRel` | -| `inql.substrait.core.set_rel_operations.001` | `core_scenario(CoreScenarioKey.SetRelOperations)` | `SetRel` | -| `inql.substrait.core.reference_rel_shared_subplan.001` | `core_scenario(CoreScenarioKey.ReferenceRelSharedSubplan)` | `ReferenceRel` | +| `incql.substrait.core.read_named_table.001` | `core_scenario(CoreScenarioKey.ReadNamedTable)` | `ReadRel` (`NamedTable`) | +| `incql.substrait.core.read_local_files.001` | `core_scenario(CoreScenarioKey.ReadLocalFiles)` | `ReadRel` (`LocalFiles`) | +| `incql.substrait.core.read_virtual_table.001` | `core_scenario(CoreScenarioKey.ReadVirtualTable)` | `ReadRel` (`VirtualTable`) | +| `incql.substrait.core.filter_rows.001` | `core_scenario(CoreScenarioKey.FilterRows)` | `FilterRel` | +| `incql.substrait.core.project_computed_columns.001` | `core_scenario(CoreScenarioKey.ProjectComputedColumns)` | `ProjectRel` | +| `incql.substrait.core.join_rel_variants.001` | `core_scenario(CoreScenarioKey.JoinRelVariants)` | `JoinRel` | +| `incql.substrait.core.cross_rel_cartesian.001` | `core_scenario(CoreScenarioKey.CrossRelCartesian)` | `CrossRel` | +| `incql.substrait.core.aggregate_grouping_sets.001` | `core_scenario(CoreScenarioKey.AggregateGroupingSets)` | `AggregateRel` | +| `incql.substrait.core.sort_rel_ordering.001` | `core_scenario(CoreScenarioKey.SortRelOrdering)` | `SortRel` | +| `incql.substrait.core.fetch_rel_limit_offset.001` | `core_scenario(CoreScenarioKey.FetchRelLimitOffset)` | `FetchRel` | +| `incql.substrait.core.set_rel_operations.001` | `core_scenario(CoreScenarioKey.SetRelOperations)` | `SetRel` | +| `incql.substrait.core.reference_rel_shared_subplan.001` | `core_scenario(CoreScenarioKey.ReferenceRelSharedSubplan)` | `ReferenceRel` | ## Taxonomy values @@ -75,7 +75,7 @@ Conformance validation for the v1 profile is expected to run against canonical o `ProjectRel` and `AggregateRel` scenarios validate the Substrait relation boundary. Package tests cover the implemented scalar computed-column and grouped-aggregate method-chain behavior; windows, grouping sets, and distinct semantics have their own capability rows in the operator catalog. -Historical design context is captured in [InQL RFC 002][rfc-002], but this page is the source of truth for the current conformance corpus representation. +Historical design context is captured in [IncQL RFC 002][rfc-002], but this page is the source of truth for the current conformance corpus representation. diff --git a/docs/language/reference/substrait/operator_catalog.md b/docs/language/reference/substrait/operator_catalog.md index 81bfecb2..8d3010c5 100644 --- a/docs/language/reference/substrait/operator_catalog.md +++ b/docs/language/reference/substrait/operator_catalog.md @@ -1,6 +1,6 @@ # Substrait operator catalog (Reference) -This page is the **operational mapping reference** for InQL's Apache Substrait integration. The normative contract — including the Logical `Rel` alphabet, pinning policy, read-root boundary, and extension URI rules — lives in [InQL RFC 002][rfc-002]. This page provides the full capability → `Rel` catalog, profile tags, gap encoding requirements, and optional mutation profile detail that are too long-lived and versioned to remain inside the RFC text itself. +This page is the **operational mapping reference** for IncQL's Apache Substrait integration. The normative contract — including the Logical `Rel` alphabet, pinning policy, read-root boundary, and extension URI rules — lives in [IncQL RFC 002][rfc-002]. This page provides the full capability → `Rel` catalog, profile tags, gap encoding requirements, and optional mutation profile detail that are too long-lived and versioned to remain inside the RFC text itself. ## Profile tags @@ -13,13 +13,13 @@ Each entry in the catalog carries one of the following profile tags: | **gap** | No stable logical `Rel` exists in current core Substrait; **must** use a documented non-core encoding (see [Gap profiles](#gap-profiles)); ad hoc or undocumented encodings are non-conforming | | **optional-mutation** | Part of the optional mutation profile; not required for read/query analytical core; may be omitted by distributions that target read-only analytical use | -The same status taxonomy is used in the [Substrait conformance corpus][ref-conformance-corpus]. Scenario contracts in that corpus are represented as typed InQL models with stable scenario IDs so CI and downstream implementations can consume a stable machine contract. +The same status taxonomy is used in the [Substrait conformance corpus][ref-conformance-corpus]. Scenario contracts in that corpus are represented as typed IncQL models with stable scenario IDs so CI and downstream implementations can consume a stable machine contract. ## Read/query analytical core profile -The following table maps InQL plan capabilities to Substrait logical relations and expression patterns for the read/query analytical core — the minimum required for InQL v0.1. +The following table maps IncQL plan capabilities to Substrait logical relations and expression patterns for the read/query analytical core — the minimum required for IncQL v0.1. -| InQL capability (conceptual) | Substrait | Profile | +| IncQL capability (conceptual) | Substrait | Profile | | ------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | ------- | | Logical table / registered name | `ReadRel` + `NamedTable` | core | | File or object scan as plan input | `ReadRel` + `LocalFiles` (format options in the pinned spec) | core | @@ -46,15 +46,15 @@ The following table maps InQL plan capabilities to Substrait logical relations a ## Optional mutation profile -The following capabilities are part of the optional mutation profile. They are **not** required for InQL read/query analytical core (v0.1). An implementation that exposes any mutation-profile capability **must** document which relations are supported for its target backend and what portability guarantees (if any) apply. +The following capabilities are part of the optional mutation profile. They are **not** required for IncQL read/query analytical core (v0.1). An implementation that exposes any mutation-profile capability **must** document which relations are supported for its target backend and what portability guarantees (if any) apply. -| InQL capability | Substrait | Profile | +| IncQL capability | Substrait | Profile | | --------------------------------------------- | ----------- | ----------------- | | Write to a table / CTAS | `WriteRel` | optional-mutation | | Table update without a full child `Rel` input | `UpdateRel` | optional-mutation | | DDL (create, drop, alter) | `DdlRel` | optional-mutation | -Absence of these in a given distribution does not make InQL incomplete for read-only analytical use. +Absence of these in a given distribution does not make IncQL incomplete for read-only analytical use. ## Extension escape hatches @@ -72,7 +72,7 @@ Using an extension escape hatch without a registered URI is non-conforming. ### Unnest / explode -Core Substrait does not define a portable unnest or explode `Rel` at the logical level. Until a stable logical `Rel` for unnest is adopted in the pinned Substrait revision and recognized by InQL: +Core Substrait does not define a portable unnest or explode `Rel` at the logical level. Until a stable logical `Rel` for unnest is adopted in the pinned Substrait revision and recognized by IncQL: - `EXPLODE`-style behavior **must** lower through a registered extension relation (`ExtensionSingleRel` or `ExtensionLeafRel`) with a declared extension URI in the toolchain's public catalog. - Alternatively, a documented rewrite (for example, expanding a virtual table) **may** be used if the encoding is unambiguously specified in the public operator catalog for the toolchain version. @@ -80,10 +80,10 @@ Core Substrait does not define a portable unnest or explode `Rel` at the logical Current package-level RFC 002 boundary registration: -- `https://inql.io/extensions/v0.1/unnest.yaml#explode` -- `https://inql.io/extensions/v0.1/unnest.yaml#explode_outer` -- `https://inql.io/extensions/v0.1/unnest.yaml#posexplode` -- `https://inql.io/extensions/v0.1/unnest.yaml#posexplode_outer` +- `https://incql.io/extensions/v0.1/unnest.yaml#explode` +- `https://incql.io/extensions/v0.1/unnest.yaml#explode_outer` +- `https://incql.io/extensions/v0.1/unnest.yaml#posexplode` +- `https://incql.io/extensions/v0.1/unnest.yaml#posexplode_outer` ### Pivot / unpivot diff --git a/docs/language/reference/substrait/read_root_binding_contract.md b/docs/language/reference/substrait/read_root_binding_contract.md index 46f79413..7b0c321a 100644 --- a/docs/language/reference/substrait/read_root_binding_contract.md +++ b/docs/language/reference/substrait/read_root_binding_contract.md @@ -1,10 +1,10 @@ # Substrait read-root and binding contract (Reference) -This page is the **operational reference** for InQL's normative boundary between logical read roots in Substrait plans and execution-context binding. The normative rule — that logical reads carry names and virtual values rather than secrets, and that the execution context resolves them — lives in [InQL RFC 002][rfc-002]. This page expands on the `ReadRel` variant requirements, what a read must and must not contain, the execution context's obligations, and the adapter layer boundary. +This page is the **operational reference** for IncQL's normative boundary between logical read roots in Substrait plans and execution-context binding. The normative rule — that logical reads carry names and virtual values rather than secrets, and that the execution context resolves them — lives in [IncQL RFC 002][rfc-002]. This page expands on the `ReadRel` variant requirements, what a read must and must not contain, the execution context's obligations, and the adapter layer boundary. ## Normative boundary -InQL relational plans **must** express all new data entering the plan as logical reads. A logical read carries a **logical identity** — a name, a virtual row set, or an opaque extension type — without normative dependence on: +IncQL relational plans **must** express all new data entering the plan as logical reads. A logical read carries a **logical identity** — a name, a virtual row set, or an opaque extension type — without normative dependence on: - Secret material: credentials, tokens, API keys, or passwords. - Host-specific connection strings, DSNs, or URIs that encode execution-context policy. @@ -14,7 +14,7 @@ The execution context **must** resolve logical reads to physical resources throu ## `ReadRel` variant reference -| Variant | Substrait field | Typical InQL use | Portability | +| Variant | Substrait field | Typical IncQL use | Portability | | -------------- | ------------------------------------------ | ---------------------------------------------------------------- | -------------------------------------------------------------------------------------- | | `NamedTable` | `named_table` (list of name parts) | Registered logical table name; resolved by session registry | Portable across conforming consumers that have registered the same logical name | | `LocalFiles` | `local_files` (file list + format options) | Parquet, CSV, Arrow IPC scan from a URI | Portable if consumers can resolve the URI; URI format is not standardized by Substrait | @@ -35,7 +35,7 @@ The execution context **must** resolve logical reads to physical resources throu ## Execution context responsibilities -The execution context ([InQL RFC 004][rfc-004] `Session`) **must**: +The execution context ([IncQL RFC 004][rfc-004] `Session`) **must**: 1. Maintain a **table registry** that maps logical names to physical data source definitions (connection parameters, catalog references, or file paths). 2. **Resolve** `ReadRel` logical names through this registry at execution time — not at plan authoring time — so the serialized plan remains independent of execution-context state. @@ -51,11 +51,11 @@ Product SDKs and higher operational layers **may** provide convenience read APIs - **Must not** embed execution-context state — resolved credentials, session tokens, resolved endpoint URLs — in the `ReadRel` payload of the normative plan. - **May** pass execution-context configuration through separate, non-normative channels (for example, `AdvancedExtension` hints, out-of-band session configuration) when needed for optimization, provided the plan remains semantically valid without them. -Adapter-specific "open connection" or "bind source" APIs **should not** be specified as core InQL. They are thin wrappers at most, with the binding contract owned by the execution context per InQL RFC 004. +Adapter-specific "open connection" or "bind source" APIs **should not** be specified as core IncQL. They are thin wrappers at most, with the binding contract owned by the execution context per IncQL RFC 004. -## Interaction with InQL RFC 001 types +## Interaction with IncQL RFC 001 types -The following table summarizes how each `Session` read method maps to a `ReadRel` variant and the resulting InQL carrier type. +The following table summarizes how each `Session` read method maps to a `ReadRel` variant and the resulting IncQL carrier type. | `Session` method | Returns | `ReadRel` variant | | ------------------------------------ | -------------- | ------------------------------------------------- | @@ -69,4 +69,4 @@ In all cases the `LazyFrame[T]` holds a deferred plan — no data is fetched unt [rfc-002]: ../../rfcs/002_apache_substrait_integration.md -[rfc-004]: ../../../rfcs/004_inql_execution_context.md +[rfc-004]: ../../../rfcs/004_incql_execution_context.md diff --git a/docs/language/reference/substrait/revision_and_extension_policy.md b/docs/language/reference/substrait/revision_and_extension_policy.md index 6659ac14..420d7210 100644 --- a/docs/language/reference/substrait/revision_and_extension_policy.md +++ b/docs/language/reference/substrait/revision_and_extension_policy.md @@ -1,14 +1,14 @@ # Substrait revision and extension policy (Reference) -This page is the **operational policy reference** for InQL's Substrait revision pinning and extension function management. The normative rules — that pinning is required and that functions outside the core bundle must use registered extension URIs — live in [InQL RFC 002][rfc-002]. This page provides the operational detail: what must be declared in a release, how extension URIs are registered, what constitutes a breaking vs additive change, and the checklist contributors follow when bumping the pinned revision. +This page is the **operational policy reference** for IncQL's Substrait revision pinning and extension function management. The normative rules — that pinning is required and that functions outside the core bundle must use registered extension URIs — live in [IncQL RFC 002][rfc-002]. This page provides the operational detail: what must be declared in a release, how extension URIs are registered, what constitutes a breaking vs additive change, and the checklist contributors follow when bumping the pinned revision. ## Revision pinning ### Requirements -Each conforming InQL toolchain release **must** declare: +Each conforming IncQL toolchain release **must** declare: -- The exact Substrait **revision** it targets (commit hash or tagged release, depending on the Substrait project's versioning model at time of the InQL release). +- The exact Substrait **revision** it targets (commit hash or tagged release, depending on the Substrait project's versioning model at time of the IncQL release). - Any **bundled extension function sets** (YAML or equivalent) shipped alongside the toolchain. Each set must identify its URI prefix, the Substrait revision it was authored against, and the set of functions it covers. This information **must** appear in: @@ -48,10 +48,10 @@ Functions not in the pinned core Substrait bundle **must** use extension URIs th ### URI structure -InQL toolchain extension URIs **should** follow the pattern: +IncQL toolchain extension URIs **should** follow the pattern: ```text -https://inql.io/extensions//.yaml +https://incql.io/extensions//.yaml ``` Where: @@ -59,7 +59,7 @@ Where: - `` is the toolchain version that introduced the extension (e.g. `v0.1`). - `` groups related functions (e.g. `aggregate`, `string`, `temporal`, `unnest`). -The exact URI scheme is part of the toolchain release process and **must** be documented alongside the release. The current InQL package code uses the `inql.io` base for registered extension examples and treats pre-1.0 entries as provisional until the wider release process is finalized. +The exact URI scheme is part of the toolchain release process and **must** be documented alongside the release. The current IncQL package code uses the `incql.io` base for registered extension examples and treats pre-1.0 entries as provisional until the wider release process is finalized. ### `AdvancedExtension` fields @@ -73,7 +73,7 @@ The exact URI scheme is part of the toolchain release process and **must** be do ### Additive changes (default) -Mapping catalog additions — new InQL capabilities mapped to Substrait, new extension URIs registered, new optional capabilities documented — are **additive changes**. They: +Mapping catalog additions — new IncQL capabilities mapped to Substrait, new extension URIs registered, new optional capabilities documented — are **additive changes**. They: - Do not require an RFC amendment. - **Must** appear in release notes with the capability name and profile tag. diff --git a/docs/release_notes/v0_1.md b/docs/release_notes/v0_1.md index 8ee4a9e0..0bfdf921 100644 --- a/docs/release_notes/v0_1.md +++ b/docs/release_notes/v0_1.md @@ -1,35 +1,35 @@ -# InQL v0.1 release notes +# IncQL v0.1 release notes -**Status:** Unreleased. InQL v0.1 is the milestone where the RFC **000–004** series becomes meaningfully usable end-to-end. The current package already exposes dataset carriers, a real Substrait boundary, Prism-backed `LazyFrame` planning, and Session-backed execution. See the [RFC index](../rfcs/README.md) for normative design. +**Status:** Unreleased. IncQL v0.1 is the milestone where the RFC **000–004** series becomes meaningfully usable end-to-end. The current package already exposes dataset carriers, a real Substrait boundary, Prism-backed `LazyFrame` planning, and Session-backed execution. See the [RFC index](../rfcs/README.md) for normative design. ## Features and enhancements Entries will be filled in as work lands (link RFCs and PRs when applicable). -- **Language:** Foundational InQL syntax and semantics (naming, query schema, layer boundaries). +- **Language:** Foundational IncQL syntax and semantics (naming, query schema, layer boundaries). - **Carriers:** `DataSet[T]` hierarchy including bounded vs unbounded traits and concrete frame/stream types. - **Plans:** Apache Substrait as the logical interchange contract. - **Authoring:** `LazyFrame` method chains are Prism-backed, and RFC 003 `query {}` blocks desugar into the same carrier calls before lowering through the current carrier planning paths and Substrait boundary. Query blocks support the brace spelling and expression-position `query:` spelling, including SELECT aliases, lateral alias reuse, grouped aggregates, `SELECT DISTINCT`, post-SELECT filters, ordering, limits, inner and left joins, generator clauses, and named window expressions. - **Aggregates:** builder-based `col`, `sum`, `count`, `count_expr`, `count_distinct`, `count_if`, `avg`, `min`, and `max` helpers now lower grouped and global aggregates through Prism, Substrait, and Session execution. `count()` counts rows, `count(expr)` counts non-null expression values, `count_expr(expr)` remains a compatibility spelling, and the first aggregate modifier slice supports `DISTINCT` plus aggregate-local `FILTER` where valid. - **Scalar expressions:** RFC 012 unifies filter predicates, computed projection values, grouping keys, and aggregate inputs around one `ColumnExpr` surface with canonical `lit(...)` and typed literal helpers. - **Core scalar functions:** RFC 015 adds registry-backed scalar function applications and the first core helper slice for casts, comparisons, boolean logic, null/NaN predicates, arithmetic, conditionals, membership/range predicates, and ordering expressions. Primitive cast targets can use source-level type tokens such as `cast(col("amount_text"), float)`, while explicit string target spellings remain available for compatibility aliases such as `int64` and `float64`. Implemented helpers lower to Substrait IR through registry metadata, built-in Rex shapes, or structural sort-field lowering; DataFusion remains the first execution adapter rather than the semantic boundary. -- **Common scalar functions:** RFC 018 adds registry-backed common math, string, text encoding, regex, and date/time helpers, including compatibility aliases such as `substr(...)`, `ucase(...)`, `lcase(...)`, `dateadd(...)`, `datediff(...)`, and `safe_cast(...)`. Helpers lower through registry-owned Substrait metadata, with DataFusion adapter UDFs only where the first backend has no direct API-level equivalent for the InQL semantic contract. +- **Common scalar functions:** RFC 018 adds registry-backed common math, string, text encoding, regex, and date/time helpers, including compatibility aliases such as `substr(...)`, `ucase(...)`, `lcase(...)`, `dateadd(...)`, `datediff(...)`, and `safe_cast(...)`. Helpers lower through registry-owned Substrait metadata, with DataFusion adapter UDFs only where the first backend has no direct API-level equivalent for the IncQL semantic contract. - **Nested data functions:** RFC 020 adds registry-backed scalar helpers for array construction/access, cardinality, containment, overlap, sorting, set-like operations, joining, slicing, reversing, scalar array flattening, map construction/access, map key/value/entry extraction, map key containment, and named struct construction. These helpers lower through Substrait extension metadata without introducing generator semantics, with representative DataFusion-backed Session coverage for composable array projection paths. - **Generator functions:** RFC 021 adds registry-backed generator applications for `explode(...)`, `explode_outer(...)`, `posexplode(...)`, `posexplode_outer(...)`, `inline(...)`, `inline_outer(...)`, portable `flatten(...)`, and `stack(...)`. Generators remain relation-shaping operations applied with `generate(...)`; they preserve input columns, require explicit output aliases, lower through the current Substrait extension-relation gap encoding, and execute through the DataFusion Session adapter with concrete output-column materialization. - **Window functions:** RFC 019 adds `window()` specs, explicit row/range frame bounds, ranking and distribution helpers (`row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, `ntile`), offset and value helpers (`lag`, `lead`, `first_value`, `last_value`, `nth_value`), and aggregate-over-window placement through `with_window_column(...)`. Portable window helpers require explicit ordering where appropriate, lower through Substrait `ConsistentPartitionWindowRel`, and execute through the DataFusion session adapter. - **Format functions:** RFC 022 adds scalar payload helpers for deterministic hashes (`md5`, `sha1`, `sha224`, `sha256`, `sha384`, `sha512`, `sha2`, `crc32`, and `xxhash64`), URL parsing/encoding/decoding, JSON validation/path/schema helpers, and CSV row/schema helpers. Format helpers lower through registry-owned Substrait metadata; the DataFusion adapter executes the full helper set with native functions where available and Incan-authored adapter callbacks for non-native helpers. -- **Approximate functions:** RFC 023 adds explicit approximate aggregate helpers for `approx_count_distinct(...)` and `approx_percentile(...)`. They carry approximation policy in registry metadata, lower through InQL-owned Substrait extension names, and keep DataFusion implementation-name rewrites inside the backend adapter. -- **Typed sketches:** RFC 025 adds typed HyperLogLog sketch logical values with `SketchLogicalType`, `SketchExpr`, `hll_type(...)`, `sketch_col(...)`, `hll_sketch(...)`, `hll_merge(...)`, `hll_estimate(...)`, `hll_serialize(...)`, and `hll_deserialize(...)`. Sketch metadata remains InQL-owned through registry and Substrait options; DataFusion reports a backend planning diagnostic because it has no sketch runtime implementation. -- **Typed variants:** RFC 026 adds typed semi-structured variant logical values with `VariantLogicalType`, `VariantExpr`, `variant_type(...)`, `variant_col(...)`, `variant_value(...)`, `parse_variant_json(...)`, `try_parse_variant_json(...)`, `variant_get(...)`, `typeof(...)`, and kind predicates for null, boolean, integer, float, string, timestamp, array, and object values. Variant parse helpers accept string value-or-column inputs, variant metadata remains InQL-owned through registry and Substrait options, and DataFusion reports a backend planning diagnostic because it has no variant runtime implementation. +- **Approximate functions:** RFC 023 adds explicit approximate aggregate helpers for `approx_count_distinct(...)` and `approx_percentile(...)`. They carry approximation policy in registry metadata, lower through IncQL-owned Substrait extension names, and keep DataFusion implementation-name rewrites inside the backend adapter. +- **Typed sketches:** RFC 025 adds typed HyperLogLog sketch logical values with `SketchLogicalType`, `SketchExpr`, `hll_type(...)`, `sketch_col(...)`, `hll_sketch(...)`, `hll_merge(...)`, `hll_estimate(...)`, `hll_serialize(...)`, and `hll_deserialize(...)`. Sketch metadata remains IncQL-owned through registry and Substrait options; DataFusion reports a backend planning diagnostic because it has no sketch runtime implementation. +- **Typed variants:** RFC 026 adds typed semi-structured variant logical values with `VariantLogicalType`, `VariantExpr`, `variant_type(...)`, `variant_col(...)`, `variant_value(...)`, `parse_variant_json(...)`, `try_parse_variant_json(...)`, `variant_get(...)`, `typeof(...)`, and kind predicates for null, boolean, integer, float, string, timestamp, array, and object values. Variant parse helpers accept string value-or-column inputs, variant metadata remains IncQL-owned through registry and Substrait options, and DataFusion reports a backend planning diagnostic because it has no variant runtime implementation. - **Function registry:** RFC 014 adds declaration-site registry decorators for the current public helper surface, including stable function references, checked signature projection, lifecycle metadata, behavior categories, alias policy, Substrait mapping categories, and checked API metadata drift validation. -- **Function extension policy:** InQL RFC 024 policy metadata now distinguishes portable core functions, namespaced extension-only functions, opt-in compatibility aliases, engine-specific functions, and rejected compatibility requests without adding an extension plugin system or backend-owned semantics. +- **Function extension policy:** IncQL RFC 024 policy metadata now distinguishes portable core functions, namespaced extension-only functions, opt-in compatibility aliases, engine-specific functions, and rejected compatibility requests without adding an extension plugin system or backend-owned semantics. - **Projection:** builder-based `with_column`, `add`, `mul`, and literal expression helpers now lower derived columns through Prism, Substrait, and Session execution. - **Substrait internals:** RFC 002 helpers are now split into focused owner modules for relation building, plan assembly, inspection, schema registry, extension bookkeeping, and expression lowering instead of one `substrait.plan` godmodule. - **Prism:** `LazyFrame` lowering applies safe canonical rewrites (`Filter(true)` elimination and adjacent `Limit`/`Project`/`OrderBy` collapse) before RFC 002 plan emission. - **Inspection:** RFCs 028–031 now have a first local evidence spine for Prism-backed `LazyFrame` plans. `inspect_plan(...)` and `inspect_lineage(...)` expose semantic targets, output schema, authored and rewritten Prism node records, lineage edges, artifact-family summaries, metadata attachment records, diagnostics shape, and explicit unsupported-evidence markers without executing or backend-binding the plan. - **Execution observations and adapter coverage:** RFC 032 adds observed `Session` variants for `execute`, `collect`, and `write` so callers can capture structured runtime evidence for success and failure attempts. RFC 033 adds adapter requirement and coverage records, inspection-inferred adapter requirements, `Session.check_coverage(requirements)` for explicit capability checks, and `Session.check_plan_coverage(data)` / `Session.check_inspection_coverage(inspection)` for plan-evidence-driven coverage checks with covered, partially covered, uncovered, and unknown states. - **Quality observations:** RFC 034 adds quality assertion declarations, `quality { ... }` / `quality:` assertion-list syntax, and `Session.observe_quality(...)` / `Session.observe_quality_pair(...)` so relation, field, group, and explicit cross-relation checks produce structured quality observations. The first helper set covers row-count thresholds, null-rate thresholds, uniqueness, group row-count thresholds, and cross-relation row-count equality without treating failed checks as implicit filters or pipeline gates. -- **Governed evidence:** RFC 035 adds governed attribute and policy checkpoint records so callers can attach provenance, confidence, status, authority, lifetime, visibility, diagnostics, and evidence references to semantic targets. Local plan inspection now emits conservative schema-derived governed attributes and planning observation checkpoints without turning InQL into a policy engine. +- **Governed evidence:** RFC 035 adds governed attribute and policy checkpoint records so callers can attach provenance, confidence, status, authority, lifetime, visibility, diagnostics, and evidence references to semantic targets. Local plan inspection now emits conservative schema-derived governed attributes and planning observation checkpoints without turning IncQL into a policy engine. - **Execution:** Session-oriented read, execute, and write (reference backend per RFC 004), with `collect(...)` now producing structured `DataFrame` materialization metadata plus preview text instead of treating rendered text as the canonical contract. Session execution dispatch now routes through a backend adapter boundary over Substrait plans; DataFusion remains the first adapter rather than being encoded directly into Session state. - **Session API:** `Session.write(data, target)` now accepts typed sink descriptors such as `csv_sink(uri)` and `parquet_sink(uri)`, while the file-specific `write_csv(...)` and `write_parquet(...)` helpers remain as convenience methods. - **Documentation:** Current package behavior is documented under `docs/language/`, while RFCs remain design records rather than implementation diaries. @@ -38,8 +38,8 @@ Pipe-forward (`|>`) is specified in RFC 005 but **out of scope** for v0.1. ## Bugfixes -- **Substrait boundary:** RFC 002 now exposes explicit join/set operation helpers, preserves `ReferenceRel` ordinals, and uses a registered `inql.io` extension URI for the current EXPLODE gap encoding. +- **Substrait boundary:** RFC 002 now exposes explicit join/set operation helpers, preserves `ReferenceRel` ordinals, and uses a registered `incql.io` extension URI for the current EXPLODE gap encoding. ## Documentation -- Architecture and RFC series: [docs/README.md](../README.md), [InQL architecture](../architecture.md), [RFCs](../rfcs/README.md). +- Architecture and RFC series: [docs/README.md](../README.md), [IncQL architecture](../architecture.md), [RFCs](../rfcs/README.md). diff --git a/docs/rfcs/000_inql_syntax.md b/docs/rfcs/000_incql_syntax.md similarity index 80% rename from docs/rfcs/000_inql_syntax.md rename to docs/rfcs/000_incql_syntax.md index b1c66a93..645c1a0d 100644 --- a/docs/rfcs/000_inql_syntax.md +++ b/docs/rfcs/000_incql_syntax.md @@ -1,21 +1,21 @@ -# InQL RFC 000: Language Specification +# IncQL RFC 000: Language Specification - **Status:** Planned - **Created:** 2026-03-18 - **Author(s):** Danny Meijer - **Related:** - -- **Issue:** [InQL #1](https://github.com/encero-systems/InQL/issues/1) +- **Issue:** [IncQL #1](https://github.com/encero-systems/IncQL/issues/1) - **RFC PR:** - - **Written against:** Incan v0.2 - **Shipped in:** - ## Summary -InQL is the typed data logic plane for the Incan ecosystem: relational queries, schema-parameterized DataFrame transformations, and backend-neutral logical planning. This RFC is the foundational specification for InQL v0.1. It defines what InQL is and what it owns, the core semantic model, naming forms and resolution rules, schema shapes, the compilation model, and layer boundaries. Companion RFCs address dataset types, plan interchange, query grammar, execution context, and pipe-forward syntax. +IncQL is the typed data logic plane for the Incan ecosystem: relational queries, schema-parameterized DataFrame transformations, and backend-neutral logical planning. This RFC is the foundational specification for IncQL v0.1. It defines what IncQL is and what it owns, the core semantic model, naming forms and resolution rules, schema shapes, the compilation model, and layer boundaries. Companion RFCs address dataset types, plan interchange, query grammar, execution context, and pipe-forward syntax. ## Core model -InQL treats all data logic as one relational semantic model exposed through multiple authoring surfaces: +IncQL treats all data logic as one relational semantic model exposed through multiple authoring surfaces: 1. **`query {}` blocks** — SQL-familiar clause surface 2. **`DataSet[T]` method chains** — programmatic API @@ -33,42 +33,42 @@ The key rule: **in relational clause positions, bare identifiers resolve against ## Motivation -Relational code in Incan must resolve field access and column names deterministically and statically where the language promises checking. Without a single foundational specification, `query {}` and method-chain surfaces would drift, schema-shape rules would be inconsistent across carriers, and the boundary between data logic and execution would blur. This RFC consolidates that model so every companion RFC can cite it rather than redefine it, and so that completion of RFCs 000–004 constitutes a usable InQL v0.1. +Relational code in Incan must resolve field access and column names deterministically and statically where the language promises checking. Without a single foundational specification, `query {}` and method-chain surfaces would drift, schema-shape rules would be inconsistent across carriers, and the boundary between data logic and execution would blur. This RFC consolidates that model so every companion RFC can cite it rather than redefine it, and so that completion of RFCs 000–004 constitutes a usable IncQL v0.1. ## Goals -- Define what InQL is, what it owns, and what it does not own. +- Define what IncQL is, what it owns, and what it does not own. - Establish the core semantic model: one relational model, multiple authoring surfaces. - Define the four naming forms and resolution rules for identifier binding inside relational contexts. - Define schema shapes: fully typed, open-ended, and dynamic. -- Define the compilation model: how InQL source flows through the Incan compiler pipeline to portable plans and execution. -- Define layer boundaries: InQL owns typed data logic and logical planning; execution, orchestration, and operational semantics belong to adjacent layers. +- Define the compilation model: how IncQL source flows through the Incan compiler pipeline to portable plans and execution. +- Define layer boundaries: IncQL owns typed data logic and logical planning; execution, orchestration, and operational semantics belong to adjacent layers. - Establish the relationship to Incan models as the schema surface for query authoring and validation. - State that `query {}` and method-chain surfaces, when both are present, **must not** change resolution rules relative to each other. - State that `.column` is only valid inside relational expression positions — `query {}` blocks, relational operation arguments (e.g. `.filter(...)`, `.join(...)`), and future pipe-forward stages — and is not a general Incan expression form. ## Non-Goals -- Dataset types, carriers, and the trait hierarchy (`DataSet[T]`, `BoundedDataSet[T]`, `UnboundedDataSet[T]`) — InQL RFC 001. -- Apache Substrait plan interchange and `Rel`-level mapping — InQL RFC 002. -- `query {}` grammar, clause inventory, and typechecking — InQL RFC 003. -- Execution context, session, DataFusion, read/write — InQL RFC 004. -- Pipe-forward (`|>`) syntax — InQL RFC 005 (not in v0.1 scope). +- Dataset types, carriers, and the trait hierarchy (`DataSet[T]`, `BoundedDataSet[T]`, `UnboundedDataSet[T]`) — IncQL RFC 001. +- Apache Substrait plan interchange and `Rel`-level mapping — IncQL RFC 002. +- `query {}` grammar, clause inventory, and typechecking — IncQL RFC 003. +- Execution context, session, DataFusion, read/write — IncQL RFC 004. +- Pipe-forward (`|>`) syntax — IncQL RFC 005 (not in v0.1 scope). - Orchestration, workflows, quality gates, contract enforcement — execution and operational layers. - Governed business meaning and semantic serving — semantic layer. - Cluster-scale scheduling, distributed fault tolerance — orchestration layer. ## Guide-level explanation -### What InQL does +### What IncQL does -InQL lets authors express typed data logic — queries, transformations, aggregations, joins — against schema-parameterized datasets, with compile-time validation and backend-neutral logical plans. Authors write relational intent; the compiler checks it against `model` schemas; the toolchain lowers it to a portable plan (Apache Substrait) that an execution context can optimize and run. +IncQL lets authors express typed data logic — queries, transformations, aggregations, joins — against schema-parameterized datasets, with compile-time validation and backend-neutral logical plans. Authors write relational intent; the compiler checks it against `model` schemas; the toolchain lowers it to a portable plan (Apache Substrait) that an execution context can optimize and run. ### The four naming forms -When you write `.amount`, `amount`, `orders.amount`, or `total_spend` inside an InQL query, what exactly are you referring to? +When you write `.amount`, `amount`, `orders.amount`, or `total_spend` inside an IncQL query, what exactly are you referring to? -InQL has four distinct naming forms, and the answer depends on which one you're using: +IncQL has four distinct naming forms, and the answer depends on which one you're using: - `.column` — the field from the current input relation - `relation.column` — the field from a named joined relation @@ -93,7 +93,7 @@ query { `.amount` and `.customer_id` are form 1 (primary relation). `customers.name` is form 2 (named join). `threshold` is form 4 (ordinary Incan binding — no column by that name exists in the query schema). -If you remember only five things about InQL naming: +If you remember only five things about IncQL naming: 1. `.column` is explicit access to the current input relation. 2. `relation.column` is explicit access to a named joined relation. @@ -103,7 +103,7 @@ If you remember only five things about InQL naming: ### Schema shapes -InQL supports several schema modes without losing its semantic model: +IncQL supports several schema modes without losing its semantic model: - **Fully typed** (`DataFrame[Order]`) — the compiler knows every field name, type, and nullability. This is the strongest and most ergonomic mode. - **Open-ended** (`DataFrame[Event]` where `Event` is declared `with OpenEnded`) — declared fields are fully typed; undeclared fields become soft-runtime references with a warning. @@ -111,10 +111,10 @@ InQL supports several schema modes without losing its semantic model: ### Two authoring surfaces (v0.1) -The examples in this section assume aggregate helpers such as `sum` are imported from `pub::inql.functions` (they are not ambient builtins). +The examples in this section assume aggregate helpers such as `sum` are imported from `pub::incql.functions` (they are not ambient builtins). ```incan -from pub::inql.functions import sum +from pub::incql.functions import sum # query {} blocks query { @@ -135,9 +135,9 @@ These lower to the same relational plan. Identifier resolution rules do not chan ## Reference-level explanation -### 1. What InQL owns +### 1. What IncQL owns -InQL owns the data-logic concerns of the platform: +IncQL owns the data-logic concerns of the platform: - query authoring - relational plan construction @@ -157,7 +157,7 @@ Its job is to preserve one deterministic, type-aware model of data intent across ### 2. Naming forms -Inside InQL relational contexts there are four distinct naming forms: +Inside IncQL relational contexts there are four distinct naming forms: 1. **`.column`** (primary relation) Explicit field access against the current input relation. In `query {}`, that is the relation established by `FROM` for `.column` positions — including after `JOIN`. `.column` does not refer to joined relations. @@ -281,7 +281,7 @@ query { } ``` -`threshold` is not a column; it resolves as the surrounding binding. This is how InQL stays composable with the rest of Incan instead of becoming an isolated mini-language. +`threshold` is not a column; it resolves as the surrounding binding. This is how IncQL stays composable with the rest of Incan instead of becoming an isolated mini-language. ### 5. Ambiguity examples @@ -365,13 +365,13 @@ Using `.field` outside these contexts **must** be a compile-time error. This kee ### 8. Schema shapes -InQL needs to support several schema modes without losing its semantic model. +IncQL needs to support several schema modes without losing its semantic model. #### 8.1 Fully typed: `DataFrame[T]` `DataFrame[T]` carries the full field-level schema through compilation. `.filter(.amount > 100)` can validate `amount` exists and has a compatible type; projection can produce a new typed output shape; grouping and aggregation can be validated against the schema. -When `T` is an Incan model that participates in contract semantics upstream, InQL **should** treat it first as a schema surface: query checking uses the model's fields and types; projection and output typing flow from that model shape. +When `T` is an Incan model that participates in contract semantics upstream, IncQL **should** treat it first as a schema surface: query checking uses the model's fields and types; projection and output typing flow from that model shape. #### 8.2 Open-ended: `model X with OpenEnded` @@ -415,7 +415,7 @@ Operationally: #### 8.4 Four schema shapes -Once both openness and subscription are allowed, InQL recognizes four common schema shapes: +Once both openness and subscription are allowed, IncQL recognizes four common schema shapes: 1. **Closed local** — a plain local model. Declared fields are the complete schema surface. 2. **Open local** — a local model `with OpenEnded`. Declared fields are the minimum guaranteed schema surface. @@ -427,7 +427,7 @@ This 2x2 framing separates two concerns cleanly: - **completeness**: closed vs open-ended - **provenance**: local vs subscribed or projected -InQL **should** be able to reason about all four shapes using the same core machinery: known guaranteed fields, openness of the schema surface, and provenance of fields where available. +IncQL **should** be able to reason about all four shapes using the same core machinery: known guaranteed fields, openness of the schema surface, and provenance of fields where available. #### 8.5 Schema boundary validation @@ -437,26 +437,26 @@ Where lower-bound structural surfaces are available, the compiler **should** use ### 9. Relationship to Incan models -InQL does not invent its own schema-definition system. It relies on Incan models as the typed schema surface for query authoring and validation. +IncQL does not invent its own schema-definition system. It relies on Incan models as the typed schema surface for query authoring and validation. -An Incan model gives InQL: +An Incan model gives IncQL: - field names - field types - nullability - structural shape for joins, projections, and outputs -InQL **must not** redefine schema language or model declarations; it consumes models as-is. Subscription and compatibility semantics (narrowing, widening, blast radius) belong in the operational and contract layers above InQL, not in InQL itself. InQL **may** surface subscription-derived schema information during typechecking and planning, but the declaration and enforcement model belongs to the contract layer. +IncQL **must not** redefine schema language or model declarations; it consumes models as-is. Subscription and compatibility semantics (narrowing, widening, blast radius) belong in the operational and contract layers above IncQL, not in IncQL itself. IncQL **may** surface subscription-derived schema information during typechecking and planning, but the declaration and enforcement model belongs to the contract layer. ### 10. Compilation model -InQL queries follow the same broad compiler pipeline as ordinary Incan code, with query-specific stages for relational planning: +IncQL queries follow the same broad compiler pipeline as ordinary Incan code, with query-specific stages for relational planning: ```text -InQL source +IncQL source → parser and AST → typechecker (name resolution, schema flow, type inference) - → InQL IR (relational operators, typed expressions) + → IncQL IR (relational operators, typed expressions) → Substrait emission → execution context optimization and execution ``` @@ -468,17 +468,17 @@ The user-facing query model stays stable while the lower layers evolve: AST shap The boundaries matter: - **Incan** owns the shared language, type system, modules, traits, compile-time safety, and the core `model` type definitions that queries use as schemas. -- **InQL** owns typed data logic, relational semantics, schema flow, and logical planning. +- **IncQL** owns typed data logic, relational semantics, schema flow, and logical planning. - **Execution layer** owns runners, workflows, adapters, quality enforcement, contract enforcement, and operational workload semantics. - **Semantic layer** owns governed business meaning and semantic-serving abstractions. -InQL sits above the language core and below execution, semantics, and product layers. +IncQL sits above the language core and below execution, semantics, and product layers. ## Design details ### Cross-RFC consistency -- All companion InQL RFCs **must** stay consistent with this document for naming forms, current query schema behavior, resolution order, schema shapes, and the relational-position restriction on `.column` described in §§2–7. +- All companion IncQL RFCs **must** stay consistent with this document for naming forms, current query schema behavior, resolution order, schema shapes, and the relational-position restriction on `.column` described in §§2–7. - Extensions in companion RFCs **must not** contradict these rules without an explicit amendment to this RFC. - Any future authoring surface (including pipe-forward), when introduced, **must** desugar to the same semantic model and **must not** change identifier resolution rules. @@ -506,16 +506,16 @@ InQL sits above the language core and below execution, semantics, and product la - **Typechecker**: resolution order, current query schema flow, `SELECT` alias publication, join alias handling, and schema-shape behavior (typed, open-ended, dynamic) must remain consistent across surfaces. - **IR / lowering**: both authoring surfaces must lower to one relational semantic model without changing identifier meaning or schema-flow behavior. - **LSP**: relational positions need resolution-aware highlighting, diagnostics, and completions that reflect the same naming rules as the compiler. -- **Documentation**: companion InQL RFCs and contributor docs must describe the same foundational naming, schema, and boundary rules. +- **Documentation**: companion IncQL RFCs and contributor docs must describe the same foundational naming, schema, and boundary rules. ## Design Decisions - **`.column` scope restriction:** `.field` is only valid in relational expression positions (`query {}` clauses, method-chain relational arguments, and future pipe-forward stages). It is a compile-time error outside those positions. This keeps the dot notation unambiguous and ensures a primary relation is always in scope when `.field` is used. - **Lateral column aliases in `SELECT`:** an alias defined in a `SELECT` list is visible to subsequent expressions in the same list, in order (lateral column alias semantics). This follows the convention of DuckDB, Snowflake, and MySQL. An alias is not visible to expressions that precede it in the list. Implementations **must** rewrite dependent expressions (e.g. inline substitution) before Substrait lowering, since Substrait projection nodes do not natively support lateral alias references. -- **Pipe-forward deferred to v0.2+:** InQL RFC 005 specifies pipe-forward as an optional surface with the same resolution rules. It is not part of v0.1 scope. This RFC (§6) states the invariant that pipe-forward **must** share §§2–4 resolution rules. -- **Method-chain API scope:** deferred to InQL RFC 001, which defines the `DataSet[T]` operation surface. This RFC does not mandate a particular chain API shape. -- **`HAVING` keyword:** not InQL syntax. Post-`SELECT` filtering uses a second `WHERE` clause (clause ordering details — InQL RFC 003). +- **Pipe-forward deferred to v0.2+:** IncQL RFC 005 specifies pipe-forward as an optional surface with the same resolution rules. It is not part of v0.1 scope. This RFC (§6) states the invariant that pipe-forward **must** share §§2–4 resolution rules. +- **Method-chain API scope:** deferred to IncQL RFC 001, which defines the `DataSet[T]` operation surface. This RFC does not mandate a particular chain API shape. +- **`HAVING` keyword:** not IncQL syntax. Post-`SELECT` filtering uses a second `WHERE` clause (clause ordering details — IncQL RFC 003). - **Schema shape priority for v0.1:** fully typed (`DataFrame[T]`) is the primary mode. Open-ended (`with OpenEnded`) and dynamic (`DataFrame[Dynamic]`) remain part of the normative model in this RFC, but any implementation that does not yet support a permitted behavior **must** reject it explicitly rather than silently reinterpreting or weakening these semantics. -- **Open-ended model marker syntax:** `model X with OpenEnded` is the normative spelling in this RFC for v0.1. If Incan changes how openness is declared, this RFC **must** be amended in lockstep; implementations follow the Incan language as shipped and update InQL specification text accordingly. -- **External / catalog-bound models:** resolution of `model X = external("catalog://...")` (or equivalent) and how such models surface in query schemas is **out of scope for InQL RFC 000** and for v0.1 closure. It depends on Incan contract and catalog semantics and **may** be specified in a later InQL RFC once those foundations exist. +- **Open-ended model marker syntax:** `model X with OpenEnded` is the normative spelling in this RFC for v0.1. If Incan changes how openness is declared, this RFC **must** be amended in lockstep; implementations follow the Incan language as shipped and update IncQL specification text accordingly. +- **External / catalog-bound models:** resolution of `model X = external("catalog://...")` (or equivalent) and how such models surface in query schemas is **out of scope for IncQL RFC 000** and for v0.1 closure. It depends on Incan contract and catalog semantics and **may** be specified in a later IncQL RFC once those foundations exist. - **Shadowing warning and LSP:** when a bare name matches both a query column and an outer Incan binding, query semantics and the diagnostic (warn; suggest `.column`) are normative above. Whether the LSP offers quick-fixes (e.g. rewrite to `.column`, rename outer binding) is **tooling policy** in the Incan LSP and is not prescribed here; any such fixes **must** preserve the resolution rules in §§2–4. diff --git a/docs/rfcs/001_inql_dataset.md b/docs/rfcs/001_incql_dataset.md similarity index 81% rename from docs/rfcs/001_inql_dataset.md rename to docs/rfcs/001_incql_dataset.md index c7219716..543e46f2 100644 --- a/docs/rfcs/001_inql_dataset.md +++ b/docs/rfcs/001_incql_dataset.md @@ -1,21 +1,21 @@ -# InQL RFC 001: Dataset types and carriers (`DataSet[T]`) +# IncQL RFC 001: Dataset types and carriers (`DataSet[T]`) - **Status:** In Progress - **Created:** 2026-03-22 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 000 (language specification — naming, schema shapes, layer boundaries) + - IncQL RFC 000 (language specification — naming, schema shapes, layer boundaries) - Incan compiler — static capability gating enforcement: [incan#187](https://github.com/encero-systems/incan/issues/187) - - InQL follow-up when enforcement lands: [InQL #10](https://github.com/encero-systems/InQL/issues/10) - - InQL aggregate helper semantics follow-up: [InQL #23](https://github.com/encero-systems/InQL/issues/23) -- **Issue:** [InQL #2](https://github.com/encero-systems/InQL/issues/2) + - IncQL follow-up when enforcement lands: [IncQL #10](https://github.com/encero-systems/IncQL/issues/10) + - IncQL aggregate helper semantics follow-up: [IncQL #23](https://github.com/encero-systems/IncQL/issues/23) +- **Issue:** [IncQL #2](https://github.com/encero-systems/IncQL/issues/2) - **RFC PR:** - - **Written against:** Incan v0.2 - **Shipped in:** — ## Summary -This RFC specifies the **dataset type hierarchy** for InQL: the traits and concrete types that carry schema-parameterized tabular data through relational pipelines. The hierarchy is rooted in the **`DataSet[T]`** trait, split into **`BoundedDataSet[T]`** (finite extent) and **`UnboundedDataSet[T]`** (streaming/unbounded), with three concrete types: **`DataFrame[T]`** (materialized/eager), **`LazyFrame[T]`** (deferred plan), and **`DataStream[T]`** (streaming). The bounded/unbounded split enables **static capability gating**: operations that require unbounded state are rejected at compile time when the target is unbounded, without requiring a separate streaming API. This RFC also defines the **relational operation API** on `DataSet[T]` and the **execution backend boundary** so implementations can delegate without exposing engine internals as the author contract. +This RFC specifies the **dataset type hierarchy** for IncQL: the traits and concrete types that carry schema-parameterized tabular data through relational pipelines. The hierarchy is rooted in the **`DataSet[T]`** trait, split into **`BoundedDataSet[T]`** (finite extent) and **`UnboundedDataSet[T]`** (streaming/unbounded), with three concrete types: **`DataFrame[T]`** (materialized/eager), **`LazyFrame[T]`** (deferred plan), and **`DataStream[T]`** (streaming). The bounded/unbounded split enables **static capability gating**: operations that require unbounded state are rejected at compile time when the target is unbounded, without requiring a separate streaming API. This RFC also defines the **relational operation API** on `DataSet[T]` and the **execution backend boundary** so implementations can delegate without exposing engine internals as the author contract. ## Core model @@ -43,20 +43,20 @@ Typed pipelines need a first-class carrier for columnar data indexed by `T`. Wit ## Non-Goals -- Normative naming rules (four naming forms, current query schema, resolution order) — InQL RFC 000. -- Apache Substrait `Rel`-level mapping and extension policy — InQL RFC 002. -- Clause-based relational grammar, aggregate rules, Substrait lowering from that surface — InQL RFC 003. -- Execution context, session, DataFusion — InQL RFC 004. -- Pipe-forward (`|>`) grammar — InQL RFC 005 (not in v0.1 scope). +- Normative naming rules (four naming forms, current query schema, resolution order) — IncQL RFC 000. +- Apache Substrait `Rel`-level mapping and extension policy — IncQL RFC 002. +- Clause-based relational grammar, aggregate rules, Substrait lowering from that surface — IncQL RFC 003. +- Execution context, session, DataFusion — IncQL RFC 004. +- Pipe-forward (`|>`) grammar — IncQL RFC 005 (not in v0.1 scope). - Cluster-scale scheduling, shuffle, distributed fault tolerance — orchestration layer. - Drop-in API compatibility with Apache Beam, Flink, or Spark SDKs. ## Guide-level explanation -Authors import dataset types from the InQL package and parameterize with a `model`: +Authors import dataset types from the IncQL package and parameterize with a `model`: ```incan -from pub::inql import LazyFrame +from pub::incql import LazyFrame from models import Order def load_orders() -> LazyFrame[Order]: @@ -66,7 +66,7 @@ def load_orders() -> LazyFrame[Order]: They compose data using methods exposed through the `DataSet[T]` trait: ```incan -from pub::inql import LazyFrame +from pub::incql import LazyFrame from models import Order def high_value_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -76,8 +76,8 @@ def high_value_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: Authors can derive computed columns through `with_column(...)`: ```incan -from pub::inql import LazyFrame -from pub::inql.functions import col, int_expr, mul +from pub::incql import LazyFrame +from pub::incql.functions import col, int_expr, mul from models import Order def enrich_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -87,7 +87,7 @@ def enrich_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: Because `DataStream[T]` shares the same operation API, streaming code looks identical — only the type signature changes: ```incan -from pub::inql import DataStream +from pub::incql import DataStream from models import Event def important_events(events: DataStream[Event]) -> DataStream[Event]: @@ -99,7 +99,7 @@ def important_events(events: DataStream[Event]) -> DataStream[Event]: The trait hierarchy gives authors three levels of specificity: ```incan -from pub::inql import DataSet, BoundedDataSet, UnboundedDataSet +from pub::incql import DataSet, BoundedDataSet, UnboundedDataSet from models import Order, Event # Accepts any carrier — generic utilities @@ -118,7 +118,7 @@ def write_to_kafka(events: UnboundedDataSet[Event]) -> None: And two levels of concrete-type specificity: ```incan -from pub::inql import DataFrame, LazyFrame, DataStream +from pub::incql import DataFrame, LazyFrame, DataStream from models import Order, Summary, Event, Alert # Materialized data in hand @@ -139,7 +139,7 @@ def process_stream(events: DataStream[Event]) -> DataStream[Alert]: ### Packaging - The dataset types and traits in this RFC **must** be exposed from a buildable Incan library package with public exports. -- This RFC **may** require vocabulary only for symbols strictly needed for the dataset API surface; vocabulary for other InQL authoring surfaces is a separate concern. +- This RFC **may** require vocabulary only for symbols strictly needed for the dataset API surface; vocabulary for other IncQL authoring surfaces is a separate concern. ### Type hierarchy @@ -175,7 +175,7 @@ For `UnboundedDataSet[T]`, the governing rule is semantic rather than ad hoc: op ### Operation API (for lowering and direct use) -The InQL library **must** expose the following instance methods on `DataSet[T]` (exact signatures may live in companion library docs; semantics **must** match this table and stay consistent with any normative lowering rules for the same logical operators elsewhere in InQL). Method names are illustrative; implementations **may** use equivalent spellings if the compiler maps them consistently. +The IncQL library **must** expose the following instance methods on `DataSet[T]` (exact signatures may live in companion library docs; semantics **must** match this table and stay consistent with any normative lowering rules for the same logical operators elsewhere in IncQL). Method names are illustrative; implementations **may** use equivalent spellings if the compiler maps them consistently. | Method | Role | | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -193,16 +193,16 @@ Additional requirements: - Operations **must** preserve or update `T` (or output model `U`) in a way the typechecker can verify. - Operations that are statically invalid on `UnboundedDataSet[T]` (e.g. unbounded-state operations) **must** produce compile-time errors, not runtime failures. -- Aggregate helpers used with `.agg(...)` are imported library symbols (from `pub::inql.functions`), not ambient builtins. +- Aggregate helpers used with `.agg(...)` are imported library symbols (from `pub::incql.functions`), not ambient builtins. - The minimum required aggregate-helper surface for the current package slice is `col`, `sum`, and `count`. -- The current InQL-only implementation uses `col(...)` builders as the semantic target that later `.column` sugar and query-block lowering should compile to. -- The current InQL-only projection implementation uses `with_column(name, expr)` plus projection builders such as `add(...)`, `mul(...)`, and `int_expr(...)` as the semantic target that later projection sugar should compile to. -- This RFC defines the minimum required aggregate-function import model for `.agg(...)`; it is not an exhaustive catalog of all present or future InQL functions. Additional functions **may** be added later through additive library evolution or follow-up RFCs, provided they do not change the semantics of the required set defined by the InQL RFC suite. +- The current IncQL-only implementation uses `col(...)` builders as the semantic target that later `.column` sugar and query-block lowering should compile to. +- The current IncQL-only projection implementation uses `with_column(name, expr)` plus projection builders such as `add(...)`, `mul(...)`, and `int_expr(...)` as the semantic target that later projection sugar should compile to. +- This RFC defines the minimum required aggregate-function import model for `.agg(...)`; it is not an exhaustive catalog of all present or future IncQL functions. Additional functions **may** be added later through additive library evolution or follow-up RFCs, provided they do not change the semantics of the required set defined by the IncQL RFC suite. ### Execution backend boundary - Implementations **must** separate the author-facing `DataSet` API from engine-specific code (Rust crates, Substrait consumers, etc.). -- Substrait consumption or emission at the collection/plan layer **may** be specified here as optional; the Substrait contract (InQL RFC 002) governs plan semantics. If more than one relational authoring surface emits Substrait, they **must not** produce contradictory plans for the same logical pipeline. +- Substrait consumption or emission at the collection/plan layer **may** be specified here as optional; the Substrait contract (IncQL RFC 002) governs plan semantics. If more than one relational authoring surface emits Substrait, they **must not** produce contradictory plans for the same logical pipeline. - The execution context owns the session, plan optimization, and concrete execution backend (DataFusion as reference implementation). - Materialization helpers such as `collect(data)` or `display(data)` belong to the execution context and concrete implementation model, not to the `DataSet[T]` trait surface defined in this RFC. @@ -219,7 +219,7 @@ The design draws on Spark Structured Streaming's core insight: a stream is an un ### Trait naming -- **`DataSet[T]`** is InQL's root trait for any schema-parameterized relational carrier. It is intentionally aligned with the Spark notion of a typed `Dataset`, but spelled `DataSet` for Incan style. +- **`DataSet[T]`** is IncQL's root trait for any schema-parameterized relational carrier. It is intentionally aligned with the Spark notion of a typed `Dataset`, but spelled `DataSet` for Incan style. - **`DataFrame[T]`** is a concrete eager kind, not Spark's untyped `DataFrame = Dataset[Row]` alias. - **`BoundedDataSet[T]`** and **`UnboundedDataSet[T]`** are intermediate traits that give clean type signatures for batch-only and streaming-only consumers respectively. @@ -247,7 +247,7 @@ Future RFCs **may** add methods on `BoundedDataSet[T]` or `UnboundedDataSet[T]`, ## Layers affected -- **InQL library** (primary): types, traits, Rust companion / interop. +- **IncQL library** (primary): types, traits, Rust companion / interop. - **Typechecker**: generics for `DataFrame[T]` etc.; static streaming constraint checks for `UnboundedDataSet[T]`; capability gating based on trait bounds. - **Parser**: only if dataset API introduces new surface syntax beyond ordinary calls. @@ -257,6 +257,6 @@ Future RFCs **may** add methods on `BoundedDataSet[T]` or `UnboundedDataSet[T]`, - **`UnboundedDataSet[T]` restrictions:** Operations requiring end-of-input semantics or unbounded retained state are not valid unless a later RFC gives them bounded-state semantics. In v0.1, disallowed examples include global `order_by`, global `limit`, unwindowed `group_by` / `agg`, eager materialization to a finite `DataFrame[T]`, and finite file writes. -- **`collect` / `display`:** Not part of the `DataSet[T]` trait surface. Helpers such as `collect(data)` or `display(data)` belong to the execution context and concrete implementation model defined in InQL RFC 004, not in this RFC. +- **`collect` / `display`:** Not part of the `DataSet[T]` trait surface. Helpers such as `collect(data)` or `display(data)` belong to the execution context and concrete implementation model defined in IncQL RFC 004, not in this RFC. - **Intermediate traits:** `BoundedDataSet[T]` and `UnboundedDataSet[T]` do not add required core relational methods in v0.1. Future RFCs may add additional methods only where the semantics are inherently boundedness-specific and remain backend-neutral. diff --git a/docs/rfcs/002_apache_substrait_integration.md b/docs/rfcs/002_apache_substrait_integration.md index 70c06e94..c9c7edc7 100644 --- a/docs/rfcs/002_apache_substrait_integration.md +++ b/docs/rfcs/002_apache_substrait_integration.md @@ -1,51 +1,51 @@ -# InQL RFC 002: Apache Substrait integration +# IncQL RFC 002: Apache Substrait integration - **Status:** In Progress - **Created:** 2026-03-23 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 000 (language specification — naming, schema shapes, compilation model) - - InQL RFC 001 (dataset types — `DataSet[T]` carriers and schema parameter) -- **Issue:** [InQL #3](https://github.com/encero-systems/InQL/issues/3) + - IncQL RFC 000 (language specification — naming, schema shapes, compilation model) + - IncQL RFC 001 (dataset types — `DataSet[T]` carriers and schema parameter) +- **Issue:** [IncQL #3](https://github.com/encero-systems/IncQL/issues/3) - **RFC PR:** - - **Written against:** Incan v0.2 - **Shipped in:** — ## Summary -This RFC defines **Apache Substrait** as the **normative logical interchange** for InQL relational plans: which **`Rel` and expression** shapes implementations produce, how **read roots** remain **backend-agnostic** while **environment binding** (adapters, credentials, runner choice) stays **outside** InQL, and how **extensions** cover capabilities that lack a stable logical `Rel` in core Substrait. The `query {}` surface requires lowering to Substrait; this RFC owns the **cross-surface contract** so method-chain APIs (InQL RFC 001), `query {}` blocks, and optional pipe-forward do not diverge at emission time. +This RFC defines **Apache Substrait** as the **normative logical interchange** for IncQL relational plans: which **`Rel` and expression** shapes implementations produce, how **read roots** remain **backend-agnostic** while **environment binding** (adapters, credentials, runner choice) stays **outside** IncQL, and how **extensions** cover capabilities that lack a stable logical `Rel` in core Substrait. The `query {}` surface requires lowering to Substrait; this RFC owns the **cross-surface contract** so method-chain APIs (IncQL RFC 001), `query {}` blocks, and optional pipe-forward do not diverge at emission time. ## Core model -1. A **checked** InQL relational tree **must** be expressible as a Substrait **`Plan`** whose executable root is a **`Rel`** tree, optionally a **DAG** via **`ReferenceRel`** when subplans are shared. +1. A **checked** IncQL relational tree **must** be expressible as a Substrait **`Plan`** whose executable root is a **`Rel`** tree, optionally a **DAG** via **`ReferenceRel`** when subplans are shared. 2. **Logical reads** are **`ReadRel`** (or extension leaf relations) carrying **names, virtual rows, or extension payloads** instead of host-specific connection strings or secrets in the normative interchange. 3. **Scalar and aggregate** computation uses Substrait **expressions** and **aggregate functions**; functions outside the pinned core set **must** use **registered extension URIs** documented with the compiler. -4. **North-star operator catalog**: InQL capabilities map to logical `Rel` kinds as specified in the [Substrait operator catalog reference][ref-operator-catalog]; implementation subsets are delivery choices but **must not** contradict this RFC for operators they expose. +4. **North-star operator catalog**: IncQL capabilities map to logical `Rel` kinds as specified in the [Substrait operator catalog reference][ref-operator-catalog]; implementation subsets are delivery choices but **must not** contradict this RFC for operators they expose. ## Motivation -Without a dedicated specification, Substrait lowering risks drifting between front-ends (`query {}`, APIs on `DataSet[T]`) and emitters, and risks smuggling execution concerns (storage URIs, credentials, engine choice) into the query IR. Substrait is the ecosystem's portable relational algebra serialization; InQL needs a single `Rel`-level contract, version pinning rules, and an explicit boundary between plan semantics and operational binding. +Without a dedicated specification, Substrait lowering risks drifting between front-ends (`query {}`, APIs on `DataSet[T]`) and emitters, and risks smuggling execution concerns (storage URIs, credentials, engine choice) into the query IR. Substrait is the ecosystem's portable relational algebra serialization; IncQL needs a single `Rel`-level contract, version pinning rules, and an explicit boundary between plan semantics and operational binding. ## Goals - Require that conforming implementations **emit Substrait** for relational features they claim to support, using logical `Rel` nodes unless a documented extension applies. -- Publish a **versioned mapping catalog** from InQL plan concepts to Substrait logical relations and expression patterns, marking **core spec**, **extension**, or **documented expansion / gap**. -- Specify **read roots**: logical `ReadRel` shapes **in** InQL vs **adapter resolution** in the host execution environment. +- Publish a **versioned mapping catalog** from IncQL plan concepts to Substrait logical relations and expression patterns, marking **core spec**, **extension**, or **documented expansion / gap**. +- Specify **read roots**: logical `ReadRel` shapes **in** IncQL vs **adapter resolution** in the host execution environment. - Require **documented pinning** of Substrait revision and of any bundled extension function sets shipped with the toolchain. -- List **known gaps** (unnest, pivot, advanced joins, streaming-specific semantics) without blocking InQL RFC 003. +- List **known gaps** (unnest, pivot, advanced joins, streaming-specific semantics) without blocking IncQL RFC 003. ## Non-Goals -- Defining orchestration, workflow, or adapter authoring syntax — out of scope; only binding boundaries relative to InQL plans are stated here. -- Mandating a default Substrait consumer (specific engine or library) — implementation detail; InQL RFC 004 names the reference backend. -- Physical Substrait relations as a normative InQL output — consumers **may** use them; InQL **may** emit them when documented as a non-portable or target-specific mode. +- Defining orchestration, workflow, or adapter authoring syntax — out of scope; only binding boundaries relative to IncQL plans are stated here. +- Mandating a default Substrait consumer (specific engine or library) — implementation detail; IncQL RFC 004 names the reference backend. +- Physical Substrait relations as a normative IncQL output — consumers **may** use them; IncQL **may** emit them when documented as a non-portable or target-specific mode. - ANSI SQL completeness — mapping is capability-based, not a SQL compliance checklist. -## Current implementation profile (InQL package path) +## Current implementation profile (IncQL package path) -The current implementation profile for this RFC is explicitly scoped to InQL package code (`.incn`) and is the contract for current delivery tracking. +The current implementation profile for this RFC is explicitly scoped to IncQL package code (`.incn`) and is the contract for current delivery tracking. -- Core read/query `Rel` coverage is implemented through a thin proto-backed Substrait boundary in InQL package code. +- Core read/query `Rel` coverage is implemented through a thin proto-backed Substrait boundary in IncQL package code. - Optional mutation relations remain modeled but are not required to be executable in the current read/query analytical core. - Gap and extension semantics are represented as typed contracts in package code and conformance scenarios, rather than ad hoc string payloads. - Richer planning semantics remain outside this profile when they logically belong to future `query {}` lowering or Prism. @@ -73,9 +73,9 @@ This profile is reflected by: ## Guide-level explanation -Authors build `DataSet[T]` values (InQL RFC 001) using `query {}` or relational method chains. After typechecking, the relational work becomes a **Substrait plan**: mostly `FilterRel`, `ProjectRel`, `JoinRel`, `AggregateRel`, and so on, rooted in a `ReadRel` when new data enters the plan. +Authors build `DataSet[T]` values (IncQL RFC 001) using `query {}` or relational method chains. After typechecking, the relational work becomes a **Substrait plan**: mostly `FilterRel`, `ProjectRel`, `JoinRel`, `AggregateRel`, and so on, rooted in a `ReadRel` when new data enters the plan. -When a plan says "read this named relation" or "read this logical asset id," the plan carries the **logical** identity. The **execution context** resolves that identity to concrete storage, applies policy, and supplies credentials. That split keeps InQL portable and keeps governance-sensitive details out of the serialized plan's normative story. +When a plan says "read this named relation" or "read this logical asset id," the plan carries the **logical** identity. The **execution context** resolves that identity to concrete storage, applies policy, and supplies credentials. That split keeps IncQL portable and keeps governance-sensitive details out of the serialized plan's normative story. ## Reference-level explanation @@ -83,15 +83,15 @@ When a plan says "read this named relation" or "read this logical asset id," the - Implementations **must** be able to produce a Substrait `Plan` for every relational operator they expose that is claimed portable in documentation. - Lowering semantics **must** be identical whether the surface is `query {}`, trait methods, or desugared pipe-forward, for the same checked tree. -- Implementations **may** additionally lower to InQL RFC 001 operations for execution; if both paths exist, they **must** match the Substrait semantics for those operators. +- Implementations **may** additionally lower to IncQL RFC 001 operations for execution; if both paths exist, they **must** match the Substrait semantics for those operators. For the full capability → `Rel` mapping, profile classifications, and gap encoding requirements, see the [Substrait operator catalog reference][ref-operator-catalog]. -Conformance scenarios **should** use stable scenario IDs and typed InQL model contracts as defined by the [Substrait conformance corpus reference][ref-conformance-corpus], so implementation and CI reporting can track portability status consistently across toolchains. +Conformance scenarios **should** use stable scenario IDs and typed IncQL model contracts as defined by the [Substrait conformance corpus reference][ref-conformance-corpus], so implementation and CI reporting can track portability status consistently across toolchains. ### Logical `Rel` alphabet -The following are the primary logical relations InQL targets. Exact protobuf message paths follow the pinned Substrait version selected by the toolchain for a given release. +The following are the primary logical relations IncQL targets. Exact protobuf message paths follow the pinned Substrait version selected by the toolchain for a given release. | Substrait `Rel` | Role | | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | @@ -112,15 +112,15 @@ The following are the primary logical relations InQL targets. Exact protobuf mes ### North-star catalog -InQL defines a north-star operator catalog that maps every InQL plan capability to a Substrait `Rel` or expression pattern, and classifies each capability as one of: **core** (maps to a standard logical `Rel` in the pinned revision), **extension** (requires a registered extension URI), **gap** (no stable logical `Rel` in core Substrait; encoding must be documented), or **optional-mutation** (not required for read/query analytical core). +IncQL defines a north-star operator catalog that maps every IncQL plan capability to a Substrait `Rel` or expression pattern, and classifies each capability as one of: **core** (maps to a standard logical `Rel` in the pinned revision), **extension** (requires a registered extension URI), **gap** (no stable logical `Rel` in core Substrait; encoding must be documented), or **optional-mutation** (not required for read/query analytical core). The full, versioned catalog — including all profile tags, gap encoding requirements, and mutation-profile operators — lives in the [Substrait operator catalog reference][ref-operator-catalog]. Conforming implementations **must** follow the mappings listed there for any capability they claim portable. ### Read roots vs binding -- InQL **must** express new data entering a plan as logical reads: names, virtual values, or opaque extension table types that still serialize as Substrait `ReadRel` (or an extension leaf) **without** normative dependence on secret material in the plan text. +- IncQL **must** express new data entering a plan as logical reads: names, virtual values, or opaque extension table types that still serialize as Substrait `ReadRel` (or an extension leaf) **without** normative dependence on secret material in the plan text. - The execution context **must** resolve logical reads to physical resources through its adapter and execution layer; that layer **must not** redefine relational semantics of the plan. -- Product SDKs **may** present a unified import surface; adapter-specific "open connection" APIs **should not** be specified as core InQL — they remain thin wrappers at most. +- Product SDKs **may** present a unified import surface; adapter-specific "open connection" APIs **should not** be specified as core IncQL — they remain thin wrappers at most. For the full `ReadRel` variant reference, the detailed execution context obligations, and the adapter boundary contract, see the [read-root and binding contract reference][ref-read-root]. @@ -133,7 +133,7 @@ For revision pin requirements, URI registration policy, bundle naming, compatibi ### Optional mutation profile progress -- InQL **may** expose `WriteRel`, `DdlRel`, or `UpdateRel` for warehouse-style mutation. Absence of these in a given distribution **does not** make InQL incomplete for read-only analytical use. +- IncQL **may** expose `WriteRel`, `DdlRel`, or `UpdateRel` for warehouse-style mutation. Absence of these in a given distribution **does not** make IncQL incomplete for read-only analytical use. The optional mutation profile operators, per-operator portability notes, and support expectations are listed in the [Substrait operator catalog reference][ref-operator-catalog]. @@ -143,7 +143,7 @@ The following reference documents expand on the operational detail that is too l | Document | What it covers | | --- | --- | -| [Substrait operator catalog][ref-operator-catalog] | Full InQL capability → `Rel` mapping; profile tags; gap encoding rules; mutation profile operators | +| [Substrait operator catalog][ref-operator-catalog] | Full IncQL capability → `Rel` mapping; profile tags; gap encoding rules; mutation profile operators | | [Substrait revision and extension policy][ref-revision-policy] | Revision pin requirements; extension URI registration policy; compatibility conventions; release-note checklist | | [Substrait read-root and binding contract][ref-read-root] | `ReadRel` variant reference; execution context obligations; adapter boundary | | [Substrait conformance corpus][ref-conformance-corpus] | Canonical corpus structure, scenario metadata schema, profile taxonomy, and stable scenario ID conventions | @@ -152,7 +152,7 @@ The following reference documents expand on the operational detail that is too l ### Interaction with Incan -- Field references and types **must** align with `model`-backed schemas (InQL RFC 001) and lower to Substrait types and field indices consistent with the emitted `NamedStruct`. +- Field references and types **must** align with `model`-backed schemas (IncQL RFC 001) and lower to Substrait types and field indices consistent with the emitted `NamedStruct`. ### Compatibility @@ -167,7 +167,7 @@ The following reference documents expand on the operational detail that is too l ## Drawbacks - Substrait lags some front-end expressiveness; extensions and rewrites add maintenance. -- Dual lowering (InQL RFC 001 APIs + Substrait) increases test surface unless one path is canonical in practice. +- Dual lowering (IncQL RFC 001 APIs + Substrait) increases test surface unless one path is canonical in practice. - Producer / consumer version skew requires disciplined pinning and clear compatibility statements. ## Implementation architecture @@ -185,7 +185,7 @@ Non-normative: toolchains **should** maintain golden Substrait plans or equivale ### Phase 1: Spec and operator catalog - Lock down the versioned Substrait revision pinning policy in compiler documentation and release artifacts. -- Publish the normative operator catalog mapping InQL capabilities to Substrait `Rel` kinds, including gap annotations for unnest, pivot, and streaming semantics. +- Publish the normative operator catalog mapping IncQL capabilities to Substrait `Rel` kinds, including gap annotations for unnest, pivot, and streaming semantics. - Document extension URI registration conventions in the public toolchain catalog. ### Phase 2: IR lowering — core boundary @@ -260,10 +260,10 @@ Non-normative: toolchains **should** maintain golden Substrait plans or equivale ## Design Decisions -- **Substrait revision pinning:** this RFC defines the pinning policy, not one timeless revision number. Each conforming InQL toolchain release **must** publish the exact Substrait revision it targets and any bundled extension sets in public release artifacts and compiler documentation. -- **Canonical unnest / explode encoding:** until core Substrait standardizes a portable unnest relation that InQL adopts, `EXPLODE`-style behavior **must** lower through a documented extension relation or another documented non-core encoding listed in the toolchain's public operator catalog. Implementations **must not** present ad hoc or undocumented encodings as portable core behavior. -- **Mutation relations:** `WriteRel`, `DdlRel`, and `UpdateRel` remain an optional mutation profile. They are not part of the minimum read/query analytical core required for InQL v0.1, and implementations **may** expose them only when the execution context and backend support them. -- **Correlated subqueries:** InQL v0.1 does not standardize a single correlated-subquery desugaring because correlated subquery surface syntax is not part of the minimum relational grammar. If a future RFC adds correlated subqueries, that RFC **must** define the lowering contract explicitly rather than relying on implicit emitter policy. +- **Substrait revision pinning:** this RFC defines the pinning policy, not one timeless revision number. Each conforming IncQL toolchain release **must** publish the exact Substrait revision it targets and any bundled extension sets in public release artifacts and compiler documentation. +- **Canonical unnest / explode encoding:** until core Substrait standardizes a portable unnest relation that IncQL adopts, `EXPLODE`-style behavior **must** lower through a documented extension relation or another documented non-core encoding listed in the toolchain's public operator catalog. Implementations **must not** present ad hoc or undocumented encodings as portable core behavior. +- **Mutation relations:** `WriteRel`, `DdlRel`, and `UpdateRel` remain an optional mutation profile. They are not part of the minimum read/query analytical core required for IncQL v0.1, and implementations **may** expose them only when the execution context and backend support them. +- **Correlated subqueries:** IncQL v0.1 does not standardize a single correlated-subquery desugaring because correlated subquery surface syntax is not part of the minimum relational grammar. If a future RFC adds correlated subqueries, that RFC **must** define the lowering contract explicitly rather than relying on implicit emitter policy. diff --git a/docs/rfcs/003_inql_query_blocks.md b/docs/rfcs/003_incql_query_blocks.md similarity index 65% rename from docs/rfcs/003_inql_query_blocks.md rename to docs/rfcs/003_incql_query_blocks.md index 51b6a89a..65c24111 100644 --- a/docs/rfcs/003_inql_query_blocks.md +++ b/docs/rfcs/003_incql_query_blocks.md @@ -1,45 +1,45 @@ -# InQL RFC 003: `query {}` blocks — syntax, typing, Substrait +# IncQL RFC 003: `query {}` blocks — syntax, typing, Substrait - **Status:** Implemented - **Created:** 2026-03-22 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 000 (language specification — naming and query schema; **must** stay consistent) - - InQL RFC 001 (dataset types — **prerequisite**; `FROM` sources must conform to `DataSet[T]`) - - InQL RFC 002 (Apache Substrait — **normative `Rel`-level contract** for lowering) -- **Issue:** [InQL #4](https://github.com/encero-systems/InQL/issues/4) -- **RFC PR:** [InQL #59](https://github.com/encero-systems/InQL/pull/59) + - IncQL RFC 000 (language specification — naming and query schema; **must** stay consistent) + - IncQL RFC 001 (dataset types — **prerequisite**; `FROM` sources must conform to `DataSet[T]`) + - IncQL RFC 002 (Apache Substrait — **normative `Rel`-level contract** for lowering) +- **Issue:** [IncQL #4](https://github.com/encero-systems/IncQL/issues/4) +- **RFC PR:** [IncQL #59](https://github.com/encero-systems/IncQL/pull/59) - **Written against:** Incan v0.3 -- **Shipped in:** InQL v0.1 +- **Shipped in:** IncQL v0.1 ## Summary -This RFC specifies the **`query { ... }`** expression: grammar, typechecking (including clause-level use of `.column`, `relation.column`, bare identifiers, and aggregate rules), vocabulary activation for the `query` keyword (InQL package as dependency), and lowering to Apache Substrait. InQL also accepts the expression-position colon spelling `query:` for consistency with Incan vocabulary declarations. Naming-form semantics and current query schema are defined in InQL RFC 000; this RFC **must** remain consistent with that document. It depends on InQL RFC 001: `FROM` sources **must** conform to InQL RFC 001's `DataSet[T]` trait (`DataFrame[T]`, `LazyFrame[T]`, or `DataStream[T]`) so that `T` supplies fields for resolution. InQL RFC 002 owns the Substrait `Rel` and expression contract, mapping catalog, and read vs binding boundaries; this RFC **must** conform to InQL RFC 002 for serialized plan semantics. `SELECT DISTINCT` is part of the minimum clause surface defined here. +This RFC specifies the **`query { ... }`** expression: grammar, typechecking (including clause-level use of `.column`, `relation.column`, bare identifiers, and aggregate rules), vocabulary activation for the `query` keyword (IncQL package as dependency), and lowering to Apache Substrait. IncQL also accepts the expression-position colon spelling `query:` for consistency with Incan vocabulary declarations. Naming-form semantics and current query schema are defined in IncQL RFC 000; this RFC **must** remain consistent with that document. It depends on IncQL RFC 001: `FROM` sources **must** conform to IncQL RFC 001's `DataSet[T]` trait (`DataFrame[T]`, `LazyFrame[T]`, or `DataStream[T]`) so that `T` supplies fields for resolution. IncQL RFC 002 owns the Substrait `Rel` and expression contract, mapping catalog, and read vs binding boundaries; this RFC **must** conform to IncQL RFC 002 for serialized plan semantics. `SELECT DISTINCT` is part of the minimum clause surface defined here. ## Motivation -A SQL-familiar surface inside Incan improves readability and enables compile-time validation of relational work against `model` schemas. `query {}` is the checked authoring form; it lowers to operations on `DataSet` values (InQL RFC 001) and/or directly to Substrait (InQL RFC 002) for portability. +A SQL-familiar surface inside Incan improves readability and enables compile-time validation of relational work against `model` schemas. `query {}` is the checked authoring form; it lowers to operations on `DataSet` values (IncQL RFC 001) and/or directly to Substrait (IncQL RFC 002) for portability. ## Goals - Specify `query { ... }` as a single expression whose body is an ordered sequence of clauses with unambiguous grammar. - Typecheck relational positions: `.column` on the primary `FROM` relation; `relation.column` for joins; bare names per schema-first rules below. -- Lower a checked `query {}` tree to Apache Substrait as the normative interchange for supported operators; document gaps, extensions, and unsupported nodes. Lowering **must** conform to InQL RFC 002. -- Integrate with InQL RFC 001 so `FROM ` resolves to a type conforming to `DataSet[T]` with known `T`. +- Lower a checked `query {}` tree to Apache Substrait as the normative interchange for supported operators; document gaps, extensions, and unsupported nodes. Lowering **must** conform to IncQL RFC 002. +- Integrate with IncQL RFC 001 so `FROM ` resolves to a type conforming to `DataSet[T]` with known `T`. - Enable end-to-end batch execution: `query {}` → Substrait → execution context. ## Non-Goals -- Defining the `DataSet[T]` trait, `DataFrame` / `LazyFrame` / `DataStream` types, or their method APIs — InQL RFC 001. -- Pipe-forward (`|>`) relational syntax — InQL RFC 005 (must stay consistent with InQL RFC 000). -- Substrait `Rel`-level mapping catalog, extension URI policy, read-root vs binding boundaries — InQL RFC 002. -- Execution context, session, DataFusion — InQL RFC 004. -- Cluster execution — out of scope for InQL. +- Defining the `DataSet[T]` trait, `DataFrame` / `LazyFrame` / `DataStream` types, or their method APIs — IncQL RFC 001. +- Pipe-forward (`|>`) relational syntax — IncQL RFC 005 (must stay consistent with IncQL RFC 000). +- Substrait `Rel`-level mapping catalog, extension URI policy, read-root vs binding boundaries — IncQL RFC 002. +- Execution context, session, DataFusion — IncQL RFC 004. +- Cluster execution — out of scope for IncQL. ## Guide-level explanation ```incan -from pub::inql import DataFrame, avg, count, desc, sum +from pub::incql import DataFrame, avg, count, desc, sum from models import Order, OrderSummary def summarize_orders(orders: DataFrame[Order]) -> DataFrame[OrderSummary]: @@ -56,20 +56,20 @@ def summarize_orders(orders: DataFrame[Order]) -> DataFrame[OrderSummary]: } ``` -The compiler checks `.status`, `.amount`, and `GROUP BY` / `SELECT` consistency. The `DataFrame[OrderSummary]` return type records the intended output row model; full field/type compatibility validation against annotated output models is tracked as schema-validation follow-up work. The checked tree lowers to Substrait (InQL RFC 002); execution uses the execution context. +The compiler checks `.status`, `.amount`, and `GROUP BY` / `SELECT` consistency. The `DataFrame[OrderSummary]` return type records the intended output row model; full field/type compatibility validation against annotated output models is tracked as schema-validation follow-up work. The checked tree lowers to Substrait (IncQL RFC 002); execution uses the execution context. ## Reference-level explanation ### Packaging and activation -- Projects that depend on InQL **must** obtain `query` through library-driven vocabulary activation in the host compiler. -- A compilation unit with InQL active **must** parse `query { ... }` as specified here. It **may** also accept +- Projects that depend on IncQL **must** obtain `query` through library-driven vocabulary activation in the host compiler. +- A compilation unit with IncQL active **must** parse `query { ... }` as specified here. It **may** also accept expression-position `query:` as an equivalent spelling. -- Aggregate helpers such as `count`, `sum`, `avg`, `min`, and `max` are library symbols, instead of ambient builtins. Examples in this RFC import them from the `pub::inql` facade; implementations **must** provide an equivalent importable surface for aggregate functions used in relational expressions. +- Aggregate helpers such as `count`, `sum`, `avg`, `min`, and `max` are library symbols, instead of ambient builtins. Examples in this RFC import them from the `pub::incql` facade; implementations **must** provide an equivalent importable surface for aggregate functions used in relational expressions. -### `FROM` and relation to InQL RFC 001 +### `FROM` and relation to IncQL RFC 001 -- `FROM ` establishes the primary relation for `.column`. `` **must** typecheck as a type conforming to `DataSet[T]` per InQL RFC 001 (`DataFrame[T]`, `LazyFrame[T]`, or `DataStream[T]`). +- `FROM ` establishes the primary relation for `.column`. `` **must** typecheck as a type conforming to `DataSet[T]` per IncQL RFC 001 (`DataFrame[T]`, `LazyFrame[T]`, or `DataStream[T]`). - `T` **must** supply fields for `.name` lookup. ### Primary relation and joins @@ -79,7 +79,7 @@ The compiler checks `.status`, `.amount`, and `GROUP BY` / `SELECT` consistency. ### Current query schema and bare identifiers -Naming forms, schema evolution, and resolution precedence — InQL RFC 000. This subsection lists relational expression positions for this RFC only. +Naming forms, schema evolution, and resolution precedence — IncQL RFC 000. This subsection lists relational expression positions for this RFC only. Inside relational expression positions (`WHERE`, `JOIN ON`, `GROUP BY`, `ORDER BY`, `SELECT`, window specs): @@ -89,7 +89,7 @@ Inside relational expression positions (`WHERE`, `JOIN ON`, `GROUP BY`, `ORDER B ### Expression operators -Relational expression bodies use ordinary Incan expression operators and lower them into InQL's public helper surface. Implementations must treat `left == right`, `left != right`, `left < right`, `left <= right`, `left > right`, and `left >= right` as equivalent to `eq(left, right)`, `ne(left, right)`, `lt(left, right)`, `lte(left, right)`, `gt(left, right)`, and `gte(left, right)` respectively. Arithmetic operators lower through `add`, `sub`, `mul`, `div`, and `modulo`; boolean and unary operators lower through their helper equivalents such as `and_`, `or_`, `not_`, and `neg`. +Relational expression bodies use ordinary Incan expression operators and lower them into IncQL's public helper surface. Implementations must treat `left == right`, `left != right`, `left < right`, `left <= right`, `left > right`, and `left >= right` as equivalent to `eq(left, right)`, `ne(left, right)`, `lt(left, right)`, `lte(left, right)`, `gt(left, right)`, and `gte(left, right)` respectively. Arithmetic operators lower through `add`, `sub`, `mul`, `div`, and `modulo`; boolean and unary operators lower through their helper equivalents such as `and_`, `or_`, `not_`, and `neg`. Inclusive comparison helpers are named `lte` and `gte`; `le` and `ge` are not part of the public helper surface. Single `=` is not a predicate equality operator in query expressions. Equality uses `==`; `=` remains reserved for assignment/binding positions such as named window declarations. @@ -101,9 +101,9 @@ Inclusive comparison helpers are named `lte` and `gte`; `le` and `ge` are not pa ### Aggregates - Under `GROUP BY`, `SELECT` references **must** be grouped or aggregated; illegal mixing **must** error. -- Aggregate function calls in relational expressions **must** resolve through imported library symbols (for example `from pub::inql import count, sum, avg`). The compiler **must not** treat `count`, `sum`, `avg`, `min`, or `max` as implicitly in scope ambient names. -- This RFC defines the minimum required aggregate-function surface and import model for `query {}`; it is not an exhaustive catalog of all InQL functions. Additional functions require additive library evolution or follow-up RFCs that do not change the semantics of the required set defined here. -- `SELECT DISTINCT` **must** be supported as a projection modifier in the minimum `query {}` surface. It removes duplicate rows from the projected schema and lowers using the distinct-row contract defined by InQL RFC 002. +- Aggregate function calls in relational expressions **must** resolve through imported library symbols (for example `from pub::incql import count, sum, avg`). The compiler **must not** treat `count`, `sum`, `avg`, `min`, or `max` as implicitly in scope ambient names. +- This RFC defines the minimum required aggregate-function surface and import model for `query {}`; it is not an exhaustive catalog of all IncQL functions. Additional functions require additive library evolution or follow-up RFCs that do not change the semantics of the required set defined here. +- `SELECT DISTINCT` **must** be supported as a projection modifier in the minimum `query {}` surface. It removes duplicate rows from the projected schema and lowers using the distinct-row contract defined by IncQL RFC 002. ### Clause inventory (minimum) @@ -114,17 +114,17 @@ This RFC **must** require at least: - `EXPLODE as ` for list-valued expressions - `WINDOW BY = ` for ranked/windowed forms in scope -Post-`SELECT` filters on the projected schema use `WHERE` again (a `WHERE` clause ordered after `SELECT` in the block). `HAVING` is **not** InQL syntax and **must not** be introduced. +Post-`SELECT` filters on the projected schema use `WHERE` again (a `WHERE` clause ordered after `SELECT` in the block). `HAVING` is **not** IncQL syntax and **must not** be introduced. ### Emission -- **Primary interchange:** checked `query {}` **must** lower to Substrait for operators in scope; document limitations. The normative `Rel`-level mapping, extension rules, and read-root policy are InQL RFC 002; this RFC **must not** contradict InQL RFC 002. +- **Primary interchange:** checked `query {}` **must** lower to Substrait for operators in scope; document limitations. The normative `Rel`-level mapping, extension rules, and read-root policy are IncQL RFC 002; this RFC **must not** contradict IncQL RFC 002. - **Batch execution** **must** be achievable through the execution context consuming Substrait (or a defined handoff). - Dialect-specific textual renderings **may** exist for inspection or debugging, but they are non-normative and **must not** become the portable interchange or an alternate execution contract for `query {}`. -### Lowering to InQL RFC 001 operations (optional path) +### Lowering to IncQL RFC 001 operations (optional path) -- Implementations **may** lower `query {}` to InQL RFC 001 trait/method calls on `DataSet[T]` for execution or optimization; semantics **must** match Substrait lowering. +- Implementations **may** lower `query {}` to IncQL RFC 001 trait/method calls on `DataSet[T]` for execution or optimization; semantics **must** match Substrait lowering. ## Design details @@ -150,21 +150,21 @@ Post-`SELECT` filters on the projected schema use `WHERE` again (a `WHERE` claus - **Parser / AST** for `query {}` and clauses. - **Typechecker** for relation registry, schema flow, aggregates. -- **IR / lowering** to Substrait (conforming to InQL RFC 002). +- **IR / lowering** to Substrait (conforming to IncQL RFC 002). - **LSP** inside `query {}`. -- **InQL package**: vocabulary registration for `query`. +- **IncQL package**: vocabulary registration for `query`. ## Design Decisions - **Lateral column aliases in `SELECT`**: an alias defined in a `SELECT` list is visible to subsequent expressions in the same list, in order (lateral column alias semantics). This follows the convention of DuckDB, Snowflake, and MySQL. Implementations must rewrite dependent expressions before Substrait lowering (inline substitution), since Substrait projection nodes are flat. An alias is not visible to expressions that precede it in the list. - **Shadowing warning**: when a bare name in a relational clause position resolves to a query column that shadows an outer Incan binding of the same name, the typechecker **should** emit a warning and suggest the `.column` form to make relational intent explicit. Writing `.column` explicitly suppresses the warning. The warning is informational — the resolution rule (column wins) still holds. Example message: `bare name \`customer_id\` shadows outer binding; use \`.customer_id\` to make relational intent explicit`. - **Return type inference**: `query {}` infers the output schema from the `SELECT` list. The result preserves the collection kind of the `FROM` source: a `query {}` over a `DataStream` yields a `DataStream`; over a `LazyFrame` yields a `LazyFrame`; over a `DataFrame` yields a `DataFrame`. Explicit type annotation at the call site is optional but recommended for documentation. -- **Post-`SELECT` clause ordering**: the canonical clause order is `FROM` → `JOIN` → `WHERE` → `GROUP BY` → `SELECT` → `WHERE` (post-`SELECT` filter) → `ORDER BY` → `LIMIT`. `HAVING` is not InQL syntax (InQL RFC 000). Exact diagnostic wording for ordering violations is an implementation detail. -- **Aggregate minimum set**: the initial implementation requires at least `count`, `sum`, `avg`, `min`, `max`. Window functions (`WINDOW BY` and ranked expressions) are part of the clause inventory but their detailed builtin set evolves during implementation. `DataStream` source restrictions follow InQL RFC 001's static capability gating: operations requiring unbounded state are statically rejected. -- **Aggregate function scope:** the minimum aggregate set is exposed through an importable InQL facade (examples use `pub::inql`). These names are ordinary imported symbols that gain aggregate meaning in aggregate-capable relational positions; they are not special ambient builtins. +- **Post-`SELECT` clause ordering**: the canonical clause order is `FROM` → `JOIN` → `WHERE` → `GROUP BY` → `SELECT` → `WHERE` (post-`SELECT` filter) → `ORDER BY` → `LIMIT`. `HAVING` is not IncQL syntax (IncQL RFC 000). Exact diagnostic wording for ordering violations is an implementation detail. +- **Aggregate minimum set**: the initial implementation requires at least `count`, `sum`, `avg`, `min`, `max`. Window functions (`WINDOW BY` and ranked expressions) are part of the clause inventory but their detailed builtin set evolves during implementation. `DataStream` source restrictions follow IncQL RFC 001's static capability gating: operations requiring unbounded state are statically rejected. +- **Aggregate function scope:** the minimum aggregate set is exposed through an importable IncQL facade (examples use `pub::incql`). These names are ordinary imported symbols that gain aggregate meaning in aggregate-capable relational positions; they are not special ambient builtins. - **`IN` clause**: not part of the RFC003 clause grammar. A separate RFC is required before introducing `IN` as a query-block clause operator, and its RHS contract must conform to `DataSet[T]` with a compatible schema. -- **Substrait version and mapping catalog**: InQL RFC 002 owns pinning policy, the north-star operator → `Rel` catalog, and extension URI requirements; the exact revision shipped with a toolchain is documented in release artifacts alongside the implementation. -- **Alternate surfaces**: pipe-forward is InQL RFC 005; method chains are InQL RFC 001. This RFC does not mandate alternative surfaces in the initial implementation. +- **Substrait version and mapping catalog**: IncQL RFC 002 owns pinning policy, the north-star operator → `Rel` catalog, and extension URI requirements; the exact revision shipped with a toolchain is documented in release artifacts alongside the implementation. +- **Alternate surfaces**: pipe-forward is IncQL RFC 005; method chains are IncQL RFC 001. This RFC does not mandate alternative surfaces in the initial implementation. - **Minimum join surface:** the required v0.1 clause inventory includes `JOIN ... ON` (inner join) and `LEFT JOIN ... ON`. `RIGHT` and `FULL OUTER` joins are not part of the required RFC003 minimum; adding them requires an additive extension that preserves the current join semantics. - **`SELECT DISTINCT`:** `query {}` **must** support `SELECT DISTINCT` in the minimum clause surface. It is the canonical clause-level spelling for duplicate elimination in this surface; method-chain APIs may expose equivalent operations, but they do not replace the query-surface keyword. - **Ordering syntax:** the implemented v0.1 query surface uses ordering helpers such as `asc(.amount)` and diff --git a/docs/rfcs/004_inql_execution_context.md b/docs/rfcs/004_incql_execution_context.md similarity index 74% rename from docs/rfcs/004_inql_execution_context.md rename to docs/rfcs/004_incql_execution_context.md index be9e727c..7597f4b4 100644 --- a/docs/rfcs/004_inql_execution_context.md +++ b/docs/rfcs/004_incql_execution_context.md @@ -1,77 +1,77 @@ -# InQL RFC 004: Execution context and DataFusion +# IncQL RFC 004: Execution context and DataFusion - **Status:** In Progress - **Created:** 2026-03-24 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 000 (language specification — compilation model, layer boundaries) - - InQL RFC 001 (dataset types — `DataSet[T]` carriers; `DataFrame[T]` as materialized result) - - InQL RFC 002 (Apache Substrait — plan interchange; `ReadRel` and logical reads) - - InQL RFC 003 (query DSL — `query {}` produces plans this RFC executes) - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 008 (optimizer boundary, statistics, cost-based optimization, and adaptive execution) -- **Issue:** [InQL #5](https://github.com/encero-systems/InQL/issues/5) + - IncQL RFC 000 (language specification — compilation model, layer boundaries) + - IncQL RFC 001 (dataset types — `DataSet[T]` carriers; `DataFrame[T]` as materialized result) + - IncQL RFC 002 (Apache Substrait — plan interchange; `ReadRel` and logical reads) + - IncQL RFC 003 (query DSL — `query {}` produces plans this RFC executes) + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 008 (optimizer boundary, statistics, cost-based optimization, and adaptive execution) +- **Issue:** [IncQL #5](https://github.com/encero-systems/IncQL/issues/5) - **RFC PR:** - - **Written against:** Incan v0.2 - **Shipped in:** — ## Summary -This RFC specifies the **execution context**: the session object that bridges InQL's **typed logical plans** and **real execution**. It defines how authors **read data** into `DataSet[T]` values, **execute plans** (lowered to Substrait per InQL RFC 002), and **write results** back to storage. **Apache DataFusion** is the **reference (and default) execution backend** for plan optimization and execution: it consumes Substrait plans, applies query optimizations (predicate pushdown, projection pruning, join reordering, constant folding), and executes against registered data sources, returning **Apache Arrow** record batches that InQL wraps in typed `DataFrame[T]` carriers. This RFC standardizes the explicit core `Session` contract; higher operational layers may compose, scope, or inject sessions and adapter conveniences on top, but they do not redefine InQL execution semantics. With RFCs 000–004, InQL is usable for read → transform → write workflows. +This RFC specifies the **execution context**: the session object that bridges IncQL's **typed logical plans** and **real execution**. It defines how authors **read data** into `DataSet[T]` values, **execute plans** (lowered to Substrait per IncQL RFC 002), and **write results** back to storage. **Apache DataFusion** is the **reference (and default) execution backend** for plan optimization and execution: it consumes Substrait plans, applies query optimizations (predicate pushdown, projection pruning, join reordering, constant folding), and executes against registered data sources, returning **Apache Arrow** record batches that IncQL wraps in typed `DataFrame[T]` carriers. This RFC standardizes the explicit core `Session` contract; higher operational layers may compose, scope, or inject sessions and adapter conveniences on top, but they do not redefine IncQL execution semantics. With RFCs 000–004, IncQL is usable for read → transform → write workflows. ## Core model -1. A **`Session`** (or **execution context**) is the entry point for InQL programs that interact with data. It holds **table registrations**, **configuration**, and a **reference to the execution backend**. -2. **Reading data** creates `LazyFrame[T]` or `DataStream[T]` values from registered sources. The session resolves logical names to physical data; the plan carries only the logical identity (InQL RFC 002). +1. A **`Session`** (or **execution context**) is the entry point for IncQL programs that interact with data. It holds **table registrations**, **configuration**, and a **reference to the execution backend**. +2. **Reading data** creates `LazyFrame[T]` or `DataStream[T]` values from registered sources. The session resolves logical names to physical data; the plan carries only the logical identity (IncQL RFC 002). 3. **Executing plans** passes Substrait plans (or equivalent IR) through the backend's **optimizer** and **executor**, producing **Arrow record batches**. -4. **Materializing results** wraps Arrow output in typed `DataFrame[T]` carriers (InQL RFC 001). +4. **Materializing results** wraps Arrow output in typed `DataFrame[T]` carriers (IncQL RFC 001). 5. **Writing results** sends `DataSet[T]` values (or materialized `DataFrame[T]`) to registered output targets. ## Motivation -InQL RFCs 000–003 define a typed query language that produces portable logical plans. Without an execution context, those plans are inert: there is no way to read data in, execute the relational work, or write results out. The execution context completes the pipeline from authored intent to running workload. +IncQL RFCs 000–003 define a typed query language that produces portable logical plans. Without an execution context, those plans are inert: there is no way to read data in, execute the relational work, or write results out. The execution context completes the pipeline from authored intent to running workload. Choosing Apache DataFusion as the reference backend is a pragmatic decision: it is Rust-native, Substrait-aware, provides serious query optimization, and operates on Apache Arrow — the de facto columnar data interchange format in the modern data ecosystem. Naming it explicitly avoids the trap of an abstract "pluggable backend" with no concrete implementation. -The `Session` surface should also feel familiar to users coming from established data runtimes such as Spark: one obvious entry point for reading data, registering logical names, executing plans, and writing results. That familiarity is an ergonomic goal, but not a semantic dependency. InQL keeps its own typed `DataSet[T]` model, explicit execution boundary, and RFC-defined semantics rather than inheriting runtime-specific API details from other systems. +The `Session` surface should also feel familiar to users coming from established data runtimes such as Spark: one obvious entry point for reading data, registering logical names, executing plans, and writing results. That familiarity is an ergonomic goal, but not a semantic dependency. IncQL keeps its own typed `DataSet[T]` model, explicit execution boundary, and RFC-defined semantics rather than inheriting runtime-specific API details from other systems. ## Goals - Define **`Session`** as the execution context: what it holds, how authors create one, and what operations it exposes. - Specify **read operations**: how named tables, files, and virtual data become `LazyFrame[T]` or `DataStream[T]` values through the session. -- Specify **plan execution**: how Substrait plans (InQL RFC 002) flow through the backend optimizer and executor to produce results. -- Specify **materialization**: how execution output (Arrow record batches) becomes typed `DataFrame[T]` (InQL RFC 001). +- Specify **plan execution**: how Substrait plans (IncQL RFC 002) flow through the backend optimizer and executor to produce results. +- Specify **materialization**: how execution output (Arrow record batches) becomes typed `DataFrame[T]` (IncQL RFC 001). - Specify **write operations**: how `DataSet[T]` values are written to registered output targets. - Name **Apache DataFusion** as the reference and default execution backend, with **Apache Arrow** as the in-memory data representation. - Define the **backend abstraction** so alternative backends (Polars, DuckDB, remote engines) can be substituted without changing author code, with backend-specific configuration exposed through a dedicated `backends` namespace rather than the root API. - Clarify the boundary between the execution backend and higher operational or adapter layers that may provide source/sink integrations or scoped session conveniences. -- Shape the `Session` surface as a familiar entry point for data work, taking ergonomic inspiration from established runtimes such as Spark while preserving InQL's typed carrier model and explicit execution boundaries. +- Shape the `Session` surface as a familiar entry point for data work, taking ergonomic inspiration from established runtimes such as Spark while preserving IncQL's typed carrier model and explicit execution boundaries. ## Non-Goals -- Normative naming rules — InQL RFC 000. -- Dataset types and trait hierarchy — InQL RFC 001. -- Substrait `Rel`-level mapping and extension policy — InQL RFC 002. -- `query {}` grammar and clause inventory — InQL RFC 003. -- Orchestration, workflow scheduling, quality gates — execution and operational layers above InQL. -- Distributed execution, cluster scheduling, shuffle — out of scope for InQL; may be addressed by runners in the operational layer. +- Normative naming rules — IncQL RFC 000. +- Dataset types and trait hierarchy — IncQL RFC 001. +- Substrait `Rel`-level mapping and extension policy — IncQL RFC 002. +- `query {}` grammar and clause inventory — IncQL RFC 003. +- Orchestration, workflow scheduling, quality gates — execution and operational layers above IncQL. +- Distributed execution, cluster scheduling, shuffle — out of scope for IncQL; may be addressed by runners in the operational layer. - Credential management, secret resolution, IAM — operational layer; the session receives resolved bindings, not raw secrets. -- Standardizing workflow-scoped session propagation, active-session lookup, or Reader/Writer convenience APIs — these may be provided by higher operational layers, but are not part of the core InQL contract in this RFC. +- Standardizing workflow-scoped session propagation, active-session lookup, or Reader/Writer convenience APIs — these may be provided by higher operational layers, but are not part of the core IncQL contract in this RFC. ## Guide-level explanation ### Creating a session ```incan -from pub::inql import Session +from pub::incql import Session session = Session.default() ``` -A session holds registered data sources and configuration. `Session.default()` creates a context with the default backend (DataFusion). Authors who need custom configuration use a builder; backend-specific configuration lives under `pub::inql.backends`. Higher operational layers may wrap this construction behind step- or pipeline-level runtime setup, but the core InQL surface remains an explicit `Session`: +A session holds registered data sources and configuration. `Session.default()` creates a context with the default backend (DataFusion). Authors who need custom configuration use a builder; backend-specific configuration lives under `pub::incql.backends`. Higher operational layers may wrap this construction behind step- or pipeline-level runtime setup, but the core IncQL surface remains an explicit `Session`: ```incan -from pub::inql import Session, backends +from pub::incql import Session, backends session = Session.builder() .with_backend(backends.DataFusion()) @@ -81,7 +81,7 @@ session = Session.builder() ### Reading data ```incan -from pub::inql import Session, LazyFrame, csv_source +from pub::incql import Session, LazyFrame, csv_source from models import Order session = Session.default() @@ -93,7 +93,7 @@ session.register("orders", csv_source("s3://bucket/orders.csv")) orders: LazyFrame[Order] = session.table("orders") ``` -`session.table("orders")` returns a `LazyFrame[Order]` — a deferred plan rooted in a `ReadRel` (InQL RFC 002) that carries the logical name `"orders"`. No data moves until the plan is executed. +`session.table("orders")` returns a `LazyFrame[Order]` — a deferred plan rooted in a `ReadRel` (IncQL RFC 002) that carries the logical name `"orders"`. No data moves until the plan is executed. For file-based sources: @@ -114,10 +114,10 @@ sample: LazyFrame[Order] = session.from_values([ ### Transforming data -Once you have a `LazyFrame[T]`, use `query {}` (InQL RFC 003) or method chains (InQL RFC 001): +Once you have a `LazyFrame[T]`, use `query {}` (IncQL RFC 003) or method chains (IncQL RFC 001): ```incan -from pub::inql.functions import count, sum +from pub::incql.functions import count, sum result = query { FROM orders @@ -134,19 +134,19 @@ result = query { ### Executing and collecting ```incan -from pub::inql import DataFrame +from pub::incql import DataFrame from models import OrderSummary # Execute the plan and materialize results materialized: DataFrame[OrderSummary] = session.collect(result) ``` -`session.collect(result)` takes the `LazyFrame`, lowers to Substrait (InQL RFC 002), passes the plan through DataFusion's optimizer, executes it, and wraps the resulting Arrow record batches in a typed `DataFrame[T]`. +`session.collect(result)` takes the `LazyFrame`, lowers to Substrait (IncQL RFC 002), passes the plan through DataFusion's optimizer, executes it, and wraps the resulting Arrow record batches in a typed `DataFrame[T]`. ### Writing results ```incan -from pub::inql import parquet_sink +from pub::incql import parquet_sink # Write to a registered output target session.write(materialized, parquet_sink("s3://bucket/summaries/")) @@ -158,8 +158,8 @@ session.write_parquet(result, "s3://bucket/summaries/") ### End-to-end example ```incan -from pub::inql import Session, LazyFrame, DataFrame, csv_source, parquet_sink -from pub::inql.functions import count, sum +from pub::incql import Session, LazyFrame, DataFrame, csv_source, parquet_sink +from pub::incql.functions import count, sum from models import Order, OrderSummary session = Session.default() @@ -192,7 +192,7 @@ session.write(result, parquet_sink("s3://bucket/summaries/")) - `Session.default()` **must** create a context with the DataFusion backend and default configuration. - `Session.builder()` **must** return a builder that allows backend selection and configuration before constructing the session. - The `Session` API **should** present a small, discoverable entry-point surface for data work, broadly analogous to familiar runtime entry points in systems such as Spark. -- That ergonomic inspiration **must not** override InQL's typed carrier model, explicit `Session.collect(...)` execution boundary, or the prohibition on raw SQL as a core execution path. +- That ergonomic inspiration **must not** override IncQL's typed carrier model, explicit `Session.collect(...)` execution boundary, or the prohibition on raw SQL as a core execution path. The intended core session surface for v0.1 is: @@ -215,7 +215,7 @@ This table defines the intended high-level API shape. The detailed normative rul ### Read operations -| Method | Returns | Substrait lowering (InQL RFC 002) | +| Method | Returns | Substrait lowering (IncQL RFC 002) | | --------------------------- | -------------- | ------------------------------------------------ | | `session.table(name)` | `LazyFrame[T]` | `ReadRel` + `NamedTable` | | `session.read_parquet(uri)` | `LazyFrame[T]` | `ReadRel` + `LocalFiles` (Parquet format) | @@ -230,15 +230,15 @@ This table defines the intended high-level API shape. The detailed normative rul ### Table registration - `session.register(logical_name, source_identifier)` binds a logical name to a data source definition. -- The `source_identifier` is an opaque string or structured descriptor that the session resolves through its integration and execution layers to a concrete scan. InQL does not define the format of source identifiers beyond requiring that the session can resolve them for the chosen execution backend. +- The `source_identifier` is an opaque string or structured descriptor that the session resolves through its integration and execution layers to a concrete scan. IncQL does not define the format of source identifiers beyond requiring that the session can resolve them for the chosen execution backend. - Registration **may** also accept explicit schema information (an Incan `model` type) for sources where the schema cannot be inferred. ### Plan execution - `session.collect(lazy_frame)` **must**: - 1. Lower the `LazyFrame`'s logical plan to Substrait (conforming to InQL RFC 002). + 1. Lower the `LazyFrame`'s logical plan to Substrait (conforming to IncQL RFC 002). 2. Pass the Substrait plan to the backend for optimization and execution. - 3. Wrap the resulting data in a typed `DataFrame[T]` (InQL RFC 001). + 3. Wrap the resulting data in a typed `DataFrame[T]` (IncQL RFC 001). - `Session.collect(...)` is the canonical execution entry point in the normative API. Implementations **may** additionally offer a convenience form on `LazyFrame`, but it **must** delegate to session-owned execution semantics rather than bypassing the session boundary. - The backend **should** apply query optimizations (predicate pushdown, projection pruning, join reordering, constant folding, common-subexpression elimination) before execution. - Execution and write methods **must** report typed failures that distinguish at least registration / binding errors, lowering or planning errors, backend optimization or execution errors, and I/O or sink errors. @@ -258,23 +258,23 @@ This table defines the intended high-level API shape. The detailed normative rul ### DataFusion as reference backend -Apache DataFusion is the **reference and default** execution backend for InQL v0.1: +Apache DataFusion is the **reference and default** execution backend for IncQL v0.1: - **Plan consumption**: DataFusion accepts Substrait plans through the `substrait` crate's consumer, converting them to DataFusion logical plans. - **Optimization**: DataFusion's optimizer applies rule-based and cost-based optimizations to the logical plan before execution. - **Execution**: DataFusion executes the optimized plan against registered table providers, producing Apache Arrow `RecordBatch` results. -- **Arrow as data plane**: DataFusion operates natively on Arrow columnar data. InQL's `DataFrame[T]` wraps Arrow record batches with the typed model `T` on top. +- **Arrow as data plane**: DataFusion operates natively on Arrow columnar data. IncQL's `DataFrame[T]` wraps Arrow record batches with the typed model `T` on top. The data flow: ```text -InQL query / method chain - → Substrait Plan (protobuf, InQL RFC 002) +IncQL query / method chain + → Substrait Plan (protobuf, IncQL RFC 002) → DataFusion LogicalPlan (via substrait consumer) → DataFusion optimizer → DataFusion physical execution → Arrow RecordBatch[] - → DataFrame[T] (typed InQL carrier, InQL RFC 001) + → DataFrame[T] (typed IncQL carrier, IncQL RFC 001) ``` ### Backend abstraction @@ -284,12 +284,12 @@ InQL query / method chain - The backend interface **must** support at minimum: plan execution from Substrait, table provider registration, and result collection as Arrow record batches (or equivalent). - DataFusion is the **default**; it is not the **only** permitted backend. Implementations **may** offer backend selection through `Session.builder()`. - `Session.builder()` **must** expose a stable portable configuration subset (at minimum backend selection plus core execution and optimizer settings). Backend-specific tuning **may** be surfaced through backend options, but this RFC does not standardize the full DataFusion configuration surface. -- Backend-specific configuration objects **should** be exposed from `pub::inql.backends` rather than from the root `pub::inql` namespace. +- Backend-specific configuration objects **should** be exposed from `pub::incql.backends` rather than from the root `pub::incql` namespace. - The normative user-facing session type remains `Session`; this RFC does not define backend-named session types such as `DuckDbSession` or `PolarsSession`. - External systems such as warehouses, databases, filesystems, or object stores are not necessarily execution backends. They **may** instead appear as sources or sinks resolved through the session's integration layer while plan execution remains owned by the session's selected backend. - Higher operational layers **may** provide scoped session propagation or convenience APIs for adapters and workflow steps, but those conveniences **must** delegate to `Session` rather than replacing the core execution model defined here. -### Interaction with InQL RFC 001 types +### Interaction with IncQL RFC 001 types - `session.table(name)` returns `LazyFrame[T]` — a `BoundedDataSet[T]`. - `session.collect(plan)` returns `DataFrame[T]` — a materialized `BoundedDataSet[T]`. @@ -301,13 +301,13 @@ InQL query / method chain ### Interaction with Incan - The session is an Incan value — it can be passed, stored, and used in ordinary Incan code. -- `model` definitions supply schema for `T` as in all other InQL RFCs. +- `model` definitions supply schema for `T` as in all other IncQL RFCs. ### Interaction with operational layers - Operational layers **may** construct, scope, and inject `Session` values for steps, jobs, or pipeline runs. - Such layers **may** offer convenience APIs on readers, writers, or workflow steps that rely on a locally scoped session, but those APIs are layered sugar over the `Session` contract rather than alternate execution semantics. -- InQL itself continues to define the session, backend selection, registration, execution, and write boundary. +- IncQL itself continues to define the session, backend selection, registration, execution, and write boundary. ### Compatibility @@ -316,8 +316,8 @@ InQL query / method chain ## Alternatives considered -- **No session / implicit context in core InQL** (Polars-style) — rejected; an explicit session makes backend selection, table registration, and configuration visible rather than ambient. It also maps cleanly to DataFusion's `SessionContext`. Higher operational layers may still provide scoped convenience on top of that explicit core. -- **Session defined in the operational layer only** — rejected; without a session in InQL itself, there is no way to write self-contained InQL programs that read, transform, and write data. The operational layer may compose sessions with workflow and adapter concerns, but the base concept belongs in InQL. +- **No session / implicit context in core IncQL** (Polars-style) — rejected; an explicit session makes backend selection, table registration, and configuration visible rather than ambient. It also maps cleanly to DataFusion's `SessionContext`. Higher operational layers may still provide scoped convenience on top of that explicit core. +- **Session defined in the operational layer only** — rejected; without a session in IncQL itself, there is no way to write self-contained IncQL programs that read, transform, and write data. The operational layer may compose sessions with workflow and adapter concerns, but the base concept belongs in IncQL. - **Abstract backend only, no named reference** — rejected; naming DataFusion as the reference avoids an abstract interface with no concrete implementation. The abstraction exists for extensibility; DataFusion is what ships. ## Drawbacks @@ -328,26 +328,26 @@ InQL query / method chain ## Implementation architecture -Non-normative: the reference implementation **should** use DataFusion's `SessionContext` as the underlying engine, with InQL's `Session` wrapping it to provide typed APIs, table registration helpers, and Substrait plan submission. The Substrait-to-DataFusion path **should** use the community `substrait` crate (or equivalent). +Non-normative: the reference implementation **should** use DataFusion's `SessionContext` as the underlying engine, with IncQL's `Session` wrapping it to provide typed APIs, table registration helpers, and Substrait plan submission. The Substrait-to-DataFusion path **should** use the community `substrait` crate (or equivalent). ## Layers affected -- **InQL library**: `Session` type, read/write methods, `backends` module, backend abstraction trait. +- **IncQL library**: `Session` type, read/write methods, `backends` module, backend abstraction trait. - **Rust interop / FFI**: DataFusion integration, Arrow record batch handling. - **Typechecker**: ensuring `T` flows correctly through session methods. - **Testing**: end-to-end tests that read, transform, and write using the session. ## Design Decisions -- **Raw SQL escape hatch:** raw SQL execution is not part of InQL and is not permitted as a `Session` escape hatch. SQL belongs in dialect-specific surfaces outside this RFC; it must not be smuggled through the execution-context API as an alternate query path. +- **Raw SQL escape hatch:** raw SQL execution is not part of IncQL and is not permitted as a `Session` escape hatch. SQL belongs in dialect-specific surfaces outside this RFC; it must not be smuggled through the execution-context API as an alternate query path. - **External catalogs:** v0.1 standardizes logical registration and backend-resolved reads, not a portable catalog API. Integrations with systems such as Unity Catalog, Hive Metastore, or Iceberg REST **may** be supported by a backend or product layer, but their APIs and binding contracts are deferred from this RFC. - **Collection API shape:** `Session.collect(...)` is the required canonical API. A convenience form on `LazyFrame` **may** exist, but it is secondary and must route through the session. - **Error model:** execution-facing APIs **must** use typed errors that distinguish at least registration or binding failures, Substrait lowering or planning failures, backend optimization or runtime execution failures, and output or I/O failures. Exact type names are implementation details. -- **Builder configuration surface:** `Session.builder()` exposes a small portable configuration surface in v0.1. It **must not** promise one-to-one access to every DataFusion knob. Backend-specific configuration may be carried through backend options without becoming part of the portable InQL contract. -- **Backend namespace and session shape:** backend-specific configuration belongs under `pub::inql.backends`, while `Session` remains the portable execution entry point. This RFC does not standardize backend-specific session types. +- **Builder configuration surface:** `Session.builder()` exposes a small portable configuration surface in v0.1. It **must not** promise one-to-one access to every DataFusion knob. Backend-specific configuration may be carried through backend options without becoming part of the portable IncQL contract. +- **Backend namespace and session shape:** backend-specific configuration belongs under `pub::incql.backends`, while `Session` remains the portable execution entry point. This RFC does not standardize backend-specific session types. - **Execution backend vs source/sink integrations:** the backend named in `Session` is the engine that optimizes and executes the plan. External systems used for reads or writes may be integrated through registration or adapter layers without becoming separate execution backends in the core model. -- **Scoped session conveniences:** workflow or adapter layers may offer locally scoped session access as ergonomic sugar, but the normative InQL contract remains the explicit `Session` API defined in this RFC. -- **Session API inspiration:** the `Session` surface intentionally takes ergonomic inspiration from familiar data-runtime entry points such as Spark's session object, but InQL keeps its own typed carrier semantics, backend abstraction, and explicit execution model. Familiarity is a usability goal, not a promise of Spark API or semantic compatibility. +- **Scoped session conveniences:** workflow or adapter layers may offer locally scoped session access as ergonomic sugar, but the normative IncQL contract remains the explicit `Session` API defined in this RFC. +- **Session API inspiration:** the `Session` surface intentionally takes ergonomic inspiration from familiar data-runtime entry points such as Spark's session object, but IncQL keeps its own typed carrier semantics, backend abstraction, and explicit execution model. Familiarity is a usability goal, not a promise of Spark API or semantic compatibility. ## Implementation plan and checklist (non-normative) @@ -379,4 +379,4 @@ This section tracks the implementation path for this RFC. It is intentionally op ### Exit criteria for RFC status change -RFC 004 can move from `In Progress` to `Implemented` when all checklist items above are complete and the InQL CI gate is green on the target release branch. +RFC 004 can move from `In Progress` to `Implemented` when all checklist items above are complete and the IncQL CI gate is green on the target release branch. diff --git a/docs/rfcs/005_inql_pipe_forward.md b/docs/rfcs/005_incql_pipe_forward.md similarity index 62% rename from docs/rfcs/005_inql_pipe_forward.md rename to docs/rfcs/005_incql_pipe_forward.md index 71ef69e6..b0277de0 100644 --- a/docs/rfcs/005_inql_pipe_forward.md +++ b/docs/rfcs/005_incql_pipe_forward.md @@ -1,21 +1,21 @@ -# InQL RFC 005: Pipe-forward relational syntax (`|>`) +# IncQL RFC 005: Pipe-forward relational syntax (`|>`) - **Status:** Blocked - **Created:** 2026-03-18 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 000 (language specification — naming and query schema; **must** stay aligned) - - InQL RFC 001 (dataset types — carriers and method APIs) - - InQL RFC 003 (`query {}` — primary clause surface) + - IncQL RFC 000 (language specification — naming and query schema; **must** stay aligned) + - IncQL RFC 001 (dataset types — carriers and method APIs) + - IncQL RFC 003 (`query {}` — primary clause surface) - Incan RFC 040 (Scoped DSL Glyph Surfaces — prerequisite for `|>` support) -- **Issue:** [InQL #6](https://github.com/encero-systems/InQL/issues/6) +- **Issue:** [IncQL #6](https://github.com/encero-systems/IncQL/issues/6) - **RFC PR:** - - **Written against:** Incan v0.2 - **Shipped in:** - ## Summary -This RFC specifies an optional **pipe-forward** surface for InQL relational pipelines: `|>`-chained stages applied to an existing `DataSet[T]` expression, with grammar, precedence, and desugaring to the same relational semantics as `query {}` and collection method chains. Identifier resolution and current query schema behavior **must** match InQL RFC 000 §§2–4; this RFC does **not** redefine naming rules. +This RFC specifies an optional **pipe-forward** surface for IncQL relational pipelines: `|>`-chained stages applied to an existing `DataSet[T]` expression, with grammar, precedence, and desugaring to the same relational semantics as `query {}` and collection method chains. Identifier resolution and current query schema behavior **must** match IncQL RFC 000 §§2–4; this RFC does **not** redefine naming rules. > **Blcoked** for the following reason: > @@ -23,27 +23,27 @@ This RFC specifies an optional **pipe-forward** surface for InQL relational pipe ## Motivation -Some authors prefer a linear, left-to-right pipeline over clause blocks or method chains. In Incan specifically, pipe-forward becomes useful when authors already have a `DataSet[T]` value in scope and want concise shorthand for a few relational steps without switching into a larger `query {}` block. A dedicated RFC keeps InQL RFC 003 focused on `query {}` while still committing to one relational semantic core across surfaces (InQL RFC 000 §6). +Some authors prefer a linear, left-to-right pipeline over clause blocks or method chains. In Incan specifically, pipe-forward becomes useful when authors already have a `DataSet[T]` value in scope and want concise shorthand for a few relational steps without switching into a larger `query {}` block. A dedicated RFC keeps IncQL RFC 003 focused on `query {}` while still committing to one relational semantic core across surfaces (IncQL RFC 000 §6). ### Prior art -This RFC draws inspiration from Google's pipe query syntax in BigQuery ([documentation](https://docs.cloud.google.com/bigquery/docs/pipe-syntax-guide)), which shows that a linear relational pipeline can remain readable for complex query construction. That reference is informative, not normative: InQL pipe-forward semantics are defined by this RFC together with InQL RFC 000 and RFC 003. This RFC aims to stay close to the core value of that design, especially expression-first pipeline authoring, while still allowing Incan-specific choices where required by the type system, library model, and RFC-defined surface conventions. +This RFC draws inspiration from Google's pipe query syntax in BigQuery ([documentation](https://docs.cloud.google.com/bigquery/docs/pipe-syntax-guide)), which shows that a linear relational pipeline can remain readable for complex query construction. That reference is informative, not normative: IncQL pipe-forward semantics are defined by this RFC together with IncQL RFC 000 and RFC 003. This RFC aims to stay close to the core value of that design, especially expression-first pipeline authoring, while still allowing Incan-specific choices where required by the type system, library model, and RFC-defined surface conventions. ## Goals - Define normative pipe-forward syntax (tokens, allowed stage heads, and how an existing `DataSet[T]` value becomes the current input relation). -- Specify desugaring or lowering to the same relational plan shape as `query {}` / InQL RFC 001 operations (without mandating a single internal representation). -- Require consistency with InQL RFC 000: `.column`, `relation.column`, bare names, and `SELECT` alias boundaries behave identically to `query {}` in equivalent pipelines. +- Specify desugaring or lowering to the same relational plan shape as `query {}` / IncQL RFC 001 operations (without mandating a single internal representation). +- Require consistency with IncQL RFC 000: `.column`, `relation.column`, bare names, and `SELECT` alias boundaries behave identically to `query {}` in equivalent pipelines. - Cover tooling expectations at a high level (LSP, diagnostics) where they differ from `query {}`. ## Non-Goals -- Replacing `query {}` as the primary checked surface — InQL RFC 003 remains the default clause grammar unless product direction changes. -- Normative naming rules — InQL RFC 000. -- `DataSet[T]` type definitions and backend boundary — InQL RFC 001. -- Substrait emission details — may reuse InQL RFC 003 paths after desugar; no duplicate Substrait spec unless a gap is found. -- Execution context and session — InQL RFC 004. -- Finalizing a dedicated aggregate-stage spelling analogous to BigQuery's `AGGREGATE` operator. The initial value of pipe-forward is expression-first relational shorthand; aggregate-stage surface details may be refined in a follow-up amendment as long as semantics stay aligned with InQL RFC 003. +- Replacing `query {}` as the primary checked surface — IncQL RFC 003 remains the default clause grammar unless product direction changes. +- Normative naming rules — IncQL RFC 000. +- `DataSet[T]` type definitions and backend boundary — IncQL RFC 001. +- Substrait emission details — may reuse IncQL RFC 003 paths after desugar; no duplicate Substrait spec unless a gap is found. +- Execution context and session — IncQL RFC 004. +- Finalizing a dedicated aggregate-stage spelling analogous to BigQuery's `AGGREGATE` operator. The initial value of pipe-forward is expression-first relational shorthand; aggregate-stage surface details may be refined in a follow-up amendment as long as semantics stay aligned with IncQL RFC 003. ## Guide-level explanation @@ -53,7 +53,7 @@ new_df = df |> select { region } ``` -This is the core value proposition: when a dataset value already exists in scope, pipe-forward gives a concise linear shorthand for a few relational steps. The semantics are identical to an equivalent `query {}` block or method chain; only syntax differs. `.amount` and `region` resolve using the same rules as the other InQL surfaces. +This is the core value proposition: when a dataset value already exists in scope, pipe-forward gives a concise linear shorthand for a few relational steps. The semantics are identical to an equivalent `query {}` block or method chain; only syntax differs. `.amount` and `region` resolve using the same rules as the other IncQL surfaces. Another common pattern is incremental narrowing over an existing dataset: @@ -86,7 +86,7 @@ filtered = query { ### Token and stage syntax - Pipe-forward uses the `|>` token to chain relational stages. -- A pipe-forward expression may begin with any expression whose type conforms to `DataSet[T]` per InQL RFC 001; that expression establishes the primary relation and its schema `T`. +- A pipe-forward expression may begin with any expression whose type conforms to `DataSet[T]` per IncQL RFC 001; that expression establishes the primary relation and its schema `T`. - A pipe chain may follow any expression whose type conforms to `DataSet[T]`, including a named dataset value, a method-chain result, or a `query { ... }` expression. - Each subsequent stage is introduced by `|>` followed by a relational operation keyword and its arguments. - Stage keywords are case-insensitive. Examples in this RFC use lowercase because pipe-forward is intended as lightweight shorthand over existing dataset values. @@ -94,15 +94,15 @@ filtered = query { ### Relational operation keywords -Pipe-forward stages use the same relational operations as `query {}` (InQL RFC 003). The keyword set for the initial implementation includes at least: `where`, `join`, `group_by`, `select`, `order_by`, `limit`, `explode`. +Pipe-forward stages use the same relational operations as `query {}` (IncQL RFC 003). The keyword set for the initial implementation includes at least: `where`, `join`, `group_by`, `select`, `order_by`, `limit`, `explode`. ### Desugaring -A pipe-forward expression **must** desugar to the same relational plan as an equivalent `query {}` block or `DataSet[T]` method chain. Implementations **may** desugar to `query {}` AST nodes, to InQL RFC 001 trait method calls, or directly to Substrait — provided the resulting plan is semantically identical. +A pipe-forward expression **must** desugar to the same relational plan as an equivalent `query {}` block or `DataSet[T]` method chain. Implementations **may** desugar to `query {}` AST nodes, to IncQL RFC 001 trait method calls, or directly to Substrait — provided the resulting plan is semantically identical. ### Identifier resolution -All naming rules from InQL RFC 000 §2–§4 apply identically: +All naming rules from IncQL RFC 000 §2–§4 apply identically: - `.column` resolves against the primary relation established by the input dataset expression. - `relation.column` resolves against named join relations. @@ -111,36 +111,36 @@ All naming rules from InQL RFC 000 §2–§4 apply identically: ### Vocabulary activation -Pipe-forward stage keywords are activated through library-driven vocabulary, consistent with InQL RFC 003's `query` keyword activation. A compilation unit with InQL active **must** recognize pipe-forward stage keywords as soft keywords. +Pipe-forward stage keywords are activated through library-driven vocabulary, consistent with IncQL RFC 003's `query` keyword activation. A compilation unit with IncQL active **must** recognize pipe-forward stage keywords as soft keywords. ## Design details -1. Dependency on InQL RFC 000 - Pipe-forward **must not** introduce a second resolution order for bare names or `.column`. Any deviation requires amending InQL RFC 000. +1. Dependency on IncQL RFC 000 + Pipe-forward **must not** introduce a second resolution order for bare names or `.column`. Any deviation requires amending IncQL RFC 000. 2. Dependency on Incan RFC 040 This RFC is blocked on Incan RFC 040. The planned implementation path for `|>` is the scoped DSL glyph mechanism defined there, not ambient global operator support in ordinary Incan expressions. ## Alternatives considered -- **Fold pipe-forward into InQL RFC 003**: rejected. Keeps `query {}` RFC smaller and allows pipe syntax to ship on an independent timeline. +- **Fold pipe-forward into IncQL RFC 003**: rejected. Keeps `query {}` RFC smaller and allows pipe syntax to ship on an independent timeline. ## Drawbacks -- **Three** user-visible relational spellings (`query {}`, chains, pipes) increase documentation and pedagogical load; InQL RFC 000 mitigates by unifying semantics. +- **Three** user-visible relational spellings (`query {}`, chains, pipes) increase documentation and pedagogical load; IncQL RFC 000 mitigates by unifying semantics. ## Layers affected - **Parser / AST** for pipe stages and `|>` token. - **Typechecker**: same relation schema flow as `query {}` after desugar or shared analysis. -- **Lowering / IR**: shared path with InQL RFC 003 where possible. +- **Lowering / IR**: shared path with IncQL RFC 003 where possible. - **LSP**: pipe-specific completions if syntax diverges from `query {}`. ## Design Decisions - **Keyword casing**: pipe-forward stage keywords are case-insensitive. Examples in this RFC use lowercase to emphasize pipe-forward as lightweight shorthand over existing values; uppercase spellings remain valid. - **Entry shape (initial)**: a pipe chain starts from an expression whose type conforms to `DataSet[T]`. There is no standalone `from ` entry sugar in this surface. -- **Vocabulary activation**: library-driven, sharing the activation mechanism with InQL RFC 003's `query` keyword. Pipe-forward stage keywords are soft keywords activated by the InQL library dependency. +- **Vocabulary activation**: library-driven, sharing the activation mechanism with IncQL RFC 003's `query` keyword. Pipe-forward stage keywords are soft keywords activated by the IncQL library dependency. - **Method chain interaction**: mixing `|>` stages and `.method()` calls in a single pipeline expression is not supported initially. Authors choose one surface per pipeline. Interoperability may be explored in a future amendment. -- **Aggregate-stage shape**: this RFC does not yet standardize a dedicated `aggregate` stage matching BigQuery's `AGGREGATE` operator. That surface may be refined in a follow-up amendment, provided the resulting semantics remain aligned with InQL RFC 003. -- **Not in v0.1 scope**: pipe-forward is deferred from InQL v0.1. It does not unlock new capability beyond what `query {}` and method chains already provide. +- **Aggregate-stage shape**: this RFC does not yet standardize a dedicated `aggregate` stage matching BigQuery's `AGGREGATE` operator. That surface may be refined in a follow-up amendment, provided the resulting semantics remain aligned with IncQL RFC 003. +- **Not in v0.1 scope**: pipe-forward is deferred from IncQL v0.1. It does not unlock new capability beyond what `query {}` and method chains already provide. diff --git a/docs/rfcs/006_unnest_core_substrait.md b/docs/rfcs/006_unnest_core_substrait.md index f55ed18e..219c22f6 100644 --- a/docs/rfcs/006_unnest_core_substrait.md +++ b/docs/rfcs/006_unnest_core_substrait.md @@ -1,26 +1,26 @@ -# InQL RFC 006: Promote unnest/explode to core Substrait lowering +# IncQL RFC 006: Promote unnest/explode to core Substrait lowering - **Status:** Blocked - **Created:** 2026-03-27 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 002 (Apache Substrait — normative gap classification for unnest; **prerequisite**) - - InQL RFC 003 (`query {}` — `EXPLODE` clause; no surface change required) - - InQL RFC 001 (dataset types — `explode` method on `DataSet[T]`; no surface change required) -- **Issue:** [InQL #14](https://github.com/encero-systems/InQL/issues/14) + - IncQL RFC 002 (Apache Substrait — normative gap classification for unnest; **prerequisite**) + - IncQL RFC 003 (`query {}` — `EXPLODE` clause; no surface change required) + - IncQL RFC 001 (dataset types — `explode` method on `DataSet[T]`; no surface change required) +- **Issue:** [IncQL #14](https://github.com/encero-systems/IncQL/issues/14) - **RFC PR:** - - **Written against:** Incan v0.2 - **Shipped in:** - -> **Blocked** on upstream Apache Substrait standardizing a portable logical unnest/explode `Rel` in a revision InQL can pin to. See [Substrait operator catalog — Gap profiles: Unnest / explode][ref-operator-catalog]. +> **Blocked** on upstream Apache Substrait standardizing a portable logical unnest/explode `Rel` in a revision IncQL can pin to. See [Substrait operator catalog — Gap profiles: Unnest / explode][ref-operator-catalog]. ## Summary -InQL RFC 002 classifies `EXPLODE`/unnest as a **gap** capability: no stable logical `Rel` exists in core Substrait, so implementations must lower through a registered extension relation with a declared URI. This RFC records the intent to promote that capability from `gap` to `core` — updating the operator catalog, retiring the extension encoding requirement, and updating Incan compiler lowering — once upstream Substrait ships a portable unnest `Rel` that InQL can adopt. +IncQL RFC 002 classifies `EXPLODE`/unnest as a **gap** capability: no stable logical `Rel` exists in core Substrait, so implementations must lower through a registered extension relation with a declared URI. This RFC records the intent to promote that capability from `gap` to `core` — updating the operator catalog, retiring the extension encoding requirement, and updating Incan compiler lowering — once upstream Substrait ships a portable unnest `Rel` that IncQL can adopt. ## Motivation -The current extension encoding for unnest adds extension URI maintenance burden to every conforming InQL toolchain release and limits plan portability to consumers that happen to support the same registered extension. The gap classification is not a permanent design choice; it reflects a gap in Substrait at the time of RFC 002. Once upstream closes that gap with a stable logical `Rel`, there is no reason for InQL to keep the extension path as the normative encoding. Reclassifying promptly gives authors core-portable `EXPLODE` semantics without requiring consumers to register or recognize InQL-specific URIs. +The current extension encoding for unnest adds extension URI maintenance burden to every conforming IncQL toolchain release and limits plan portability to consumers that happen to support the same registered extension. The gap classification is not a permanent design choice; it reflects a gap in Substrait at the time of RFC 002. Once upstream closes that gap with a stable logical `Rel`, there is no reason for IncQL to keep the extension path as the normative encoding. Reclassifying promptly gives authors core-portable `EXPLODE` semantics without requiring consumers to register or recognize IncQL-specific URIs. ## Goals @@ -30,13 +30,13 @@ The current extension encoding for unnest adds extension URI maintenance burden ## Non-Goals -- Changing the InQL surface syntax for unnest — `EXPLODE` in `query {}` (InQL RFC 003) and `generate(explode(...))` remain unchanged. -- Defining the semantics of the new core `Rel` — that is an upstream Substrait concern; InQL aligns to whatever the pinned revision specifies. +- Changing the IncQL surface syntax for unnest — `EXPLODE` in `query {}` (IncQL RFC 003) and `generate(explode(...))` remain unchanged. +- Defining the semantics of the new core `Rel` — that is an upstream Substrait concern; IncQL aligns to whatever the pinned revision specifies. - Keeping the extension encoding as an alternate path — once the core `Rel` is adopted, the extension path is retired. ## Guide-level explanation -From an author's perspective, nothing changes. `EXPLODE` in `query {}` and `generate(explode(...))` work exactly as before. The only observable difference is in the serialized Substrait plan: before promotion, the emitted plan contains an `ExtensionSingleRel` or `ExtensionLeafRel` with an InQL-registered URI; after promotion, it contains the standard logical unnest `Rel` from the pinned Substrait revision. Consumers that previously required the InQL extension URI to execute unnest plans no longer do. +From an author's perspective, nothing changes. `EXPLODE` in `query {}` and `generate(explode(...))` work exactly as before. The only observable difference is in the serialized Substrait plan: before promotion, the emitted plan contains an `ExtensionSingleRel` or `ExtensionLeafRel` with an IncQL-registered URI; after promotion, it contains the standard logical unnest `Rel` from the pinned Substrait revision. Consumers that previously required the IncQL extension URI to execute unnest plans no longer do. ## Reference-level explanation @@ -66,14 +66,14 @@ Serialized Substrait plans containing the extension encoding for unnest will nee ## Design details -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces No surface changes. The `EXPLODE` clause in `query {}` and the `generate(explode(...))` dataset form retain their existing semantics; only the Substrait emission changes. ### Compatibility / migration - Breaking for serialized plans: existing plans with the extension encoding for unnest must be re-emitted. No author source code changes are required. -- Non-breaking for InQL source: `EXPLODE` and `generate(explode(...))` continue to compile and type-check identically. +- Non-breaking for IncQL source: `EXPLODE` and `generate(explode(...))` continue to compile and type-check identically. ## Alternatives considered @@ -83,18 +83,18 @@ No surface changes. The `EXPLODE` clause in `query {}` and the `generate(explode ## Drawbacks - Bumping the Substrait pin for this change is a plan-level breaking change, requiring coordinated consumer updates if any downstream tooling relies on the extension encoding. -- Timing depends entirely on upstream Substrait; InQL cannot control when a portable unnest `Rel` ships. +- Timing depends entirely on upstream Substrait; IncQL cannot control when a portable unnest `Rel` ships. ## Layers affected -- **InQL specification** — operator catalog reference updated; revision and extension policy retirement entry required. +- **IncQL specification** — operator catalog reference updated; revision and extension policy retirement entry required. - **Incan compiler** — lowering for `EXPLODE` / `explode` updated to emit the core `Rel` (work in the Incan repository). - **Documentation** — release notes entry; operator catalog update. ## Unresolved questions - Which exact Substrait revision introduces the portable unnest `Rel`? (Blocked on upstream; track `substrait-io/substrait`.) -- Are there semantic edge cases between the InQL extension encoding and the upstream core `Rel` that require a compatibility shim or a lowering-time rewrite? +- Are there semantic edge cases between the IncQL extension encoding and the upstream core `Rel` that require a compatibility shim or a lowering-time rewrite? diff --git a/docs/rfcs/007_prism_planning_engine.md b/docs/rfcs/007_prism_planning_engine.md index 55a74350..7e0713e4 100644 --- a/docs/rfcs/007_prism_planning_engine.md +++ b/docs/rfcs/007_prism_planning_engine.md @@ -1,30 +1,30 @@ -# InQL RFC 007: Prism logical planning and optimization engine +# IncQL RFC 007: Prism logical planning and optimization engine - **Status:** In Progress - **Created:** 2026-04-02 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 001 (dataset types and carriers — Prism-backed carriers must remain consistent with `DataSet[T]` semantics) - - InQL RFC 002 (Apache Substrait integration — Substrait remains the normative emitted contract at the boundary) - - InQL RFC 003 (`query {}` — lowers through Prism-managed logical work before Substrait emission) - - InQL RFC 004 (execution context — session executes Prism-backed plans but does not define Prism) - - InQL RFC 005 (optional pipe-forward — must stay Prism-consistent with equivalent surfaces) -- **Issue:** [InQL #16](https://github.com/encero-systems/InQL/issues/16) + - IncQL RFC 001 (dataset types and carriers — Prism-backed carriers must remain consistent with `DataSet[T]` semantics) + - IncQL RFC 002 (Apache Substrait integration — Substrait remains the normative emitted contract at the boundary) + - IncQL RFC 003 (`query {}` — lowers through Prism-managed logical work before Substrait emission) + - IncQL RFC 004 (execution context — session executes Prism-backed plans but does not define Prism) + - IncQL RFC 005 (optional pipe-forward — must stay Prism-consistent with equivalent surfaces) +- **Issue:** [IncQL #16](https://github.com/encero-systems/IncQL/issues/16) - **RFC PR:** — - **Written against:** Incan v0.2 - **Shipped in:** — ## Summary -This RFC defines **Prism** as InQL's immutable internal logical planning and optimization engine. Prism owns persistent plan storage, cheap branching through structural sharing, lineage-preserving rewrites, and logical optimization prior to Substrait emission or session execution. Prism is an **internal planning substrate**, not the normative interchange contract: **Apache Substrait** remains the boundary format per InQL RFC 002. `LazyFrame`, `DataFrame`, and `DataStream` are carrier experiences over Prism-managed plan state; `Session` and `SessionContext` bind and execute those plans per InQL RFC 004. +This RFC defines **Prism** as IncQL's immutable internal logical planning and optimization engine. Prism owns persistent plan storage, cheap branching through structural sharing, lineage-preserving rewrites, and logical optimization prior to Substrait emission or session execution. Prism is an **internal planning substrate**, not the normative interchange contract: **Apache Substrait** remains the boundary format per IncQL RFC 002. `LazyFrame`, `DataFrame`, and `DataStream` are carrier experiences over Prism-managed plan state; `Session` and `SessionContext` bind and execute those plans per IncQL RFC 004. -RFC 007 is the design and implementation record for the first Prism adoption slice. Optimizer-boundary ownership is further clarified by [InQL RFC 008](008_optimizer_boundary_stats_cbo_aqe.md): Prism owns immutable authored state, lineage-preserving logical work, and internal optimized views, while RFC 008 narrows the split with `Session` around backend-facing statistics, physical planning, and adaptive execution concerns. Follow-on RFC 007 hardening includes an Incan-native typed store-id allocator (`static` + `newtype`) and cross-store adoption dedup for equivalent reachable RHS nodes; this remains internal Prism substrate work and does not expand RFC 008 scope. +RFC 007 is the design and implementation record for the first Prism adoption slice. Optimizer-boundary ownership is further clarified by [IncQL RFC 008](008_optimizer_boundary_stats_cbo_aqe.md): Prism owns immutable authored state, lineage-preserving logical work, and internal optimized views, while RFC 008 narrows the split with `Session` around backend-facing statistics, physical planning, and adaptive execution concerns. Follow-on RFC 007 hardening includes an Incan-native typed store-id allocator (`static` + `newtype`) and cross-store adoption dedup for equivalent reachable RHS nodes; this remains internal Prism substrate work and does not expand RFC 008 scope. ## Motivation -InQL already has a strong external story around typed carriers, Substrait emission, and the execution boundary, but it lacks a dedicated specification for the internal planning layer that sits between authored logic and emitted plans. Without that layer being named and scoped, plan construction, optimization, lineage, interactive behavior, and future explain/debug tooling risk becoming an accidental mix of implementation details spread across InQL RFC 001, InQL RFC 002, and InQL RFC 004. +IncQL already has a strong external story around typed carriers, Substrait emission, and the execution boundary, but it lacks a dedicated specification for the internal planning layer that sits between authored logic and emitted plans. Without that layer being named and scoped, plan construction, optimization, lineage, interactive behavior, and future explain/debug tooling risk becoming an accidental mix of implementation details spread across IncQL RFC 001, IncQL RFC 002, and IncQL RFC 004. -Prism gives that layer a home. It lets InQL say clearly that: +Prism gives that layer a home. It lets IncQL say clearly that: - authored transformations build immutable logical plans - carriers stay cheap by sharing planning state instead of cloning whole plans @@ -35,24 +35,24 @@ This matters for more than simple query lowering. Complex multi-hop pipelines, f ## Goals -- Define **Prism** as the immutable logical planning engine for InQL. +- Define **Prism** as the immutable logical planning engine for IncQL. - Specify Prism's core responsibilities: persistent plan storage, logical optimization, lineage preservation, and preparation for Substrait emission. -- Clarify the relationship between Prism and InQL carriers (`LazyFrame`, `DataFrame`, `DataStream`, `DataSet`). +- Clarify the relationship between Prism and IncQL carriers (`LazyFrame`, `DataFrame`, `DataStream`, `DataSet`). - Clarify the relationship between Prism and sibling boundaries: Substrait at interchange boundaries and `Session` / `SessionContext` at execution boundaries. - Require that Prism-backed plan construction remain cheap through structural sharing rather than deep-cloning carrier state. - Define the conceptual distinction between authored plan state and optimized plan state without over-constraining the final implementation. ## Non-Goals -- Replacing Apache Substrait as InQL's normative emitted logical contract — that remains InQL RFC 002. -- Defining physical execution behavior, backend binding, or secret management — that remains outside Prism and is scoped by InQL RFC 004 and surrounding operational layers. +- Replacing Apache Substrait as IncQL's normative emitted logical contract — that remains IncQL RFC 002. +- Defining physical execution behavior, backend binding, or secret management — that remains outside Prism and is scoped by IncQL RFC 004 and surrounding operational layers. - Defining new author-facing query syntax — Prism is an internal planning engine, not a new language surface. - Forcing one exact in-memory data structure implementation for authored and optimized plan state. -- Promising Prism as a general-purpose platform beyond InQL today. This RFC scopes Prism normatively to InQL; future extraction remains a possible consequence of a clean boundary, not a current requirement. +- Promising Prism as a general-purpose platform beyond IncQL today. This RFC scopes Prism normatively to IncQL; future extraction remains a possible consequence of a clean boundary, not a current requirement. ## Guide-level explanation -From an author's point of view, Prism is not something they use directly. Authors work with InQL carriers such as `LazyFrame[T]`, `DataFrame[T]`, and (later) `DataStream[T]`. Those carriers build or operate over logical work that Prism stores and optimizes internally. +From an author's point of view, Prism is not something they use directly. Authors work with IncQL carriers such as `LazyFrame[T]`, `DataFrame[T]`, and (later) `DataStream[T]`. Those carriers build or operate over logical work that Prism stores and optimizes internally. ```incan orders: LazyFrame[Order] = session.table("orders")? @@ -79,7 +79,7 @@ Prism should be thought of as the internal engine that **thinks** about the plan ### Prism role -Prism is the internal logical planning and optimization substrate for InQL. +Prism is the internal logical planning and optimization substrate for IncQL. Prism **must**: @@ -109,13 +109,13 @@ The relationship is: - Prism = internal logical planning, lineage, and optimization - Substrait = emitted logical interchange contract -An implementation **may** use Prism-native node kinds or derived optimized views internally, but emitted plans that claim conformance **must** still follow InQL RFC 002. +An implementation **may** use Prism-native node kinds or derived optimized views internally, but emitted plans that claim conformance **must** still follow IncQL RFC 002. ### Relationship to session execution Prism does not execute plans. `Session` / `SessionContext` own execution. -Execution-oriented flows **must** treat Prism as an input to lowering and execution, not as the executor itself. Session-backed operations may request optimized views from Prism before emission or execution, but the existence of Prism **must not** collapse the execution boundary defined in InQL RFC 004. +Execution-oriented flows **must** treat Prism as an input to lowering and execution, not as the executor itself. Session-backed operations may request optimized views from Prism before emission or execution, but the existence of Prism **must not** collapse the execution boundary defined in IncQL RFC 004. ### Authored state vs optimized state @@ -162,7 +162,7 @@ This RFC introduces no new author-facing syntax. ### Semantics -Prism is the internal engine that owns logical planning and optimization for InQL carriers. +Prism is the internal engine that owns logical planning and optimization for IncQL carriers. At minimum, a Prism-backed carrier should be representable as: @@ -172,25 +172,25 @@ At minimum, a Prism-backed carrier should be representable as: The exact representation is intentionally not fixed by this RFC, but the semantics of immutability, structural sharing, and lineage preservation are. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces -- **`DataSet[T]` APIs:** method-chain surfaces defined by InQL RFC 001 **must** build or manipulate Prism-backed logical state without violating carrier immutability. -- **`query {}`:** checked query blocks defined by InQL RFC 003 **should** lower into Prism-managed logical work before final Substrait emission. -- **Pipe-forward (`|>`):** if supported per InQL RFC 005, desugared pipe-forward **must** remain Prism-consistent with the equivalent method-chain or query-block form. +- **`DataSet[T]` APIs:** method-chain surfaces defined by IncQL RFC 001 **must** build or manipulate Prism-backed logical state without violating carrier immutability. +- **`query {}`:** checked query blocks defined by IncQL RFC 003 **should** lower into Prism-managed logical work before final Substrait emission. +- **Pipe-forward (`|>`):** if supported per IncQL RFC 005, desugared pipe-forward **must** remain Prism-consistent with the equivalent method-chain or query-block form. - **Incan `model` types:** Prism optimization legality **must** remain consistent with model-derived schema semantics and must not fall back to runtime-authored schema truth. -- **Substrait / execution:** Prism prepares plans for InQL RFC 002 emission and InQL RFC 004 execution, but it does not replace either sibling boundary. +- **Substrait / execution:** Prism prepares plans for IncQL RFC 002 emission and IncQL RFC 004 execution, but it does not replace either sibling boundary. ### Compatibility / migration -This RFC is additive and architectural. It clarifies and stabilizes internal InQL planning semantics; it does not by itself introduce a source-level breaking change for authors or a serialized-plan breaking change for Substrait consumers. +This RFC is additive and architectural. It clarifies and stabilizes internal IncQL planning semantics; it does not by itself introduce a source-level breaking change for authors or a serialized-plan breaking change for Substrait consumers. It may, however, motivate refactoring of implementation architecture so that planning, optimization, and emission concerns are separated more clearly than they were before this RFC existed. ## Alternatives considered - **Keep Prism as a research note only** — rejected for now; the planning and optimization substrate is foundational enough that leaving it undocumented as an implementation note would keep key architectural boundaries implicit. -- **Fold Prism fully into InQL RFC 002** — rejected; Substrait emission and internal planning are related but distinct concerns. Keeping them in one RFC makes the internal engine look like a boundary-format detail. -- **Define Prism as a cross-cutting platform beyond InQL immediately** — rejected for now; Prism may eventually be reused elsewhere, but this RFC keeps the normative scope concrete by defining Prism first as an InQL component with a clean standalone module boundary. +- **Fold Prism fully into IncQL RFC 002** — rejected; Substrait emission and internal planning are related but distinct concerns. Keeping them in one RFC makes the internal engine look like a boundary-format detail. +- **Define Prism as a cross-cutting platform beyond IncQL immediately** — rejected for now; Prism may eventually be reused elsewhere, but this RFC keeps the normative scope concrete by defining Prism first as an IncQL component with a clean standalone module boundary. ## Drawbacks @@ -200,9 +200,9 @@ It may, however, motivate refactoring of implementation architecture so that pla ## Layers affected -- **InQL specification** — sibling RFCs that reference logical planning, carrier behavior, Substrait lowering, or session execution **should** remain consistent with Prism as the internal planning substrate. -- **InQL library package** — public carriers and internal planning modules **should** preserve immutable carrier semantics over shared Prism-managed state. -- **Incan compiler** — if InQL surfaces lower through compiler-managed intermediate representations, those integrations **should** respect Prism's lineage and optimization invariants. +- **IncQL specification** — sibling RFCs that reference logical planning, carrier behavior, Substrait lowering, or session execution **should** remain consistent with Prism as the internal planning substrate. +- **IncQL library package** — public carriers and internal planning modules **should** preserve immutable carrier semantics over shared Prism-managed state. +- **Incan compiler** — if IncQL surfaces lower through compiler-managed intermediate representations, those integrations **should** respect Prism's lineage and optimization invariants. - **Execution / interchange** — Session-backed lowering and execution flows **must** treat Prism as internal preparation and Substrait as the boundary contract. - **Documentation** — RFC indexes, architecture notes, and implementation planning notes **should** distinguish Prism from Substrait and from session execution. @@ -301,7 +301,7 @@ It may, however, motivate refactoring of implementation architecture so that pla - The first Prism slice commits only to safe logical rewrites: projection pruning, predicate pushdown, redundant-node elimination, normalization of equivalent logical shapes, and optional simple shared-subplan detection. Heavier work such as join reordering, cost-based optimization, and sink-aware splitting is explicitly deferred. - The minimum lineage contract is stable authored node IDs plus optimized-to-authored origin mappings. Richer explain/debug structures may be added later, but they are not required for the RFC to be complete. - RFC 007 does not require a new upstream Incan RFC before moving to `Planned`. Implementation may expose compiler or tooling gaps later, but those are implementation dependencies rather than specification blockers. -- Prism remains an internal InQL planning substrate for now; the first implementation does not expose public `Prism*` package APIs. +- Prism remains an internal IncQL planning substrate for now; the first implementation does not expose public `Prism*` package APIs. - `LazyFrame[T]` is the first real Prism-backed carrier. `DataFrame[T]`, `DataStream[T]`, and `query {}` integration remain follow-on work unless the `LazyFrame[T]` slice proves a hard dependency. - `PrismCursor[T]` is the current backend-native handle beneath `LazyFrame[T]`. It is an internal convergence target for future `query {}` and pipe-forward lowering, not a public package API. - The research prototype demonstrated the seam, but its clone-heavy storage and same-graph-only join restriction are not the intended production design. diff --git a/docs/rfcs/008_optimizer_boundary_stats_cbo_aqe.md b/docs/rfcs/008_optimizer_boundary_stats_cbo_aqe.md index e6e8c1ed..db8bec81 100644 --- a/docs/rfcs/008_optimizer_boundary_stats_cbo_aqe.md +++ b/docs/rfcs/008_optimizer_boundary_stats_cbo_aqe.md @@ -1,29 +1,29 @@ -# InQL RFC 008: Optimizer boundary, statistics, cost-based optimization, and adaptive execution +# IncQL RFC 008: Optimizer boundary, statistics, cost-based optimization, and adaptive execution - **Status:** Planned - **Created:** 2026-04-07 - **Author(s):** Danny Meijer - **Related:** - - InQL RFC 004 (execution context — `Session` remains the execution and backend boundary) - - InQL RFC 007 (Prism planning engine — this RFC narrows optimizer-boundary ownership without replacing Prism adoption) -- **Issue:** [InQL #18](https://github.com/encero-systems/InQL/issues/18) + - IncQL RFC 004 (execution context — `Session` remains the execution and backend boundary) + - IncQL RFC 007 (Prism planning engine — this RFC narrows optimizer-boundary ownership without replacing Prism adoption) +- **Issue:** [IncQL #18](https://github.com/encero-systems/IncQL/issues/18) - **RFC PR:** — - **Written against:** Incan v0.2 - **Shipped in:** — ## Summary -This RFC defines the optimizer boundary between **Prism** and **`Session`** as InQL grows beyond the first Prism adoption slice. Prism remains the owner of analyzed logical planning, semantic rewrites, canonicalization, schema-preserving logical optimization, and any static planning facts that do not depend on runtime feedback. `Session` remains the owner of backend capabilities, physical planning, backend pushdown policy, runtime statistics, execution metrics, and adaptive re-planning during execution. This RFC does not replace RFC 007's role in establishing Prism as the internal planning substrate; it settles the ownership boundary needed for RFC 004 and defers deeper statistics, CBO, and AQE mechanics until the execution side is better grounded. +This RFC defines the optimizer boundary between **Prism** and **`Session`** as IncQL grows beyond the first Prism adoption slice. Prism remains the owner of analyzed logical planning, semantic rewrites, canonicalization, schema-preserving logical optimization, and any static planning facts that do not depend on runtime feedback. `Session` remains the owner of backend capabilities, physical planning, backend pushdown policy, runtime statistics, execution metrics, and adaptive re-planning during execution. This RFC does not replace RFC 007's role in establishing Prism as the internal planning substrate; it settles the ownership boundary needed for RFC 004 and defers deeper statistics, CBO, and AQE mechanics until the execution side is better grounded. ## Motivation -RFC 007 was intentionally written to get Prism named, scoped, and implemented as a real planning substrate. That was the right move for the first Prism adoption slice. However, once InQL aims for stronger optimization, the remaining ambiguity becomes a liability: +RFC 007 was intentionally written to get Prism named, scoped, and implemented as a real planning substrate. That was the right move for the first Prism adoption slice. However, once IncQL aims for stronger optimization, the remaining ambiguity becomes a liability: - If Prism owns all optimization in the abstract, it will tend to absorb backend policy and runtime behavior. - If `Session` owns all optimization in practice, Prism becomes a passive container rather than a serious optimizer substrate. - If statistics, cost-based optimization, and adaptive re-planning are not assigned cleanly, explain output, reproducibility, and backend substitution all become muddled. -InQL needs the same kind of separation that high-performance query engines converge on in practice: +IncQL needs the same kind of separation that high-performance query engines converge on in practice: - a semantic logical optimizer that can reason about equivalence and properties - a backend boundary that can exploit concrete storage and runtime facts @@ -41,7 +41,7 @@ This RFC intentionally stops at the minimum boundary needed to keep the architec - Define which optimizer responsibilities belong to Prism versus `Session`. - Establish **statistics ownership** clearly enough to support cost-based optimization without collapsing the Prism / `Session` boundary. -- Define the minimum optimizer artifacts Prism should expose as InQL moves past the initial identity-shaped optimized view. +- Define the minimum optimizer artifacts Prism should expose as IncQL moves past the initial identity-shaped optimized view. - Reserve adaptive query re-planning as a `Session` concern rather than a Prism concern. - Make precedence against RFC 007 explicit so future work does not rely on ambiguous wording. @@ -75,8 +75,8 @@ The important mental model is: Conceptual example; exact explain API names may differ: ```incan -from pub::inql import Session, LazyFrame, DataFrame -from pub::inql.functions import count +from pub::incql import Session, LazyFrame, DataFrame +from pub::incql.functions import count from models import Customer, Order, RegionalSummary session = Session.default() @@ -97,7 +97,7 @@ summary: LazyFrame[RegionalSummary] = query { # Prism-owned logical surfaces summary.raw_plan() summary.analyzed_plan() -summary.plan_after_inql_rules() +summary.plan_after_incql_rules() # Session-owned execution and runtime behavior (current slice uses execute/write) executed: LazyFrame[RegionalSummary] = session.execute(summary)? @@ -106,7 +106,7 @@ session.session_plan(summary) session.executed_plan(summary) ``` -For explain and tooling, InQL should make the stages explicit instead of collapsing them into one vague “optimized plan” label. A future author or tool should be able to distinguish at least: +For explain and tooling, IncQL should make the stages explicit instead of collapsing them into one vague “optimized plan” label. A future author or tool should be able to distinguish at least: - authored Prism state - analyzed Prism state @@ -160,13 +160,13 @@ Ownership rules: ### 3. Cost-based optimization -Cost-based optimization in InQL is split across Prism and `Session`: +Cost-based optimization in IncQL is split across Prism and `Session`: - Prism **should** compare logical alternatives using a stable cost interface over logical properties, available statistics, and backend capability hints. - `Session` **may** provide the statistics and capability inputs Prism needs to make better logical choices. - The exact cost model is not standardized by this RFC, but the ownership boundary is. -Inference from this boundary: if InQL later adds join reordering, memo-based exploration, or reuse/materialization decisions, those features belong primarily in Prism, but they rely on inputs that `Session` can provide. +Inference from this boundary: if IncQL later adds join reordering, memo-based exploration, or reuse/materialization decisions, those features belong primarily in Prism, but they rely on inputs that `Session` can provide. ### 4. Adaptive execution @@ -187,7 +187,7 @@ Illustrative names: - `raw_plan()` - `analyzed_plan()` -- `plan_after_inql_rules()` +- `plan_after_incql_rules()` - `session_plan()` - `executed_plan()` @@ -197,7 +197,7 @@ Equivalent names are acceptable if they preserve the same separation. The exact RFC 007 remains authoritative for: -- Prism as InQL's internal planning substrate +- Prism as IncQL's internal planning substrate - immutable authored state - structural sharing - lineage and optimized-to-authored provenance expectations @@ -228,11 +228,11 @@ As Prism evolves beyond the initial implementation slice, the intended logical s - optional logical alternative exploration and cost comparison - `Session` handoff for backend planning and execution -Prism's optimizer legality **must** continue to derive from InQL semantics, schema facts, and expression rules rather than backend quirks. +Prism's optimizer legality **must** continue to derive from IncQL semantics, schema facts, and expression rules rather than backend quirks. `Session` **may** pass backend capability hints and statistics into Prism before execution, but those inputs **must not** collapse the ownership boundary defined above. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces - **`DataSet[T]` APIs:** carrier method chains continue to build Prism-managed authored state. Stronger optimization does not change carrier immutability rules. - **`query {}`:** query-block lowering should target the same Prism analysis and rewrite pipeline as method chains. @@ -260,20 +260,20 @@ Existing prototype APIs that use vague names like `optimized_view` remain accept ## Alternatives considered - **Keep RFC 007 as the only optimizer RFC** — rejected; RFC 007 already serves as the Prism adoption record for the first implementation slice, and retrofitting a more detailed optimizer boundary into it would mix historical adoption work with the follow-on architecture. -- **Move all optimization to `Session`** — rejected; that would reduce Prism to a plan container and make InQL-owned semantic optimization too backend-dependent. +- **Move all optimization to `Session`** — rejected; that would reduce Prism to a plan container and make IncQL-owned semantic optimization too backend-dependent. - **Move AQE into Prism** — rejected; adaptive re-planning depends on runtime execution facts and should remain session-owned. - **Treat statistics as purely backend-private** — rejected; Prism needs a clean way to consume statistics if it is going to perform serious cost-based logical optimization. ## Drawbacks - Adds another foundational RFC and another precedence edge contributors must understand. -- Commits InQL to a sharper optimizer vocabulary earlier than a minimal prototype would require. +- Commits IncQL to a sharper optimizer vocabulary earlier than a minimal prototype would require. - Memo-based exploration, property inference, and stats plumbing will increase implementation complexity once work begins. ## Layers affected -- **InQL specification** — RFC 004 and RFC 007 references to optimization ownership **should** stay consistent with this boundary. -- **InQL library package** — future Prism internals **should** separate authored, analyzed, and rewritten artifacts more explicitly than the current prototype does. +- **IncQL specification** — RFC 004 and RFC 007 references to optimization ownership **should** stay consistent with this boundary. +- **IncQL library package** — future Prism internals **should** separate authored, analyzed, and rewritten artifacts more explicitly than the current prototype does. - **Execution / interchange** — `Session` and backend integration layers **must** own physical planning, runtime stats, and adaptive re-planning policy. - **Documentation** — explain surfaces and architecture notes **should** stop using “optimized plan” as an undifferentiated term. diff --git a/docs/rfcs/009_session_format_handler_registry.md b/docs/rfcs/009_session_format_handler_registry.md index 91e89e27..d4e2a7eb 100644 --- a/docs/rfcs/009_session_format_handler_registry.md +++ b/docs/rfcs/009_session_format_handler_registry.md @@ -1,4 +1,4 @@ -# InQL RFC 009: Session Format Handler Registry +# IncQL RFC 009: Session Format Handler Registry - **Status:** Draft - **Created:** 2026-04-18 @@ -11,7 +11,7 @@ ## Summary -This RFC introduces a Session-owned format handler registry so InQL can support built-in and third-party source formats through one stable contract, instead of hardcoding format-specific branches in Session and backend integration code. +This RFC introduces a Session-owned format handler registry so IncQL can support built-in and third-party source formats through one stable contract, instead of hardcoding format-specific branches in Session and backend integration code. ## Motivation @@ -26,7 +26,7 @@ A handler registry makes format support extensible without destabilizing Session ## Goals -- Define a stable InQL-level contract for source format handlers. +- Define a stable IncQL-level contract for source format handlers. - Route built-in formats through the same handler contract as custom formats. - Let Session resolve format behavior by format key, not hardcoded branching. - Preserve typed Session errors and stable diagnostics when handlers fail. @@ -44,7 +44,7 @@ A handler registry makes format support extensible without destabilizing Session Authors should be able to register a handler and then read data through that format key. ```incan -from pub::inql import LazyFrame, Session +from pub::incql import LazyFrame, Session from session.formats import SessionFormatHandler class FooFormatHandler with SessionFormatHandler: @@ -92,7 +92,7 @@ Errors and diagnostics: ### Syntax -No new core language syntax is required. This RFC adds InQL library API surface. +No new core language syntax is required. This RFC adds IncQL library API surface. ### Semantics @@ -104,7 +104,7 @@ A format handler owns format-specific adaptation only: A format handler does not own logical planning or optimization semantics. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces - RFC 004: Session execution boundaries remain unchanged; format handlers only provide source adaptation. - RFC 007: Prism remains backend-agnostic at logical planning level; handlers influence source setup before execution, not logical rewrite strategy. @@ -130,8 +130,8 @@ A format handler does not own logical planning or optimization semantics. ## Layers affected -- **InQL specification**: RFC 004 alignment must remain explicit; Session format behavior becomes contract-based. -- **InQL library package**: Session API must add handler registration and format-key read path; built-ins must be re-expressed as handlers. +- **IncQL specification**: RFC 004 alignment must remain explicit; Session format behavior becomes contract-based. +- **IncQL library package**: Session API must add handler registration and format-key read path; built-ins must be re-expressed as handlers. - **Incan compiler**: no mandatory syntax/parser work in first slice; typechecking/emission must continue to support method calls and generic read APIs used by handler-based Session flows. - **Execution / interchange**: source registration path must be handler-driven before backend planning/execution. - **Documentation**: Session and execution-context docs must describe handler registration and format-key behavior. diff --git a/docs/rfcs/010_csv_ingestion_contract.md b/docs/rfcs/010_csv_ingestion_contract.md index afc54d0a..c2fd5ffb 100644 --- a/docs/rfcs/010_csv_ingestion_contract.md +++ b/docs/rfcs/010_csv_ingestion_contract.md @@ -1,12 +1,12 @@ -# InQL RFC 010: CSV dialect and interpretation contract +# IncQL RFC 010: CSV dialect and interpretation contract - **Status:** Draft - **Created:** 2026-04-19 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 001 (dataset types and carrier schema surfaces) - - InQL RFC 004 (execution context and session read boundaries) - - InQL RFC 009 (session format handler registry) + - IncQL RFC 001 (dataset types and carrier schema surfaces) + - IncQL RFC 004 (execution context and session read boundaries) + - IncQL RFC 009 (session format handler registry) - **Issue:** — - **RFC PR:** — - **Written against:** Incan v0.2-rc5 @@ -14,23 +14,23 @@ ## Summary -This RFC defines InQL's north-star CSV dialect and interpretation contract. It standardizes how authors describe CSV dialect, header policy, malformed-row behavior, and schema or type inference so that the `csv` format behaves predictably across execution backends and future format-handler implementations. The core claim is that CSV parsing semantics must be expressed as stable structured InQL configuration and validated as part of the InQL contract, rather than being left to whatever parser quirks a specific backend happens to expose. +This RFC defines IncQL's north-star CSV dialect and interpretation contract. It standardizes how authors describe CSV dialect, header policy, malformed-row behavior, and schema or type inference so that the `csv` format behaves predictably across execution backends and future format-handler implementations. The core claim is that CSV parsing semantics must be expressed as stable structured IncQL configuration and validated as part of the IncQL contract, rather than being left to whatever parser quirks a specific backend happens to expose. ## Core model 1. CSV ingestion configuration is structured data, not ad-hoc booleans or backend-specific string flags. 2. CSV **dialect**, **schema or type interpretation**, and **malformed-row policy** are distinct concerns and must remain separate in the public API. -3. CSV reads participate in InQL's schema layering: declared schema, planned schema, and resolved schema are related but not interchangeable. +3. CSV reads participate in IncQL's schema layering: declared schema, planned schema, and resolved schema are related but not interchangeable. 4. Execution backends and format handlers must either honor the requested CSV contract or reject unsupported configuration before query execution begins. 5. `Session.read_csv(...)` is convenience sugar over the session's format-dispatch surface for the `csv` format key. ## Motivation -CSV is deceptively simple. In practice, a read surface that says "read CSV" without a precise contract leaves critical behavior undefined: quoting, embedded delimiters, multiline fields, header handling, null tokens, numeric grouping, timestamp recognition, and malformed-row policy. That creates two problems for InQL. +CSV is deceptively simple. In practice, a read surface that says "read CSV" without a precise contract leaves critical behavior undefined: quoting, embedded delimiters, multiline fields, header handling, null tokens, numeric grouping, timestamp recognition, and malformed-row policy. That creates two problems for IncQL. -First, authors cannot reason about portability. A workflow that succeeds with one execution backend may silently change meaning under another backend or after an internal parser change. Second, InQL's typed carrier model becomes weaker if CSV ingestion can quietly reinterpret columns or infer different shapes based on parser quirks rather than an explicit contract. +First, authors cannot reason about portability. A workflow that succeeds with one execution backend may silently change meaning under another backend or after an internal parser change. Second, IncQL's typed carrier model becomes weaker if CSV ingestion can quietly reinterpret columns or infer different shapes based on parser quirks rather than an explicit contract. -InQL should not inherit the accidental complexity of backend-specific CSV readers as its user-facing semantics. It needs its own contract: explicit enough that authors can rely on it, and precise enough that conformance tests can verify it. +IncQL should not inherit the accidental complexity of backend-specific CSV readers as its user-facing semantics. It needs its own contract: explicit enough that authors can rely on it, and precise enough that conformance tests can verify it. ## Goals @@ -39,7 +39,7 @@ InQL should not inherit the accidental complexity of backend-specific CSV reader - Specify the default CSV behavior authors get when they do not override options. - Define how CSV reads interact with declared, planned, and resolved schema layers. - Require early rejection when a backend or format handler cannot satisfy the requested CSV contract. -- Make conformance testing possible without binding InQL semantics to one specific parser implementation. +- Make conformance testing possible without binding IncQL semantics to one specific parser implementation. ## Non-Goals @@ -61,8 +61,8 @@ Authors should think of CSV reads as a contract between their program and the ru The public surface should therefore expose one structured options value for CSV reads. ```incan -from pub::inql import LazyFrame, Session -from pub::inql.formats import ( +from pub::incql import LazyFrame, Session +from pub::incql.formats import ( CsvDialect, CsvHeaderMode, CsvInference, @@ -116,7 +116,7 @@ orders: LazyFrame[Order] = session.read_format( ) ``` -If an author does not provide options, InQL still has a defined default contract rather than an implementation accident. +If an author does not provide options, IncQL still has a defined default contract rather than an implementation accident. ```incan orders: LazyFrame[Order] = session.read_csv("orders", "s3://warehouse/orders.csv") @@ -127,7 +127,7 @@ That default should mean "standard CSV with headers, quoted-field support, stric Headerless CSV is still allowed, but it should be explicit because it changes how schema is established. ```incan -from pub::inql.formats import CsvDialect, CsvHeaderMode, CsvReadOptions +from pub::incql.formats import CsvDialect, CsvHeaderMode, CsvReadOptions rows: LazyFrame[Order] = session.read_csv( "orders", @@ -142,7 +142,7 @@ rows: LazyFrame[Order] = session.read_csv( ### Public configuration model -InQL must expose one stable structured CSV configuration surface for read operations. The exact namespace may evolve, but the public API must include the equivalent of: +IncQL must expose one stable structured CSV configuration surface for read operations. The exact namespace may evolve, but the public API must include the equivalent of: - `CsvReadOptions` - `CsvDialect` @@ -164,7 +164,7 @@ The normative contract is owned by the `csv` format entry in the session's forma - `header`: whether the first logical record is a header row - `multiline_fields`: whether quoted fields may span multiple physical lines -InQL's default dialect must behave as follows: +IncQL's default dialect must behave as follows: - delimiter is `,` - quote is `"` @@ -173,14 +173,14 @@ InQL's default dialect must behave as follows: - quoted fields may contain delimiters and line breaks - malformed rows are not silently ignored -The default contract should align with standard CSV expectations, but InQL owns the contract text. It must not outsource semantics to a backend documentation page. +The default contract should align with standard CSV expectations, but IncQL owns the contract text. It must not outsource semantics to a backend documentation page. ### Schema and type interpretation CSV ingestion must distinguish three schema layers: - **declared schema**: the author- or model-level schema the program is written against -- **planned schema**: the schema InQL can establish before execution from configuration, headers, and compatible schema analysis +- **planned schema**: the schema IncQL can establish before execution from configuration, headers, and compatible schema analysis - **resolved schema**: the schema observed from materialized output after execution When the call context requires `LazyFrame[T]`, `T` is the declared schema contract. CSV ingestion must validate compatibility between the file and `T` according to the configured header and inference policy. A backend must not silently reorder columns or reinterpret field names in a way that breaks the declared schema. @@ -197,7 +197,7 @@ If numeric grouping separators are allowed, the contract must define which separ ### Malformed-row policy -InQL must expose an explicit malformed-row policy. At minimum the contract must support an `Error` mode that fails the read. Additional policies may exist, but they must be explicit and portable. +IncQL must expose an explicit malformed-row policy. At minimum the contract must support an `Error` mode that fails the read. Additional policies may exist, but they must be explicit and portable. Malformed input includes at least: @@ -210,9 +210,9 @@ If a backend cannot implement a requested malformed-row policy faithfully, it mu ### Early validation and capability rejection -CSV option validation must happen before query execution starts. InQL must reject invalid option combinations and unsupported backend capabilities at the session or planning boundary. +CSV option validation must happen before query execution starts. IncQL must reject invalid option combinations and unsupported backend capabilities at the session or planning boundary. -Backends and format handlers may support more behavior internally than the portable contract defines, but that additional behavior must not leak into InQL's normative semantics unless a future RFC standardizes it. +Backends and format handlers may support more behavior internally than the portable contract defines, but that additional behavior must not leak into IncQL's normative semantics unless a future RFC standardizes it. ### Diagnostics @@ -238,13 +238,13 @@ This RFC does not introduce new language grammar. It standardizes library-surfac The default path must be strict enough to be predictable. Convenience cannot come from ambiguity. A permissive parser mode may exist, but only as an explicit policy choice. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces -CSV ingestion must compose with carrier-schema layering from InQL RFC 001. `planned_columns` may be established from declared schema, header contract, or compatible pre-execution analysis, while `resolved_columns` remain a runtime fact on materialized output. +CSV ingestion must compose with carrier-schema layering from IncQL RFC 001. `planned_columns` may be established from declared schema, header contract, or compatible pre-execution analysis, while `resolved_columns` remain a runtime fact on materialized output. -CSV ingestion must compose with the execution boundary from InQL RFC 004. Session-owned read configuration is part of the execution contract, not an implementation detail hidden in a backend adapter. +CSV ingestion must compose with the execution boundary from IncQL RFC 004. Session-owned read configuration is part of the execution contract, not an implementation detail hidden in a backend adapter. -CSV-specific configuration should also remain compatible with the format-handler model in InQL RFC 009. The `csv` format entry owns the contract; `read_csv` is convenience sugar over that dispatch path. A CSV handler may implement the contract, but it does not get to redefine the contract. +CSV-specific configuration should also remain compatible with the format-handler model in IncQL RFC 009. The `csv` format entry owns the contract; `read_csv` is convenience sugar over that dispatch path. A CSV handler may implement the contract, but it does not get to redefine the contract. ### Compatibility / migration @@ -254,10 +254,10 @@ Implementations should provide a clear migration path for currently implicit def ## Alternatives considered -- Leaving CSV semantics to the execution backend: rejected, because it makes InQL's read surface non-portable and weakens the typed carrier contract. +- Leaving CSV semantics to the execution backend: rejected, because it makes IncQL's read surface non-portable and weakens the typed carrier contract. - Defining only a minimal RFC 4180 subset and deferring all inference semantics: rejected, because authors still need stable rules for nulls, numerics, booleans, and timestamps when reading typed data. -- Copying pandas or Spark behavior wholesale: rejected, because those systems optimize for their own ecosystems and backward-compatibility constraints. InQL should learn from them, not inherit them uncritically. -- Exposing only backend-specific option maps: rejected, because it would bypass InQL's role as the contract owner. +- Copying pandas or Spark behavior wholesale: rejected, because those systems optimize for their own ecosystems and backward-compatibility constraints. IncQL should learn from them, not inherit them uncritically. +- Exposing only backend-specific option maps: rejected, because it would bypass IncQL's role as the contract owner. ## Drawbacks @@ -267,14 +267,14 @@ Implementations should provide a clear migration path for currently implicit def ## Layers affected -- **InQL specification** must define CSV ingestion as a portable contract rather than a backend-specific convenience. -- **InQL library package** must expose stable structured CSV read options and typed diagnostics consistent with this RFC. +- **IncQL specification** must define CSV ingestion as a portable contract rather than a backend-specific convenience. +- **IncQL library package** must expose stable structured CSV read options and typed diagnostics consistent with this RFC. - **Execution / interchange** must validate backend or handler capability against the requested CSV contract before execution begins. - **Documentation** must explain the default CSV contract and its explicit override model without relying on backend documentation as the source of truth. ## Unresolved questions -- Should headerless CSV be supported only when a declared schema is present, or may InQL synthesize stable placeholder column names as part of the portable contract? +- Should headerless CSV be supported only when a declared schema is present, or may IncQL synthesize stable placeholder column names as part of the portable contract? - Should the default numeric-grouping policy remain strict and reject separators such as `_` and `,`, or should common grouping separators be allowed by default? - Should unquoted leading and trailing whitespace be preserved exactly by default, or normalized as part of the portable CSV contract? diff --git a/docs/rfcs/011_source_discovery_contract.md b/docs/rfcs/011_source_discovery_contract.md index 57821e3e..962f5a45 100644 --- a/docs/rfcs/011_source_discovery_contract.md +++ b/docs/rfcs/011_source_discovery_contract.md @@ -1,12 +1,12 @@ -# InQL RFC 011: Source discovery and parse-unit expansion +# IncQL RFC 011: Source discovery and parse-unit expansion - **Status:** Draft - **Created:** 2026-04-19 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 004 (execution context and session read boundaries) - - InQL RFC 009 (session format handler registry) - - InQL RFC 010 (CSV dialect and interpretation contract) + - IncQL RFC 004 (execution context and session read boundaries) + - IncQL RFC 009 (session format handler registry) + - IncQL RFC 010 (CSV dialect and interpretation contract) - **Issue:** — - **RFC PR:** — - **Written against:** Incan v0.2-rc5 @@ -14,7 +14,7 @@ ## Summary -This RFC defines InQL's north-star source-discovery contract for file-backed reads. It standardizes how a logical input target becomes one or more parse units before any format-specific handler such as `csv`, `parquet`, or `arrow` interprets those units. The core claim is that input discovery must be its own contract layer, separate from format dialect or schema inference, so that InQL can describe single-file reads, directory or prefix expansion, and future container-aware discovery without mixing storage semantics into format semantics. +This RFC defines IncQL's north-star source-discovery contract for file-backed reads. It standardizes how a logical input target becomes one or more parse units before any format-specific handler such as `csv`, `parquet`, or `arrow` interprets those units. The core claim is that input discovery must be its own contract layer, separate from format dialect or schema inference, so that IncQL can describe single-file reads, directory or prefix expansion, and future container-aware discovery without mixing storage semantics into format semantics. ## Core model @@ -26,15 +26,15 @@ This RFC defines InQL's north-star source-discovery contract for file-backed rea ## Motivation -InQL already needs a clean distinction between "what is a CSV" and "what does this path mean." Those are not the same question. A folder, prefix, archive, or compressed object changes how input is enumerated, but it says nothing by itself about delimiter rules, headers, null tokens, or numeric inference. If those concerns are collapsed into a single format-specific options bag, the API becomes harder to reason about and portability becomes fragile. +IncQL already needs a clean distinction between "what is a CSV" and "what does this path mean." Those are not the same question. A folder, prefix, archive, or compressed object changes how input is enumerated, but it says nothing by itself about delimiter rules, headers, null tokens, or numeric inference. If those concerns are collapsed into a single format-specific options bag, the API becomes harder to reason about and portability becomes fragile. -This matters even before advanced runtime ingestion work exists. A simple read API still needs to answer whether a target is one file, many files, or some higher-level container that expands into parse units. Without a source-discovery contract, those semantics become accidental backend behavior. That is precisely the kind of ambiguity InQL should own and remove. +This matters even before advanced runtime ingestion work exists. A simple read API still needs to answer whether a target is one file, many files, or some higher-level container that expands into parse units. Without a source-discovery contract, those semantics become accidental backend behavior. That is precisely the kind of ambiguity IncQL should own and remove. At the same time, this space can expand too far if we are not disciplined. Incremental discovery, checkpoints, file notifications, and streaming progress semantics belong to a different RFC. This document is about source discovery and parse-unit expansion only. ## Goals -- Define a stable InQL-level contract for source discovery ahead of format parsing. +- Define a stable IncQL-level contract for source discovery ahead of format parsing. - Let source target kind drive default discovery behavior for the common path. - Distinguish single-target reads from multi-target expansion such as directory or prefix discovery. - Define parse-unit expansion as a format-agnostic concept that can be reused by CSV and other file-backed formats. @@ -51,15 +51,15 @@ At the same time, this space can expand too far if we are not disciplined. Incre ## Guide-level explanation (how authors think about it) -Authors should think about source discovery as one layer below format parsing. First, InQL figures out what concrete parse units exist. Then the selected format handler parses each unit according to its own contract. +Authors should think about source discovery as one layer below format parsing. First, IncQL figures out what concrete parse units exist. Then the selected format handler parses each unit according to its own contract. For the common path, authors should not have to restate obvious discovery behavior. A file target should imply one parse unit. A directory or prefix target should imply expansion. For a single file, the discovery contract is trivial. ```incan -from pub::inql import LazyFrame, Session -from pub::inql.sources import SourceDiscovery, SourceTarget +from pub::incql import LazyFrame, Session +from pub::incql.sources import SourceDiscovery, SourceTarget from models import Order session = Session.default() @@ -74,7 +74,7 @@ orders: LazyFrame[Order] = session.read_format( For a directory or prefix, the discovery contract becomes explicit: discover multiple files, then parse them individually under the same format contract. ```incan -from pub::inql.sources import ParseUnitExpansion, SourceDiscovery, SourceTarget +from pub::incql.sources import ParseUnitExpansion, SourceDiscovery, SourceTarget orders: LazyFrame[Order] = session.read_format( logical_name="orders", @@ -89,7 +89,7 @@ If the default implied by target kind is not sufficient, an explicit discovery o ### Public configuration model -InQL must expose a stable structured source-discovery surface for file-backed reads. The exact namespace may evolve, but the public API must include the equivalent of: +IncQL must expose a stable structured source-discovery surface for file-backed reads. The exact namespace may evolve, but the public API must include the equivalent of: - `SourceTarget` - `SourceDiscovery` @@ -114,13 +114,13 @@ Source discovery must define how a logical target expands into parse units befor When the target is one concrete file or object, the default discovery result is one parse unit. -When the target is a directory, prefix, or file set, the default discovery result must be a set of individually parsed units. InQL must not define multi-file reads as raw byte concatenation unless a future RFC explicitly standardizes such behavior. +When the target is a directory, prefix, or file set, the default discovery result must be a set of individually parsed units. IncQL must not define multi-file reads as raw byte concatenation unless a future RFC explicitly standardizes such behavior. Discovery must remain format-agnostic. It may determine parse-unit boundaries, but it must not define delimiter, quoting, schema inference, or malformed-record rules. Those belong to the selected format contract. ### Explicit overrides -InQL may expose explicit discovery overrides, but those overrides must refine or replace default target-kind behavior in a well-defined way. They must not be required in cases where the target kind already determines the obvious default. +IncQL may expose explicit discovery overrides, but those overrides must refine or replace default target-kind behavior in a well-defined way. They must not be required in cases where the target kind already determines the obvious default. ### Capability validation @@ -128,11 +128,11 @@ If a backend or format handler cannot honor the requested discovery contract, it ### Ordering and determinism -If the discovery contract does not guarantee parse-unit ordering for a target class, that lack of ordering must be explicit. InQL must not leave discovered-unit ordering as an undocumented backend accident. +If the discovery contract does not guarantee parse-unit ordering for a target class, that lack of ordering must be explicit. IncQL must not leave discovered-unit ordering as an undocumented backend accident. ### Containers and archive-like targets -Container-aware discovery such as archive-member expansion may be supported in the future, but this RFC does not fully standardize that behavior. If an implementation supports container-aware discovery before a follow-on RFC exists, that support should be treated as implementation-specific rather than as portable InQL semantics. +Container-aware discovery such as archive-member expansion may be supported in the future, but this RFC does not fully standardize that behavior. If an implementation supports container-aware discovery before a follow-on RFC exists, that support should be treated as implementation-specific rather than as portable IncQL semantics. ## Design details @@ -144,17 +144,17 @@ This RFC does not introduce new core language grammar. It standardizes library-s Source discovery is the contract layer that maps a logical input target to parse units. Format handlers sit downstream from that mapping and interpret each unit according to their own contract. -That separation is not cosmetic. It is what lets InQL describe folder or prefix expansion once, instead of reinventing those semantics inside each format-specific API. +That separation is not cosmetic. It is what lets IncQL describe folder or prefix expansion once, instead of reinventing those semantics inside each format-specific API. The target kind should carry as much default meaning as possible. Discovery overrides are for non-default cases, not for making authors repeat what the target already says. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces -This RFC composes directly with InQL RFC 004 because Session-owned reads need a stable pre-execution boundary for input resolution. +This RFC composes directly with IncQL RFC 004 because Session-owned reads need a stable pre-execution boundary for input resolution. -This RFC composes with InQL RFC 009 because a format-handler registry should receive already-classified parse units or discovery configuration, not be forced to invent global discovery semantics independently per format. +This RFC composes with IncQL RFC 009 because a format-handler registry should receive already-classified parse units or discovery configuration, not be forced to invent global discovery semantics independently per format. -This RFC composes with InQL RFC 010 because CSV dialect and interpretation begin only after source discovery has established the parse units to which that dialect applies. The same pattern should hold for other format contracts as well. +This RFC composes with IncQL RFC 010 because CSV dialect and interpretation begin only after source discovery has established the parse units to which that dialect applies. The same pattern should hold for other format contracts as well. ### Compatibility / migration @@ -165,19 +165,19 @@ Implementations may begin with a small subset of target kinds while still adopti ## Alternatives considered - Fold discovery into each format-specific RFC: rejected, because that duplicates storage semantics across formats and makes the API harder to reason about. -- Leave discovery entirely backend-defined: rejected, because InQL would lose control over path meaning and parse-unit semantics. +- Leave discovery entirely backend-defined: rejected, because IncQL would lose control over path meaning and parse-unit semantics. - Combine discovery with future streaming ingestion/runtime semantics: rejected, because discovery and runtime progress are related but distinct design layers. ## Drawbacks - Adds another explicit configuration layer to read APIs. -- Forces InQL to decide where portability stops instead of inheriting backend behavior wholesale. +- Forces IncQL to decide where portability stops instead of inheriting backend behavior wholesale. - Raises design pressure around containers and archive-like sources before backend convergence exists. ## Layers affected -- **InQL specification** must define source discovery as a contract distinct from format parsing. -- **InQL library package** must expose structured discovery configuration for file-backed reads. +- **IncQL specification** must define source discovery as a contract distinct from format parsing. +- **IncQL library package** must expose structured discovery configuration for file-backed reads. - **Execution / interchange** must validate that the selected backend or handler can satisfy the requested discovery contract before execution starts. - **Documentation** must explain the distinction between source discovery and format interpretation. @@ -187,6 +187,6 @@ Implementations may begin with a small subset of target kinds while still adopti - Should directory and prefix expansion be one portable target class, or should they remain distinct because storage systems expose them differently? - What should the explicit discovery-override surface look like once target-kind defaults cover the common path? - Should parse-unit ordering for multi-file discovery be explicitly unspecified by default, or should the contract require stable ordering when the backend can provide it? -- How much of container-aware discovery should become portable InQL behavior, and how much should remain backend- or integration-specific until a dedicated follow-on RFC exists? +- How much of container-aware discovery should become portable IncQL behavior, and how much should remain backend- or integration-specific until a dedicated follow-on RFC exists? diff --git a/docs/rfcs/012_unified_scalar_expression_surface.md b/docs/rfcs/012_unified_scalar_expression_surface.md index b7ab244b..4a9d30b0 100644 --- a/docs/rfcs/012_unified_scalar_expression_surface.md +++ b/docs/rfcs/012_unified_scalar_expression_surface.md @@ -1,30 +1,30 @@ -# InQL RFC 012: Unified scalar expression surface +# IncQL RFC 012: Unified scalar expression surface - **Status:** Implemented - **Created:** 2026-04-22 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 001 (dataset carriers and method-chain API surface) - - InQL RFC 003 (`query {}` blocks and relational authoring) - - InQL RFC 004 (execution context and backend execution boundary) - - InQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 001 (dataset carriers and method-chain API surface) + - IncQL RFC 003 (`query {}` blocks and relational authoring) + - IncQL RFC 004 (execution context and backend execution boundary) + - IncQL RFC 007 (Prism logical planning and optimization engine) - Incan RFC 025 (multi-instantiation trait dispatch) - Incan RFC 028 (trait-based operator overloading) - Incan RFC 029 (union types and type narrowing) - Incan RFC 040 (scoped DSL glyph surfaces) - Incan RFC 045 (scoped DSL symbol surfaces) -- **Issue:** [InQL #25](https://github.com/encero-systems/InQL/issues/25) +- **Issue:** [IncQL #25](https://github.com/encero-systems/IncQL/issues/25) - **RFC PR:** — - **Written against:** Incan v0.3.0 - **Shipped in:** v0.1 ## Summary -This RFC defines a single canonical scalar expression model for row-level relational meaning in InQL. Filter predicates, computed projection values, grouping keys, and aggregate arguments must all be expressed through the same scalar expression surface, while aggregate outputs remain a distinct aggregate-measure layer. The goal is to replace split mini-DSLs for predicates, literals, and projection expressions with one coherent authoring and lowering contract that all InQL surfaces share. +This RFC defines a single canonical scalar expression model for row-level relational meaning in IncQL. Filter predicates, computed projection values, grouping keys, and aggregate arguments must all be expressed through the same scalar expression surface, while aggregate outputs remain a distinct aggregate-measure layer. The goal is to replace split mini-DSLs for predicates, literals, and projection expressions with one coherent authoring and lowering contract that all IncQL surfaces share. ## Motivation -InQL has reached the point where split builder surfaces are becoming a design liability rather than a harmless implementation detail. Filters, computed projections, grouping keys, and aggregate arguments all describe row-level meaning, but they are currently easy to model as separate families because features landed incrementally. That split creates three problems. +IncQL has reached the point where split builder surfaces are becoming a design liability rather than a harmless implementation detail. Filters, computed projections, grouping keys, and aggregate arguments all describe row-level meaning, but they are currently easy to model as separate families because features landed incrementally. That split creates three problems. First, it makes the author contract harder to understand. Authors have to learn which helper family belongs to which surface even when the underlying intent is the same. A numeric literal used in a filter and a numeric literal used in a computed projection should not feel like different concepts. @@ -32,16 +32,16 @@ Second, it encourages duplicated or drifting semantics across package layers. If Third, split expression families produce the worst kind of failure mode: silent degradation. If a public method accepts a broad expression type but only truly supports direct column references in that position, unsupported shapes can be dropped or rewritten instead of rejected explicitly. A query library must not treat "unsupported" as "quietly mean something else." -This RFC is also needed before InQL can take proper advantage of the scoped DSL work in Incan. RFC 040 and RFC 045 create a path toward concise surfaces such as `.amount > 100` or ambient `sum(.amount)`, but those surfaces need one canonical lowering target. Without that, InQL would accumulate parallel semantic paths instead of one coherent expression system. +This RFC is also needed before IncQL can take proper advantage of the scoped DSL work in Incan. RFC 040 and RFC 045 create a path toward concise surfaces such as `.amount > 100` or ambient `sum(.amount)`, but those surfaces need one canonical lowering target. Without that, IncQL would accumulate parallel semantic paths instead of one coherent expression system. ## Goals -- Define one canonical scalar expression model for row-level relational authoring in InQL. +- Define one canonical scalar expression model for row-level relational authoring in IncQL. - Require row-level authoring surfaces such as `filter(...)`, `with_column(...)`, future `select(...)`, and grouping keys to consume that same scalar expression model. - Keep aggregate outputs as a distinct aggregate-measure layer while allowing aggregates to consume scalar-expression inputs. - Require explicit errors for unsupported expression shapes; silent degradation is not allowed. - Provide one lowering target for future concise DSL surfaces in method chains and `query {}` blocks. -- Give the InQL package, Prism, and Substrait emission layers one shared semantic contract for row-level expressions. +- Give the IncQL package, Prism, and Substrait emission layers one shared semantic contract for row-level expressions. ## Non-Goals @@ -49,7 +49,7 @@ This RFC is also needed before InQL can take proper advantage of the scoped DSL - Introducing new parser syntax in this RFC. - Defining join output typing, relation schema evolution, or materialization semantics beyond expression authoring. - Making aggregate outputs behave as ordinary row-level scalar expressions in all positions. -- Standardizing every public helper spelling across all possible InQL libraries or future extensions. +- Standardizing every public helper spelling across all possible IncQL libraries or future extensions. ## Guide-level explanation (how authors think about it) @@ -60,8 +60,8 @@ The canonical public literal helper is `lit(...)`. Typed literal helpers are ent If an author filters rows or computes a new column, those operations should be using the same underlying scalar expression model: ```incan -from pub::inql import LazyFrame -from pub::inql.functions import col, lit, gt, add +from pub::incql import LazyFrame +from pub::incql.functions import col, lit, gt, add from models import Order def enrich_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -75,8 +75,8 @@ def enrich_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: If an author groups rows and supplies arguments to aggregates, those aggregate inputs should still be ordinary scalar expressions even though the aggregate outputs are not: ```incan -from pub::inql import LazyFrame -from pub::inql.functions import col, sum, count +from pub::incql import LazyFrame +from pub::incql.functions import col, sum, count from models import Order, OrderSummary def summarize_orders(orders: LazyFrame[Order]) -> LazyFrame[OrderSummary]: @@ -97,7 +97,7 @@ In that example: - `sum(...)` consumes a scalar expression and produces an aggregate measure - `count()` is a distinct aggregate form because it does not require a scalar input -This RFC does not require concise sugar, but it defines what concise sugar should mean later. If InQL later supports surfaces such as: +This RFC does not require concise sugar, but it defines what concise sugar should mean later. If IncQL later supports surfaces such as: ```incan orders @@ -115,7 +115,7 @@ Authors should also get explicit failure for unsupported shapes. If a library or ### Canonical scalar expression model -InQL must define one canonical scalar expression model for row-level relational meaning. That model may evolve over time, but it must be the semantic target for all row-level expression-bearing surfaces in the package. +IncQL must define one canonical scalar expression model for row-level relational meaning. That model may evolve over time, but it must be the semantic target for all row-level expression-bearing surfaces in the package. At minimum, the canonical scalar expression model must be able to represent: @@ -127,7 +127,7 @@ Separate public wrapper types may exist as implementation details, but they must ### Row-level consumers -The following InQL positions must consume scalar expressions: +The following IncQL positions must consume scalar expressions: - row filters - computed projection values @@ -139,7 +139,7 @@ Each position must still enforce its own result-type contract: - `filter(...)` must require a scalar expression whose result type is `bool` - `with_column(...)` and projection positions must require a non-aggregate row-level scalar expression -- grouping-key positions must require scalar expressions that are valid grouping keys under the current InQL contract +- grouping-key positions must require scalar expressions that are valid grouping keys under the current IncQL contract ### Aggregate measures @@ -151,7 +151,7 @@ Aggregate measures must not be treated as ordinary row-level scalar expressions ### Explicit failure requirement -InQL package APIs, Prism planning, and Substrait lowering must not silently degrade unsupported expression shapes. +IncQL package APIs, Prism planning, and Substrait lowering must not silently degrade unsupported expression shapes. If a public authoring surface accepts an expression shape that cannot be represented or executed faithfully in the target position, the system must produce an explicit diagnostic or planning error. @@ -163,19 +163,19 @@ The following behaviors are forbidden: ### Canonical literal concept -The semantic concept of a scalar literal must be unified. InQL standardizes `lit(...)` as the canonical public helper for scalar literals. Because Incan RFC 029 gives Incan first-class union types, `lit(...)` may accept a closed union of supported literal input types such as `int | float | str | bool` while still returning one scalar-expression representation. +The semantic concept of a scalar literal must be unified. IncQL standardizes `lit(...)` as the canonical public helper for scalar literals. Because Incan RFC 029 gives Incan first-class union types, `lit(...)` may accept a closed union of supported literal input types such as `int | float | str | bool` while still returning one scalar-expression representation. Typed helpers such as `int_expr(...)`, `float_expr(...)`, `str_expr(...)`, `bool_expr(...)`, `int_lit(...)`, `str_lit(...)`, and `bool_lit(...)` must lower into the same scalar-literal representation as `lit(...)`. The system must not preserve separate literal hierarchies for filters, projections, and other row-level positions. ### Lowering target for future authoring surfaces -If InQL adopts future concise method-chain sugar or richer `query {}` syntax using Incan RFC 025, RFC 028, RFC 029, RFC 040, and RFC 045 facilities, those surfaces must lower into the canonical scalar expression model for row-level meaning and the canonical aggregate-measure model for aggregate meaning. +If IncQL adopts future concise method-chain sugar or richer `query {}` syntax using Incan RFC 025, RFC 028, RFC 029, RFC 040, and RFC 045 facilities, those surfaces must lower into the canonical scalar expression model for row-level meaning and the canonical aggregate-measure model for aggregate meaning. ## Design details ### Syntax -This RFC does not require new InQL syntax. Existing builder-call surfaces are sufficient to define the contract. +This RFC does not require new IncQL syntax. Existing builder-call surfaces are sufficient to define the contract. The long-term intention is that concise surfaces may exist, but they are only acceptable if they lower into the same canonical expression model defined here. @@ -188,19 +188,19 @@ The core semantic split is: Boolean predicates are ordinary scalar expressions whose result type is `bool`; they are not a separate semantic species. -Grouping keys belong on the scalar-expression side. They determine grouping identity by evaluating one scalar expression per input row. The north-star contract allows deterministic, row-level scalar expressions as grouping keys when their result type is valid for grouping. InQL may initially support only a subset of scalar expressions for grouping, but if so, that restriction must be explicit and diagnosable, not a permanent semantic narrowing and not a silent fallback to direct-column grouping. +Grouping keys belong on the scalar-expression side. They determine grouping identity by evaluating one scalar expression per input row. The north-star contract allows deterministic, row-level scalar expressions as grouping keys when their result type is valid for grouping. IncQL may initially support only a subset of scalar expressions for grouping, but if so, that restriction must be explicit and diagnosable, not a permanent semantic narrowing and not a silent fallback to direct-column grouping. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces -- **Dataset carriers and method chains (InQL RFC 001):** method-chain surfaces such as `filter(...)`, `with_column(...)`, `group_by(...)`, and future projection methods should consume one scalar-expression model rather than independent mini-DSLs. -- **`query {}` blocks (InQL RFC 003):** query-block expressions should lower into the same scalar-expression and aggregate-measure contracts rather than defining a separate semantic path. -- **Execution context (InQL RFC 004):** session execution should receive one row-level expression contract and one aggregate-measure contract, not surface-specific variants. -- **Prism (InQL RFC 007):** Prism should represent row-level expression meaning once and reuse it across logical operators instead of duplicating per-surface expression semantics. +- **Dataset carriers and method chains (IncQL RFC 001):** method-chain surfaces such as `filter(...)`, `with_column(...)`, `group_by(...)`, and future projection methods should consume one scalar-expression model rather than independent mini-DSLs. +- **`query {}` blocks (IncQL RFC 003):** query-block expressions should lower into the same scalar-expression and aggregate-measure contracts rather than defining a separate semantic path. +- **Execution context (IncQL RFC 004):** session execution should receive one row-level expression contract and one aggregate-measure contract, not surface-specific variants. +- **Prism (IncQL RFC 007):** Prism should represent row-level expression meaning once and reuse it across logical operators instead of duplicating per-surface expression semantics. - **Incan `model` types and lexical scope:** model fields remain the source of column naming and typing, and ordinary lexical scope rules still govern explicit helper references until scoped DSL facilities are adopted. ### Surface cleanup -This RFC consolidates builder families before InQL's first released API. +This RFC consolidates builder families before IncQL's first released API. The expected cleanup shape is: @@ -209,7 +209,7 @@ The expected cleanup shape is: - docs and diagnostics should steer authors toward one canonical scalar-expression concept - split row-level helper families should be collapsed before release -Correctness takes precedence over convenience. If a permissive helper path would silently change semantics, InQL should reject that path instead. +Correctness takes precedence over convenience. If a permissive helper path would silently change semantics, IncQL should reject that path instead. ## Alternatives considered @@ -220,16 +220,16 @@ Correctness takes precedence over convenience. If a permissive helper path would ## Drawbacks -- InQL surfaces that grew independently may need cleanup before release. +- IncQL surfaces that grew independently may need cleanup before release. - Tooling and diagnostics become more demanding because the system must enforce the scalar-versus-aggregate boundary more consistently. - Some previously tolerated expression shapes may need to become hard errors if they only "worked" through accidental or degraded behavior. - The RFC makes inconsistencies more visible, which can force earlier cleanup across docs, examples, planning, and lowering. ## Layers affected -- **InQL specification** — RFCs 001, 003, 004, and 007 must stay coherent with one shared scalar-expression and aggregate-measure contract. -- **InQL library package** — public `.incn` APIs should converge on one canonical row-level expression model and explicit aggregate-measure wrappers. -- **Incan compiler** — if InQL adopts scoped DSL sugar later, parser, checker, lowering, and diagnostics must preserve the scalar-expression lowering contract rather than inventing a separate semantic path. +- **IncQL specification** — RFCs 001, 003, 004, and 007 must stay coherent with one shared scalar-expression and aggregate-measure contract. +- **IncQL library package** — public `.incn` APIs should converge on one canonical row-level expression model and explicit aggregate-measure wrappers. +- **Incan compiler** — if IncQL adopts scoped DSL sugar later, parser, checker, lowering, and diagnostics must preserve the scalar-expression lowering contract rather than inventing a separate semantic path. - **Execution / interchange** — Prism and Substrait lowering must preserve the scalar-versus-aggregate boundary and must not silently rewrite unsupported expression shapes. - **Documentation** — user docs and reference pages should describe one row-level expression model instead of multiple parallel mini-DSLs. @@ -261,7 +261,7 @@ Correctness takes precedence over convenience. If a permissive helper path would - Add package tests covering `lit(...)` and typed literal helpers across filters, computed projections, grouping keys, and aggregate inputs. - Add negative tests for unsupported scalar-expression shapes in grouping and aggregate positions when the implementation cannot lower them faithfully. - Update reference and explanation docs so users see one scalar expression model instead of separate filter/projection literal families. -- Decide whether this user-visible package change requires an InQL package version bump, and if so keep `incan.toml` and `src/metadata.incn` synchronized. +- Decide whether this user-visible package change requires an IncQL package version bump, and if so keep `incan.toml` and `src/metadata.incn` synchronized. ## Implementation Log diff --git a/docs/rfcs/013_function_catalog_program.md b/docs/rfcs/013_function_catalog_program.md index 67c42103..b4626cd8 100644 --- a/docs/rfcs/013_function_catalog_program.md +++ b/docs/rfcs/013_function_catalog_program.md @@ -1,43 +1,43 @@ -# InQL RFC 013: Function catalog program +# IncQL RFC 013: Function catalog program - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 015 (core scalar functions and operators) - - InQL RFC 016 (core aggregate functions) - - InQL RFC 017 (aggregate modifiers) - - InQL RFC 018 (common scalar function catalog) - - InQL RFC 019 (window functions) - - InQL RFC 020 (nested data functions) - - InQL RFC 021 (generator and table-valued functions) - - InQL RFC 022 (semi-structured and format functions) - - InQL RFC 023 (approximate and sketch functions) - - InQL RFC 024 (function extension policy) - - InQL RFC 025 (typed sketch logical values) - - InQL RFC 026 (semi-structured variant logical values) -- **Issue:** [InQL #30](https://github.com/encero-systems/InQL/issues/30) -- **RFC PR:** [InQL #59](https://github.com/encero-systems/InQL/pull/59) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 015 (core scalar functions and operators) + - IncQL RFC 016 (core aggregate functions) + - IncQL RFC 017 (aggregate modifiers) + - IncQL RFC 018 (common scalar function catalog) + - IncQL RFC 019 (window functions) + - IncQL RFC 020 (nested data functions) + - IncQL RFC 021 (generator and table-valued functions) + - IncQL RFC 022 (semi-structured and format functions) + - IncQL RFC 023 (approximate and sketch functions) + - IncQL RFC 024 (function extension policy) + - IncQL RFC 025 (typed sketch logical values) + - IncQL RFC 026 (semi-structured variant logical values) +- **Issue:** [IncQL #30](https://github.com/encero-systems/IncQL/issues/30) +- **RFC PR:** [IncQL #59](https://github.com/encero-systems/IncQL/pull/59) - **Written against:** Incan v0.2 -- **Shipped in:** InQL v0.1 +- **Shipped in:** IncQL v0.1 ## Summary -This RFC is the umbrella tracking RFC for InQL's function catalog expansion. It defines the target shape and acceptance boundary for the related child RFCs that turn InQL from a small builder-first function surface into a typed, registry-backed dataframe and query function catalog. This RFC is complete only when the child RFCs needed for the catalog program are implemented or deliberately rejected. +This RFC is the umbrella tracking RFC for IncQL's function catalog expansion. It defines the target shape and acceptance boundary for the related child RFCs that turn IncQL from a small builder-first function surface into a typed, registry-backed dataframe and query function catalog. This RFC is complete only when the child RFCs needed for the catalog program are implemented or deliberately rejected. ## Core model 1. Function catalog work is organized by semantic boundary, not by individual function. -2. InQL must establish a registry and governance model before broad function families become stable. +2. IncQL must establish a registry and governance model before broad function families become stable. 3. Core scalar and aggregate functions form the first author-facing expansion. 4. Aggregate modifiers, window functions, nested data, generators, format functions, approximate/sketch functions, typed sketch values, semi-structured values, and extensions each require their own semantic contracts. 5. The umbrella RFC tracks program completion; child RFCs define normative behavior. ## Motivation -InQL currently exposes only a small number of functions relative to mature dataframe, warehouse, and SQL systems. Spark, DataFusion, Arrow, Beam, Snowflake, dbt, and standard SQL references show that a credible relational data surface needs many more functions, but adding them without structure would create catalog sprawl and inconsistent semantics. +IncQL currently exposes only a small number of functions relative to mature dataframe, warehouse, and SQL systems. Spark, DataFusion, Arrow, Beam, Snowflake, dbt, and standard SQL references show that a credible relational data surface needs many more functions, but adding them without structure would create catalog sprawl and inconsistent semantics. The function expansion is too broad for one normative RFC if every family is specified precisely. It is also too connected to leave as unrelated RFCs, because decisions about registry metadata, aliases, null behavior, type coercion, aggregate modifiers, backend support, and extension policy affect every child document. An umbrella RFC gives the program a clear lifecycle while letting each semantic cluster remain reviewable. @@ -53,15 +53,15 @@ The function expansion is too broad for one normative RFC if every family is spe - Defining individual function semantics directly in this RFC. - Replacing the child RFCs as normative contracts. -- Requiring every Spark, DataFusion, Arrow, Beam, Snowflake, dbt, or SQL function to enter the InQL portable core. +- Requiring every Spark, DataFusion, Arrow, Beam, Snowflake, dbt, or SQL function to enter the IncQL portable core. - Defining implementation tickets, milestones, or file-level work. ## Guide-level explanation (how authors think about it) -Authors should eventually see InQL as having a coherent function catalog rather than a handful of unrelated helpers: +Authors should eventually see IncQL as having a coherent function catalog rather than a handful of unrelated helpers: ```incan -from pub::inql.functions import avg, col, count, date_trunc, lower, sum, trim +from pub::incql.functions import avg, col, count, date_trunc, lower, sum, trim summary = ( orders @@ -79,23 +79,23 @@ The child RFCs define which names exist, how they typecheck, how aliases work, a The function catalog program must consist of the following child RFCs unless this RFC is amended: -- InQL RFC 014 (function registry and catalog governance) -- InQL RFC 015 (core scalar functions and operators) -- InQL RFC 016 (core aggregate functions) -- InQL RFC 017 (aggregate modifiers) -- InQL RFC 018 (common scalar function catalog) -- InQL RFC 019 (window functions) -- InQL RFC 020 (nested data functions) -- InQL RFC 021 (generator and table-valued functions) -- InQL RFC 022 (semi-structured and format functions) -- InQL RFC 023 (approximate and sketch functions) -- InQL RFC 024 (function extension policy) -- InQL RFC 025 (typed sketch logical values) -- InQL RFC 026 (semi-structured variant logical values) +- IncQL RFC 014 (function registry and catalog governance) +- IncQL RFC 015 (core scalar functions and operators) +- IncQL RFC 016 (core aggregate functions) +- IncQL RFC 017 (aggregate modifiers) +- IncQL RFC 018 (common scalar function catalog) +- IncQL RFC 019 (window functions) +- IncQL RFC 020 (nested data functions) +- IncQL RFC 021 (generator and table-valued functions) +- IncQL RFC 022 (semi-structured and format functions) +- IncQL RFC 023 (approximate and sketch functions) +- IncQL RFC 024 (function extension policy) +- IncQL RFC 025 (typed sketch logical values) +- IncQL RFC 026 (semi-structured variant logical values) This umbrella RFC must not be marked Implemented while any required child RFC remains Draft, Planned, In Progress, Blocked, or otherwise unresolved. A child RFC may be removed from the required completion set only by a design decision recorded in this RFC or by a superseding RFC. -Child RFCs must use the registry and governance model once InQL RFC 014 is Planned. If a child RFC needs behavior that contradicts InQL RFC 014, it must explicitly amend or supersede the relevant registry rule rather than creating a local exception. +Child RFCs must use the registry and governance model once IncQL RFC 014 is Planned. If a child RFC needs behavior that contradicts IncQL RFC 014, it must explicitly amend or supersede the relevant registry rule rather than creating a local exception. Portable core functions must be distinguished from compatibility aliases and extensions. Compatibility with external systems is an input to catalog design, not an automatic inclusion rule. @@ -109,9 +109,9 @@ This RFC introduces no syntax. Syntax belongs in the relevant child RFCs or exis This RFC is normative for program structure and lifecycle. Function behavior is normative only in the child RFC that owns the corresponding semantic family. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces -All function catalog work must stay consistent with InQL RFC 012. Query blocks and dataframe method chains must not acquire divergent function semantics as the catalog expands. +All function catalog work must stay consistent with IncQL RFC 012. Query blocks and dataframe method chains must not acquire divergent function semantics as the catalog expands. ### Compatibility / migration @@ -131,14 +131,14 @@ Existing helpers may remain while the child RFCs migrate them into the registry- ## Layers affected -- **InQL specification** — the function catalog program must remain coherent across child RFCs and existing expression/query RFCs. -- **InQL library package** — catalog implementation should follow the child RFC sequence rather than accumulating unrelated helpers. +- **IncQL specification** — the function catalog program must remain coherent across child RFCs and existing expression/query RFCs. +- **IncQL library package** — catalog implementation should follow the child RFC sequence rather than accumulating unrelated helpers. - **Incan compiler** — compiler-facing function behavior should be driven by the registry and child RFC contracts. - **Execution / interchange** — backend lowering and support diagnostics should follow the child RFCs and registry metadata. - **Documentation** — function documentation should reflect the program structure and distinguish core, compatibility, and extension surfaces. ## Design Decisions -- **Child RFC scope:** the current child RFC set is the scope of the function catalog program. InQL RFC 014 through InQL RFC 026 are required children unless this umbrella RFC is later amended or superseded. -- **Implemented status:** this umbrella RFC may be marked Implemented only when all required child RFCs through InQL RFC 026 are implemented, rejected, or superseded by explicit design decision. Extension, sketch, typed sketch value, and semi-structured value families are part of the umbrella scope, not optional follow-on scope. -- **Closeout:** the required child RFC set is now resolved. InQL RFC 014 through InQL RFC 026 are implemented, and the shared catalog surface also includes typed value-or-column inputs plus schema-aware scalar input validation as cross-cutting support grounded in InQL RFC 000, InQL RFC 012, and InQL RFC 014. No additional child RFC is required for that support because it tightens the existing registry and query-schema contracts rather than adding a new function family. +- **Child RFC scope:** the current child RFC set is the scope of the function catalog program. IncQL RFC 014 through IncQL RFC 026 are required children unless this umbrella RFC is later amended or superseded. +- **Implemented status:** this umbrella RFC may be marked Implemented only when all required child RFCs through IncQL RFC 026 are implemented, rejected, or superseded by explicit design decision. Extension, sketch, typed sketch value, and semi-structured value families are part of the umbrella scope, not optional follow-on scope. +- **Closeout:** the required child RFC set is now resolved. IncQL RFC 014 through IncQL RFC 026 are implemented, and the shared catalog surface also includes typed value-or-column inputs plus schema-aware scalar input validation as cross-cutting support grounded in IncQL RFC 000, IncQL RFC 012, and IncQL RFC 014. No additional child RFC is required for that support because it tightens the existing registry and query-schema contracts rather than adding a new function family. diff --git a/docs/rfcs/014_function_registry.md b/docs/rfcs/014_function_registry.md index 6987d405..e60d90cb 100644 --- a/docs/rfcs/014_function_registry.md +++ b/docs/rfcs/014_function_registry.md @@ -1,15 +1,15 @@ -# InQL RFC 014: Function registry and catalog governance +# IncQL RFC 014: Function registry and catalog governance - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 000 (language specification and layer boundaries) - - InQL RFC 002 (Substrait lowering and extension policy) - - InQL RFC 003 (`query {}` blocks and relational authoring) - - InQL RFC 007 (Prism planning and optimization) - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 013 (function catalog program) + - IncQL RFC 000 (language specification and layer boundaries) + - IncQL RFC 002 (Substrait lowering and extension policy) + - IncQL RFC 003 (`query {}` blocks and relational authoring) + - IncQL RFC 007 (Prism planning and optimization) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 013 (function catalog program) - Incan RFC 048 (contract-backed models, emit, and interrogation tooling) - Incan issue #437 (top-level callable aliases) - Incan issue #438 (`incan docs` API documentation extraction) @@ -19,33 +19,33 @@ - Incan issue #645 (method-call decorators for registry registration) - Incan issue #658 / PR #660 (`const` model constructor initializers for typed version constants) - Incan issue #659 / PR #660 (lowercase exported static import/codegen mismatch) -- **Issue:** [InQL #31](https://github.com/encero-systems/InQL/issues/31) -- **RFC PR:** [InQL #42](https://github.com/encero-systems/InQL/pull/42) +- **Issue:** [IncQL #31](https://github.com/encero-systems/IncQL/issues/31) +- **RFC PR:** [IncQL #42](https://github.com/encero-systems/IncQL/pull/42) - **Written against:** Incan v0.2 - **Shipped in:** v0.1 ## Summary -This RFC defines the InQL function registry: the single source of truth for scalar, aggregate, window, generator, and extension functions across query blocks, dataframe method chains, planning, diagnostics, generated documentation, and Substrait interchange. The registry records canonical names, compatibility aliases, arity, type rules, null and error behavior, determinism, function class, boundedness restrictions, documentation metadata, lifecycle metadata, and Substrait mapping strategy so that future function expansion is coherent rather than a pile of ad hoc helpers. +This RFC defines the IncQL function registry: the single source of truth for scalar, aggregate, window, generator, and extension functions across query blocks, dataframe method chains, planning, diagnostics, generated documentation, and Substrait interchange. The registry records canonical names, compatibility aliases, arity, type rules, null and error behavior, determinism, function class, boundedness restrictions, documentation metadata, lifecycle metadata, and Substrait mapping strategy so that future function expansion is coherent rather than a pile of ad hoc helpers. ## Core model -1. A function has one canonical InQL identity and zero or more compatibility aliases. +1. A function has one canonical IncQL identity and zero or more compatibility aliases. 2. A function belongs to one function class: scalar, aggregate, window, generator, table-valued, partition transform, or extension-only. 3. A function signature defines accepted argument shapes, type coercion, return type rules, null behavior, error behavior, and determinism. -4. A function entry records the required Substrait interchange strategy; backend availability must be declared by adapters and must not redefine the InQL semantic contract. +4. A function entry records the required Substrait interchange strategy; backend availability must be declared by adapters and must not redefine the IncQL semantic contract. 5. A function entry is registered by attaching one `register_function(...)` decorator to a normal public helper; the helper declaration supplies the canonical name and signature, the decorator call supplies typed machine-readable metadata, and the helper docstring carries human-facing explanation. 6. A function entry records lifecycle metadata such as introduced, changed, deprecated, removed, and replacement versions. ## Motivation -Spark and similar systems expose a very large function surface. Copying that surface one helper at a time would make InQL harder to reason about, because related decisions such as null semantics, overflow policy, aliases, aggregate modifiers, and backend fallbacks would be scattered across individual additions. InQL needs a registry-level contract first so later RFCs can add catalog breadth without reopening the same foundational questions. +Spark and similar systems expose a very large function surface. Copying that surface one helper at a time would make IncQL harder to reason about, because related decisions such as null semantics, overflow policy, aliases, aggregate modifiers, and backend fallbacks would be scattered across individual additions. IncQL needs a registry-level contract first so later RFCs can add catalog breadth without reopening the same foundational questions. -This is also necessary for diagnostics. If a function is known to InQL but cannot be represented by the current Prism/Substrait contract, authors should get a precise error or fallback explanation. If a name is a Spark-compatible, Snowflake-compatible, or dbt-style portability alias for a canonical InQL function, docs and tooling should be able to say so consistently. +This is also necessary for diagnostics. If a function is known to IncQL but cannot be represented by the current Prism/Substrait contract, authors should get a precise error or fallback explanation. If a name is a Spark-compatible, Snowflake-compatible, or dbt-style portability alias for a canonical IncQL function, docs and tooling should be able to say so consistently. ## Goals -- Define the required metadata every InQL function entry must carry. +- Define the required metadata every IncQL function entry must carry. - Distinguish canonical function names from compatibility aliases. - Define function classes and require class-specific validation. - Require registered functions to carry Incan-standard human-facing docstrings without making docstring section policy part of this RFC. @@ -66,10 +66,10 @@ This is also necessary for diagnostics. If a function is known to InQL but canno ## Guide-level explanation (how authors think about it) -Authors should be able to call functions through the normal InQL surfaces and rely on one semantic catalog: +Authors should be able to call functions through the normal IncQL surfaces and rely on one semantic catalog: ```incan -from pub::inql.functions import avg, col, count, lit, lower, trim +from pub::incql.functions import avg, col, count, lit, lower, trim cleaned = orders.with_column("normalized_email", lower(trim(col("email")))) summary = cleaned.group_by([col("customer_id")]).agg([count(), avg(col("amount"))]) @@ -81,7 +81,7 @@ The author does not need to know whether `avg` maps to a core Substrait function Each registered function must have a canonical name. Canonical names must be stable once an RFC reaches Planned unless a later RFC explicitly supersedes them. -Each registered function may have aliases. An alias must resolve to exactly one canonical function in a given scope. If two imported extensions introduce the same alias for different canonical functions, InQL must report an ambiguity rather than choosing silently. +Each registered function may have aliases. An alias must resolve to exactly one canonical function in a given scope. If two imported extensions introduce the same alias for different canonical functions, IncQL must report an ambiguity rather than choosing silently. Each registered function must declare a function class. A scalar function must produce one value per input row. An aggregate function must produce one value per group or relation. A window function must require a window specification unless another RFC explicitly defines a default. A generator or table-valued function must be represented as a relation-shaping operation rather than as a scalar expression. @@ -104,13 +104,13 @@ Each portable core function must declare a Substrait interchange strategy. The s - deterministic rewrite to supported Substrait expressions - structural relation-context lowering, such as sort-field helpers consumed by `SortRel` -Prism must only accept portable core function calls that can be represented by the active InQL/Substrait contract. A function with no valid Substrait mapping must remain Draft, extension-only, or rejected for portable core until that mapping exists. +Prism must only accept portable core function calls that can be represented by the active IncQL/Substrait contract. A function with no valid Substrait mapping must remain Draft, extension-only, or rejected for portable core until that mapping exists. -Execution backends must adapt from the Substrait representation rather than redefining InQL function semantics. A backend may declare that it supports, rewrites, emulates, approximates, or rejects a Substrait function representation, but that declaration belongs to the backend capability layer. +Execution backends must adapt from the Substrait representation rather than redefining IncQL function semantics. A backend may declare that it supports, rewrites, emulates, approximates, or rejects a Substrait function representation, but that declaration belongs to the backend capability layer. -Each registered function must declare lifecycle metadata. The minimum lifecycle field is the InQL package version where the function was introduced. If a function's signature, semantics, alias set, Substrait mapping, or documentation contract changes in a user-visible way, the registry must record a versioned change entry. Deprecated functions must record the deprecation version, replacement guidance when a replacement exists, and removal status if removal is planned or completed. +Each registered function must declare lifecycle metadata. The minimum lifecycle field is the IncQL package version where the function was introduced. If a function's signature, semantics, alias set, Substrait mapping, or documentation contract changes in a user-visible way, the registry must record a versioned change entry. Deprecated functions must record the deprecation version, replacement guidance when a replacement exists, and removal status if removal is planned or completed. -Each registered function must have a typed registry entry for non-derivable machine metadata and an Incan-standard docstring for human-facing explanation. For ordinary public built-in functions, the canonical declaration shape is a normal public helper annotated with `register_function(...)`. The decorator call registers the helper, derives its stable function reference from the checked helper name by default, and supplies typed metadata that cannot be recovered from the public declaration. The checked helper declaration is the source for parameters, parameter types, and return type. A decorator may supply an explicit canonical-name override for host-language spelling constraints, such as `modulo(...)` registering canonical `mod`; this is not an alias mechanism. The checked InQL package source and typed registry data are the source from which compiler-facing metadata, generated docs, diagnostics metadata, and lowering tables are produced. Generated registry entries may exist for mechanically produced functions, and explicit registry objects may exist for advanced extension cases, but the registry must not depend on arbitrary body inspection, stringly alias metadata, or prose inference. +Each registered function must have a typed registry entry for non-derivable machine metadata and an Incan-standard docstring for human-facing explanation. For ordinary public built-in functions, the canonical declaration shape is a normal public helper annotated with `register_function(...)`. The decorator call registers the helper, derives its stable function reference from the checked helper name by default, and supplies typed metadata that cannot be recovered from the public declaration. The checked helper declaration is the source for parameters, parameter types, and return type. A decorator may supply an explicit canonical-name override for host-language spelling constraints, such as `modulo(...)` registering canonical `mod`; this is not an alias mechanism. The checked IncQL package source and typed registry data are the source from which compiler-facing metadata, generated docs, diagnostics metadata, and lowering tables are produced. Generated registry entries may exist for mechanically produced functions, and explicit registry objects may exist for advanced extension cases, but the registry must not depend on arbitrary body inspection, stringly alias metadata, or prose inference. This RFC intentionally defines required metadata shapes rather than exact enum, model, class, or tagged-union implementations. The implementation may represent lifecycle, declaration-derived signatures, behavior categories, and Substrait mappings as enums, models, classes, generated records, or another typed representation, as long as the resulting normalized function catalog exposes the same fields to docs, typechecking, diagnostics, Prism, and backend capability checks. @@ -130,7 +130,7 @@ Generated Markdown must preserve the canonical registry facts and must use docst ### Syntax -This RFC does not introduce new syntax. Function calls may appear through existing and future InQL expression surfaces. +This RFC does not introduce new syntax. Function calls may appear through existing and future IncQL expression surfaces. The `mean = avg` alias form used below depends on Incan support for top-level callable aliases. Until that exists, aliases may be represented by forwarding functions or omitted from the first implementation phase, but aliases must not be modeled as string fields inside function specs. @@ -144,7 +144,7 @@ Function documentation is part of the registry contract, but exact required docs For ordinary built-in functions, the declaration-site `register_function(...)` decorator is the canonical registration surface. The public helper should be ordinary code that delegates to the existing expression or aggregate builder, so authors call a normal function while tooling inspects explicit typed metadata from the decorator call. Docs, LSP, typechecking, Prism, and Substrait lowering must all inspect the same resulting function catalog entry produced from the checked source and typed registry metadata. -The registration decorator must be the single declaration-side registry event for ordinary built-ins. It links exactly one helper symbol to exactly one stable function reference and receives the typed function spec that records lifecycle, determinism, null behavior, error behavior, alias policy, optional canonical-name override, and Substrait mapping. Helper signature and ordinary helper name come from the checked declaration. Backend capability declarations consume those facts; they do not redefine InQL function semantics. +The registration decorator must be the single declaration-side registry event for ordinary built-ins. It links exactly one helper symbol to exactly one stable function reference and receives the typed function spec that records lifecycle, determinism, null behavior, error behavior, alias policy, optional canonical-name override, and Substrait mapping. Helper signature and ordinary helper name come from the checked declaration. Backend capability declarations consume those facts; they do not redefine IncQL function semantics. Compatibility aliases must be real callable symbols rather than strings inside a function spec. For example, `mean = avg` should make `mean` an alias of the registered `avg` helper. The function catalog may record that alias after name resolution, but the aggregate spec must not contain `aliases=["mean"]`. Backend spellings and backend aliases remain backend capability concerns. @@ -183,22 +183,22 @@ pub def avg(expr: ScalarExpr[number]) -> AggregateMeasure[number]: The exact type names and helper constructors may change during implementation. Tooling must not infer null behavior, error behavior, determinism, lifecycle status, or Substrait mapping from prose alone, and it must not infer examples or simple user-facing explanation from registry entries alone. A public helper must be explicitly linked to exactly one function reference, and generation or validation must fail if the helper signature and registry signature drift apart. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces -`query {}` blocks and `DataSet[T]` method chains must resolve function names through the same registry semantics. InQL RFC 012 owns the scalar-expression contract; this RFC owns function identity and metadata within that expression model. +`query {}` blocks and `DataSet[T]` method chains must resolve function names through the same registry semantics. IncQL RFC 012 owns the scalar-expression contract; this RFC owns function identity and metadata within that expression model. Aggregate, window, generator, nested-data, format, sketch, and extension RFCs must add functions through this registry model instead of defining incompatible local catalogs. ### Compatibility / migration -Existing helper names such as `sum`, `count`, `add`, `mul`, `eq`, and `gt` may continue as compatibility shims if they resolve to registered canonical functions. InQL should migrate documentation toward canonical names while preserving useful aliases where semantics match. +Existing helper names such as `sum`, `count`, `add`, `mul`, `eq`, and `gt` may continue as compatibility shims if they resolve to registered canonical functions. IncQL should migrate documentation toward canonical names while preserving useful aliases where semantics match. ## Alternatives considered - **Ad hoc helpers only.** Rejected because it spreads function semantics across unrelated APIs and makes backend diagnostics weaker. -- **Copy a backend catalog directly.** Rejected because InQL needs a portable author contract even when DataFusion, Spark, Snowflake, Arrow, dbt adapters, or another engine differs. +- **Copy a backend catalog directly.** Rejected because IncQL needs a portable author contract even when DataFusion, Spark, Snowflake, Arrow, dbt adapters, or another engine differs. - **Expose only SQL strings.** Rejected because it loses typed Incan authoring and makes compiler diagnostics harder. -- **Let backends define function semantics.** Rejected because Prism's canonical interchange is Substrait. InQL functions must define backend-independent semantics and a Substrait representation strategy; adapters may only declare whether they can execute that representation faithfully. +- **Let backends define function semantics.** Rejected because Prism's canonical interchange is Substrait. IncQL functions must define backend-independent semantics and a Substrait representation strategy; adapters may only declare whether they can execute that representation faithfully. - **Use only explicit registry data.** Rejected for ordinary built-ins because a hand-authored `FunctionRegistry([...])` list creates a second binding surface and makes drift likely. Explicit registry objects may still be appropriate for generated functions or extension loading. - **Split decorator links from a central metadata list.** Rejected for ordinary built-ins because it creates two authoring events for one function: one declaration-side link and one catalog-side entry. The decorator should be the registration event and should receive the typed spec that carries the machine-readable contract. @@ -211,9 +211,9 @@ Existing helper names such as `sum`, `count`, `add`, `mul`, `eq`, and `gt` may c ## Layers affected -- **InQL specification** — future function RFCs must use the registry model for canonical names, aliases, function class, type rules, null behavior, determinism, lifecycle metadata, and Substrait mapping strategy. -- **InQL library package** — public function helpers should resolve to registered functions rather than independent helper-family concepts. -- **Incan compiler** — function resolution and diagnostics should preserve registry identity when checking InQL expressions. +- **IncQL specification** — future function RFCs must use the registry model for canonical names, aliases, function class, type rules, null behavior, determinism, lifecycle metadata, and Substrait mapping strategy. +- **IncQL library package** — public function helpers should resolve to registered functions rather than independent helper-family concepts. +- **Incan compiler** — function resolution and diagnostics should preserve registry identity when checking IncQL expressions. - **Execution / interchange** — Prism lowering must use registry Substrait mapping metadata to choose core Substrait, registered extension, semantics-preserving rewrite, or rejection. Backend capability declarations live outside the function entry. - **Documentation** — function reference pages must be generated from, or mechanically checked against, structured registry docstrings and metadata. @@ -276,10 +276,10 @@ Existing helper names such as `sum`, `count`, `add`, `mul`, `eq`, and `gt` may c ## Design Decisions -- **Registry ownership:** the checked InQL package source is the source of truth. Compiler-facing metadata, generated docs metadata, diagnostics metadata, and lowering tables are derived from checked package source and typed registry data. The compiler must not maintain an independent InQL function registry. +- **Registry ownership:** the checked IncQL package source is the source of truth. Compiler-facing metadata, generated docs metadata, diagnostics metadata, and lowering tables are derived from checked package source and typed registry data. The compiler must not maintain an independent IncQL function registry. - **Authoring DX:** ordinary function authors should write normal public helpers and attach one registry decorator whose arguments contain the typed function spec. The registry derives the stable function reference from the checked public helper name, avoiding a separate authored constant or central list. - **Decorator capability:** Incan issue #636 / PR #637 is required for decorator-authored helpers because checked API metadata must preserve source signatures for decorated functions. Incan issue #638 / PR #641 is required for decorator string argument materialization. Incan issue #640 / PR #643 provides generic signature-preserving decorator factories. Incan issue #645 is required for method-call decorators such as `register_function(...)`. The RFC design is one registry decorator attached to the public helper. - **Lifecycle constants:** typed lifecycle metadata uses immutable version constants such as `v0_1`. These are `const` model values, not mutable registry state or generated strings. - **Alias policy:** core semantic aliases may be available through normal public imports when they are real callable aliases of the canonical function. Dialect, warehouse, Spark, Snowflake, dbt, and backend compatibility aliases require explicit opt-in modules. - **Docstrings:** exact docstring section requirements are not an RFC 014 concern. Public registered functions must follow the repository's Incan-standard docstring policy, but registry metadata and public helper signatures own machine facts. -- **Substrait mapping:** typed registry entries must represent whether a function maps to a core Substrait function, a registered extension function, a deterministic rewrite, or structural relation-context lowering. Backend capability declarations consume the emitted Substrait representation; they do not redefine InQL semantics. +- **Substrait mapping:** typed registry entries must represent whether a function maps to a core Substrait function, a registered extension function, a deterministic rewrite, or structural relation-context lowering. Backend capability declarations consume the emitted Substrait representation; they do not redefine IncQL semantics. diff --git a/docs/rfcs/015_core_scalar_functions.md b/docs/rfcs/015_core_scalar_functions.md index 9aaa81da..9a122052 100644 --- a/docs/rfcs/015_core_scalar_functions.md +++ b/docs/rfcs/015_core_scalar_functions.md @@ -1,25 +1,25 @@ -# InQL RFC 015: Core scalar functions and operators +# IncQL RFC 015: Core scalar functions and operators - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 003 (`query {}` blocks and relational authoring) -- **Issue:** [InQL #32](https://github.com/encero-systems/InQL/issues/32) -- **RFC PR:** [InQL #43](https://github.com/encero-systems/InQL/pull/43) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 003 (`query {}` blocks and relational authoring) +- **Issue:** [IncQL #32](https://github.com/encero-systems/IncQL/issues/32) +- **RFC PR:** [IncQL #43](https://github.com/encero-systems/IncQL/pull/43) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines the first required scalar function and operator slice for InQL: column references, literals, casts, comparisons, boolean logic, null checks, basic arithmetic, conditionals, membership predicates, range predicates, and ordering expressions. These functions are the minimum scalar vocabulary needed for credible typed dataframe and query-block authoring before broader math, string, and date/time catalogs are added. +This RFC defines the first required scalar function and operator slice for IncQL: column references, literals, casts, comparisons, boolean logic, null checks, basic arithmetic, conditionals, membership predicates, range predicates, and ordering expressions. These functions are the minimum scalar vocabulary needed for credible typed dataframe and query-block authoring before broader math, string, and date/time catalogs are added. ## Motivation -InQL currently has a split helper surface for filters and projections. That is enough for early examples but too narrow for real dataframe work. Authors need one predictable core scalar vocabulary that can express common predicates, computed columns, grouping keys, aggregate arguments, and sort keys across all InQL authoring surfaces. +IncQL currently has a split helper surface for filters and projections. That is enough for early examples but too narrow for real dataframe work. Authors need one predictable core scalar vocabulary that can express common predicates, computed columns, grouping keys, aggregate arguments, and sort keys across all IncQL authoring surfaces. The core slice should be intentionally small. Functions such as advanced trigonometry, regex extraction, JSON parsing, and collection transforms are useful, but they should not delay the basic relational expression contract. @@ -37,14 +37,14 @@ The core slice should be intentionally small. Functions such as advanced trigono - Defining aggregate functions. - Defining window functions. - Defining SQL string parsing. -- Replacing InQL RFC 012's canonical scalar expression model. +- Replacing IncQL RFC 012's canonical scalar expression model. ## Guide-level explanation (how authors think about it) Authors should be able to write everyday filters and computed columns without switching helper families: ```incan -from pub::inql.functions import add, and_, cast, col, coalesce, gt, is_not_null, lit, mul +from pub::incql.functions import add, and_, cast, col, coalesce, gt, is_not_null, lit, mul enriched = ( orders @@ -58,15 +58,15 @@ The exact surface may later gain operator sugar, but the semantic set should be ## Reference-level explanation (precise rules) -InQL must define canonical scalar entries for column reference, literal construction, cast, try-cast, equality, inequality, less-than, less-than-or-equal, greater-than, greater-than-or-equal, null-safe equality, boolean conjunction, boolean disjunction, boolean negation, null tests, NaN tests for floating types, addition, subtraction, multiplication, division, modulo, unary negation, `coalesce`, `nullif`, searched conditional expressions, membership predicates, range predicates, and ordering expressions. +IncQL must define canonical scalar entries for column reference, literal construction, cast, try-cast, equality, inequality, less-than, less-than-or-equal, greater-than, greater-than-or-equal, null-safe equality, boolean conjunction, boolean disjunction, boolean negation, null tests, NaN tests for floating types, addition, subtraction, multiplication, division, modulo, unary negation, `coalesce`, `nullif`, searched conditional expressions, membership predicates, range predicates, and ordering expressions. -Column references must resolve through the same schema rules used by InQL RFC 000 and InQL RFC 012. Literal construction must produce a scalar expression with a statically known literal type unless the literal type is intentionally dynamic. +Column references must resolve through the same schema rules used by IncQL RFC 000 and IncQL RFC 012. Literal construction must produce a scalar expression with a statically known literal type unless the literal type is intentionally dynamic. Strict casts must fail when a value cannot be represented in the target type. `try_cast` must not raise the same value-conversion failure as strict `cast`; it must produce a null or other explicitly specified recoverable result. Ordinary equality and comparison must follow SQL-style three-valued null behavior in relational predicates unless another RFC explicitly chooses a different policy. Null-safe equality must compare nulls as values according to its own registry entry. -Boolean operators must define three-valued logic for nullable boolean operands. If InQL exposes host-language operator sugar later, that sugar must preserve the same truth table. +Boolean operators must define three-valued logic for nullable boolean operands. If IncQL exposes host-language operator sugar later, that sugar must preserve the same truth table. Arithmetic functions must define numeric type promotion and overflow behavior. If exact overflow behavior remains backend-dependent in Draft, implementations must reject ambiguous cases rather than silently changing semantics. @@ -92,7 +92,7 @@ The minimum core scalar catalog is: - predicates: `in_`, `between` - ordering: `asc`, `desc`, `asc_nulls_first`, `asc_nulls_last`, `desc_nulls_first`, `desc_nulls_last` -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces `query {}` syntax may present SQL-familiar operators such as `==`, `>`, `AND`, `OR`, and `IS NULL`, but those operators must lower to the same scalar function semantics defined here. Dataframe methods must consume the same scalar expression model. @@ -103,7 +103,7 @@ Typed literal helpers such as `int_expr`, `float_expr`, `str_expr`, `bool_expr`, ## Alternatives considered - **Wait for the broad function catalog.** Rejected because authors need a stable scalar core before broad catalog work can be coherent. -- **Keep separate filter and projection helper families.** Rejected by InQL RFC 012 because row-level scalar meaning should be unified. +- **Keep separate filter and projection helper families.** Rejected by IncQL RFC 012 because row-level scalar meaning should be unified. - **Use backend SQL expression strings for all scalar expressions.** Rejected because it avoids typed expression checking. ## Drawbacks @@ -114,8 +114,8 @@ Typed literal helpers such as `int_expr`, `float_expr`, `str_expr`, `bool_expr`, ## Layers affected -- **InQL specification** — the core scalar vocabulary must remain consistent with InQL RFC 012 and InQL RFC 014. -- **InQL library package** — public helpers should expose the core scalar entries and selected typed helper entrypoints. +- **IncQL specification** — the core scalar vocabulary must remain consistent with IncQL RFC 012 and IncQL RFC 014. +- **IncQL library package** — public helpers should expose the core scalar entries and selected typed helper entrypoints. - **Incan compiler** — query syntax and any future operator sugar must lower to the same scalar function semantics. - **Execution / interchange** — Prism and Substrait lowering must preserve casts, null-safe equality, boolean logic, ordering null placement, and `try_` behavior. - **Documentation** — scalar function reference docs should distinguish canonical names from typed helper entrypoints. @@ -210,7 +210,7 @@ Typed literal helpers such as `int_expr`, `float_expr`, `str_expr`, `bool_expr`, - **Boolean helper names:** the canonical boolean helper names are `and_`, `or_`, and `not_`; the trailing underscore avoids host-language keyword collisions while keeping SQL-familiar names recognizable. - **`try_cast` failure result:** `try_cast` uses null-on-conversion-failure semantics for this RFC. Typed recoverable error values remain a future extension point rather than part of the core scalar slice. -- **Numeric promotion boundary:** the current v0.3-era InQL package records numeric helper intent and checked helper signatures but does not introduce a package-local numeric promotion table. Lowering must only use mappings that are currently represented honestly; ambiguous numeric behavior must remain explicit instead of silently choosing backend-dependent behavior. +- **Numeric promotion boundary:** the current v0.3-era IncQL package records numeric helper intent and checked helper signatures but does not introduce a package-local numeric promotion table. Lowering must only use mappings that are currently represented honestly; ambiguous numeric behavior must remain explicit instead of silently choosing backend-dependent behavior. - **`in_` scope:** `in_` accepts literal or expression lists in this RFC. Relation-valued subquery membership is a future query-surface feature and is out of scope for this core scalar package slice. - **Registry-backed applications:** structural scalar nodes remain appropriate for `ColumnRefExpr` and typed literals, but function/operator calls such as `add`, `mul`, `eq`, `gt`, `and_`, `or_`, and `cast` are represented as registry-backed scalar function applications rather than one bespoke model per function forever. - **Current lowering boundary:** the implemented RFC 015 slice lowers through registered Substrait extension mappings, built-in Substrait Rex shapes (`Cast`, `SingularOrList`, and `IfThen`), and structural `SortRel.sorts` lowering for ordering helpers. DataFusion is the first execution adapter that consumes the emitted Substrait plan; it does not define the portable helper semantics. diff --git a/docs/rfcs/016_core_aggregate_functions.md b/docs/rfcs/016_core_aggregate_functions.md index 0935bda7..7bdb32db 100644 --- a/docs/rfcs/016_core_aggregate_functions.md +++ b/docs/rfcs/016_core_aggregate_functions.md @@ -1,26 +1,26 @@ -# InQL RFC 016: Core aggregate functions +# IncQL RFC 016: Core aggregate functions - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 001 (dataset carriers and aggregation surface) - - InQL RFC 003 (`query {}` aggregate rules) - - InQL RFC 012 (scalar expressions and aggregate measures) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) -- **Issue:** [InQL #33](https://github.com/encero-systems/InQL/issues/33) -- **RFC PR:** [InQL #44](https://github.com/encero-systems/InQL/pull/44) + - IncQL RFC 001 (dataset carriers and aggregation surface) + - IncQL RFC 003 (`query {}` aggregate rules) + - IncQL RFC 012 (scalar expressions and aggregate measures) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) +- **Issue:** [IncQL #33](https://github.com/encero-systems/IncQL/issues/33) +- **RFC PR:** [IncQL #44](https://github.com/encero-systems/IncQL/pull/44) - **Written against:** Incan v0.2 - **Shipped in:** v0.1 ## Summary -This RFC defines InQL's core aggregate function set: `count`, `sum`, `avg`, `min`, and `max`, including argument forms, input type rules, null behavior, empty-input behavior, result type rules, aliases, and required diagnostics. These aggregates form the minimum portable aggregate surface for dataframe and query-block work. +This RFC defines IncQL's core aggregate function set: `count`, `sum`, `avg`, `min`, and `max`, including argument forms, input type rules, null behavior, empty-input behavior, result type rules, aliases, and required diagnostics. These aggregates form the minimum portable aggregate surface for dataframe and query-block work. ## Motivation -InQL already exposes `sum` and argument-free `count`, but real grouped analysis needs a stable minimum aggregate family. Beam's SQL aggregate surface centers on `COUNT`, `AVG`, `SUM`, `MAX`, and `MIN`; DataFusion, Spark, and Snowflake also support these as foundational aggregates. InQL should define these before broader statistical, collection, approximate, or ordered aggregates. +IncQL already exposes `sum` and argument-free `count`, but real grouped analysis needs a stable minimum aggregate family. Beam's SQL aggregate surface centers on `COUNT`, `AVG`, `SUM`, `MAX`, and `MIN`; DataFusion, Spark, and Snowflake also support these as foundational aggregates. IncQL should define these before broader statistical, collection, approximate, or ordered aggregates. Without explicit rules, aggregate behavior can drift across authoring surfaces and backends. Null skipping, empty groups, `count(*)` versus `count(expr)`, and numeric result type rules are observable semantics, not implementation details. @@ -45,7 +45,7 @@ Without explicit rules, aggregate behavior can drift across authoring surfaces a Authors can summarize grouped or whole-relation data with the core aggregate set: ```incan -from pub::inql.functions import avg, col, count, max, min, sum +from pub::incql.functions import avg, col, count, max, min, sum summary = ( orders @@ -65,7 +65,7 @@ summary = ( ## Reference-level explanation (precise rules) -InQL must define canonical aggregate entries for `count`, `sum`, `avg`, `min`, and `max`. +IncQL must define canonical aggregate entries for `count`, `sum`, `avg`, `min`, and `max`. `count()` must count input rows in the current relation or group. `count(expr)` must count rows where `expr` evaluates to a non-null value. `count_expr(expr)` remains available as a compatibility spelling for `count(expr)`, but must lower through the same canonical aggregate mapping. `count` must return a non-null integer count. For an empty input relation or empty group, `count()` and expression-count semantics must return zero. @@ -77,7 +77,7 @@ InQL must define canonical aggregate entries for `count`, `sum`, `avg`, `min`, a Aggregate outputs must be aggregate measures, not row-level scalar expressions. They may appear only in aggregate-capable positions defined by query blocks, dataframe aggregation methods, or later RFCs. -The registry must record compatibility aliases where useful. `mean` may be an alias for `avg` if InQL accepts dataframe-style naming. Warehouse compatibility names such as `count_if` remain outside this core RFC unless modeled as aggregate modifiers by InQL RFC 017. +The registry must record compatibility aliases where useful. `mean` may be an alias for `avg` if IncQL accepts dataframe-style naming. Warehouse compatibility names such as `count_if` remain outside this core RFC unless modeled as aggregate modifiers by IncQL RFC 017. ## Design details @@ -89,9 +89,9 @@ This RFC requires importable aggregate functions. Query syntax may support SQL s Core aggregates skip null values except for `count()`, which counts rows regardless of null values. Expression-count semantics count non-null expression results and are exposed by `count(expr)`. -Aggregate argument expressions must be scalar expressions under InQL RFC 012. Aggregate arguments must not themselves contain aggregate outputs unless a later RFC defines nested aggregate semantics. +Aggregate argument expressions must be scalar expressions under IncQL RFC 012. Aggregate arguments must not themselves contain aggregate outputs unless a later RFC defines nested aggregate semantics. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces `query {}` blocks must apply grouped-reference rules consistently with these aggregate definitions. Dataframe `.group_by(...).agg(...)` calls must produce the same aggregate semantics as equivalent query-block aggregates. @@ -113,8 +113,8 @@ Existing `sum` and `count` helpers should be treated as compatibility-compatible ## Layers affected -- **InQL specification** — aggregate measures must remain distinct from scalar expressions and consistent with InQL RFC 012. -- **InQL library package** — public aggregate helpers should support the core aggregate set and argument forms. +- **IncQL specification** — aggregate measures must remain distinct from scalar expressions and consistent with IncQL RFC 012. +- **IncQL library package** — public aggregate helpers should support the core aggregate set and argument forms. - **Incan compiler** — query-block aggregate checking must validate grouped and aggregated positions against these rules. - **Execution / interchange** — Prism and Substrait lowering must preserve null skipping, empty-input behavior, and count forms. - **Documentation** — aggregate reference docs should document null and empty-input behavior explicitly. @@ -130,5 +130,5 @@ Existing `sum` and `count` helpers should be treated as compatibility-compatible ## Design Decisions - **Expression-count spelling:** v0.1 exposes canonical `count(expr)` for expression-count semantics. `count_expr(expr)` remains a compatibility spelling that returns the same canonical aggregate measure. -- **Count result type:** the v0.1 package records count as a non-null aggregate count measure and validates concrete execution through DataFusion-backed session tests. A more precise static numeric return type belongs with the broader InQL numeric type policy. +- **Count result type:** the v0.1 package records count as a non-null aggregate count measure and validates concrete execution through DataFusion-backed session tests. A more precise static numeric return type belongs with the broader IncQL numeric type policy. - **Average result type:** the v0.1 package records `avg` as a numeric aggregate and relies on the backend/interchange path for concrete materialized numeric representation. Static decimal/floating promotion rules remain tied to the broader numeric type policy rather than this aggregate helper slice. diff --git a/docs/rfcs/017_aggregate_modifiers.md b/docs/rfcs/017_aggregate_modifiers.md index d10947d6..77d085e6 100644 --- a/docs/rfcs/017_aggregate_modifiers.md +++ b/docs/rfcs/017_aggregate_modifiers.md @@ -1,28 +1,28 @@ -# InQL RFC 017: Aggregate modifiers +# IncQL RFC 017: Aggregate modifiers - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 003 (`query {}` aggregate rules) - - InQL RFC 012 (scalar expressions and aggregate measures) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 016 (core aggregate functions) -- **Issue:** [InQL #34](https://github.com/encero-systems/InQL/issues/34) -- **RFC PR:** [InQL #45](https://github.com/encero-systems/InQL/pull/45) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 003 (`query {}` aggregate rules) + - IncQL RFC 012 (scalar expressions and aggregate measures) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 016 (core aggregate functions) +- **Issue:** [IncQL #34](https://github.com/encero-systems/IncQL/issues/34) +- **RFC PR:** [IncQL #45](https://github.com/encero-systems/IncQL/pull/45) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines aggregate modifiers for InQL: `DISTINCT`, aggregate-local `FILTER`, ordered aggregate input, and compatibility helpers such as `count_if`. These are modeled as modifiers on aggregate measures rather than as a separate function for every combination, so InQL can support SQL-style aggregate semantics without multiplying the catalog unnecessarily. +This RFC defines aggregate modifiers for IncQL: `DISTINCT`, aggregate-local `FILTER`, ordered aggregate input, and compatibility helpers such as `count_if`. These are modeled as modifiers on aggregate measures rather than as a separate function for every combination, so IncQL can support SQL-style aggregate semantics without multiplying the catalog unnecessarily. ## Motivation -Many useful aggregate forms are not truly new aggregate functions. `count_distinct`, `sum_distinct`, `count_if`, and ordered string/list aggregates are better understood as an aggregate plus a modifier. If InQL implements each spelling as an unrelated function, the catalog becomes larger and less coherent while still failing to represent SQL's compositional aggregate shape. +Many useful aggregate forms are not truly new aggregate functions. `count_distinct`, `sum_distinct`, `count_if`, and ordered string/list aggregates are better understood as an aggregate plus a modifier. If IncQL implements each spelling as an unrelated function, the catalog becomes larger and less coherent while still failing to represent SQL's compositional aggregate shape. -DataFusion documents aggregate `FILTER (WHERE ...)` and ordered aggregate forms; Spark exposes many convenience names; Snowflake makes `COUNT_IF`, `LISTAGG`, ordered percentile forms, `MIN_BY`, and `MAX_BY` important warehouse-compatibility cases. InQL should support both a clean canonical model and compatibility aliases that map into it. +DataFusion documents aggregate `FILTER (WHERE ...)` and ordered aggregate forms; Spark exposes many convenience names; Snowflake makes `COUNT_IF`, `LISTAGG`, ordered percentile forms, `MIN_BY`, and `MAX_BY` important warehouse-compatibility cases. IncQL should support both a clean canonical model and compatibility aliases that map into it. ## Goals @@ -44,7 +44,7 @@ DataFusion documents aggregate `FILTER (WHERE ...)` and ordered aggregate forms; Authors should be able to express distinct and filtered aggregates without learning separate function names for every combination: ```incan -from pub::inql.functions import col, count, sum +from pub::incql.functions import col, count, sum summary = ( orders @@ -71,7 +71,7 @@ Ordered aggregate input must define one or more ordering expressions and optiona Compatibility helpers must lower to modifiers when possible. `count_distinct(expr)` should be equivalent to `count(expr).distinct()`. `count_if(predicate)` should be equivalent to `count().filter(predicate)` unless a later RFC chooses different null behavior. Ordered string aggregation names such as `listagg` should lower to ordered aggregate semantics rather than a separate non-composable function family. -If a backend cannot support a modifier faithfully, InQL must report an explicit diagnostic or use a semantics-preserving fallback. It must not ignore the modifier. +If a backend cannot support a modifier faithfully, IncQL must report an explicit diagnostic or use a semantics-preserving fallback. It must not ignore the modifier. ## Design details @@ -83,7 +83,7 @@ This RFC does not require one final public syntax. It permits method-like aggreg Modifiers are part of the aggregate measure. Modifier order must be semantically defined. Filtering happens before distinctness unless a specific aggregate entry defines otherwise; ordering applies to the input sequence seen by order-sensitive aggregates after filtering and distinct handling. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces `query {}` blocks may expose `COUNT(DISTINCT .id)` and `sum(.amount) FILTER (WHERE .status == "paid")` if the grammar supports those spellings. Dataframe method chains may expose method-like modifier builders. Both must produce the same aggregate-measure semantics. @@ -105,8 +105,8 @@ Existing aggregate helpers remain valid. New compatibility helpers such as `coun ## Layers affected -- **InQL specification** — aggregate modifiers must compose with aggregate measures without violating the scalar-versus-aggregate boundary. -- **InQL library package** — aggregate builders should expose modifier construction or compatibility helpers. +- **IncQL specification** — aggregate modifiers must compose with aggregate measures without violating the scalar-versus-aggregate boundary. +- **IncQL library package** — aggregate builders should expose modifier construction or compatibility helpers. - **Incan compiler** — query-block aggregate syntax must lower modifiers faithfully where supported. - **Execution / interchange** — Prism and Substrait lowering must preserve filter, distinct, and ordering semantics or reject unsupported forms. - **Documentation** — aggregate docs should prefer the modifier model and list compatibility helper aliases. diff --git a/docs/rfcs/018_common_scalar_function_catalog.md b/docs/rfcs/018_common_scalar_function_catalog.md index 997db513..7264776e 100644 --- a/docs/rfcs/018_common_scalar_function_catalog.md +++ b/docs/rfcs/018_common_scalar_function_catalog.md @@ -1,15 +1,15 @@ -# InQL RFC 018: Common scalar function catalog +# IncQL RFC 018: Common scalar function catalog - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 015 (core scalar functions and operators) -- **Issue:** [InQL #35](https://github.com/encero-systems/InQL/issues/35) -- **RFC PR:** [InQL #44](https://github.com/encero-systems/InQL/pull/44) (initial math slice); [InQL #54](https://github.com/encero-systems/InQL/pull/54) (full implementation) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 015 (core scalar functions and operators) +- **Issue:** [IncQL #35](https://github.com/encero-systems/IncQL/issues/35) +- **RFC PR:** [IncQL #44](https://github.com/encero-systems/IncQL/pull/44) (initial math slice); [IncQL #54](https://github.com/encero-systems/IncQL/pull/54) (full implementation) - **Written against:** Incan v0.2 - **Shipped in:** v0.1 @@ -19,7 +19,7 @@ This RFC defines the common scalar function catalog beyond the core scalar slice ## Motivation -After the core scalar vocabulary exists, authors still need everyday data cleaning and feature engineering functions. Spark, Snowflake, DataFusion, Arrow, dbt, and SQL systems all provide broad scalar coverage because real tabular work needs string normalization, date extraction, numeric transforms, regex predicates, and parsing helpers. InQL should add that breadth deliberately rather than through scattered helper additions. +After the core scalar vocabulary exists, authors still need everyday data cleaning and feature engineering functions. Spark, Snowflake, DataFusion, Arrow, dbt, and SQL systems all provide broad scalar coverage because real tabular work needs string normalization, date extraction, numeric transforms, regex predicates, and parsing helpers. IncQL should add that breadth deliberately rather than through scattered helper additions. The catalog should still avoid copying every backend-specific function. Functions that require nested types, JSON/CSV values, geospatial types, sketch state, encryption policy, or physical execution metadata belong elsewhere. @@ -42,7 +42,7 @@ The catalog should still avoid copying every backend-specific function. Function Authors should be able to clean and enrich ordinary scalar columns using familiar functions: ```incan -from pub::inql.functions import col, concat, date_trunc, lower, round, substring, trim +from pub::incql.functions import col, concat, date_trunc, lower, round, substring, trim cleaned = ( orders @@ -53,21 +53,21 @@ cleaned = ( ) ``` -Compatibility aliases are useful, but authors should see one canonical InQL name in docs for each semantic function. +Compatibility aliases are useful, but authors should see one canonical IncQL name in docs for each semantic function. ## Reference-level explanation (precise rules) -InQL should define common math functions including `abs`, `ceil`, `floor`, `round`, `sqrt`, `power`, `exp`, `ln`, `log`, `log10`, `sign`, `least`, `greatest`, `sin`, `cos`, `tan`, `asin`, `acos`, `atan`, `atan2`, `degrees`, and `radians`. +IncQL should define common math functions including `abs`, `ceil`, `floor`, `round`, `sqrt`, `power`, `exp`, `ln`, `log`, `log10`, `sign`, `least`, `greatest`, `sin`, `cos`, `tan`, `asin`, `acos`, `atan`, `atan2`, `degrees`, and `radians`. -InQL should define common string functions including `char_length`, `octet_length`, `upper`, `lower`, `trim`, `ltrim`, `rtrim`, `substring`, `position`, `overlay`, `concat`, `concat_ws`, `replace`, `translate`, `repeat`, `left`, `right`, `lpad`, `rpad`, and `split_part`. +IncQL should define common string functions including `char_length`, `octet_length`, `upper`, `lower`, `trim`, `ltrim`, `rtrim`, `substring`, `position`, `overlay`, `concat`, `concat_ws`, `replace`, `translate`, `repeat`, `left`, `right`, `lpad`, `rpad`, and `split_part`. -InQL should define common text encoding functions including `encode`, `decode`, `base64`, `unbase64`, `hex`, and `unhex`, provided their character encoding and invalid-input behavior are specified. +IncQL should define common text encoding functions including `encode`, `decode`, `base64`, `unbase64`, `hex`, and `unhex`, provided their character encoding and invalid-input behavior are specified. -InQL should define common regex functions including `regexp_like`, `regexp_replace`, and `regexp_extract` once the regex flavor and capture semantics are specified. +IncQL should define common regex functions including `regexp_like`, `regexp_replace`, and `regexp_extract` once the regex flavor and capture semantics are specified. -InQL should define common date/time functions including `current_date`, `current_time`, `current_timestamp`, `extract`, `date_part`, `date_trunc`, `time_trunc`, `date_add`, `date_sub`, `date_diff`, `timestamp_diff`, `to_date`, `to_time`, `to_timestamp`, `from_unixtime`, `unix_seconds`, `unix_millis`, `unix_micros`, `make_date`, `make_time`, `make_timestamp`, and `last_day`. +IncQL should define common date/time functions including `current_date`, `current_time`, `current_timestamp`, `extract`, `date_part`, `date_trunc`, `time_trunc`, `date_add`, `date_sub`, `date_diff`, `timestamp_diff`, `to_date`, `to_time`, `to_timestamp`, `from_unixtime`, `unix_seconds`, `unix_millis`, `unix_micros`, `make_date`, `make_time`, `make_timestamp`, and `last_day`. -The catalog should explicitly account for dbt-style portability names such as `dateadd`, `datediff`, and `safe_cast`. These names may be canonical helpers or compatibility aliases, but their adapter-specific rendering requirements must be represented through the registry rather than undocumented backend conditionals. Deterministic hash helpers belong to InQL RFC 022. Semi-structured type and variant helpers belong to InQL RFC 026. +The catalog should explicitly account for dbt-style portability names such as `dateadd`, `datediff`, and `safe_cast`. These names may be canonical helpers or compatibility aliases, but their adapter-specific rendering requirements must be represented through the registry rather than undocumented backend conditionals. Deterministic hash helpers belong to IncQL RFC 022. Semi-structured type and variant helpers belong to IncQL RFC 026. Every function added by this RFC must be entered in the function registry with type rules, null behavior, determinism, and backend support. Current time functions must be marked stable within a query or explicitly nondeterministic; the registry must not leave that ambiguous. @@ -85,9 +85,9 @@ String functions must use one-based SQL-compatible positions for functions such Regex functions use a safe Rust-regex-compatible flavor: no lookaround or backreferences are guaranteed by the portable contract. `regexp_extract` defaults to capture group `1`; group `0` returns the full match. -Current date, time, and timestamp helpers are registry-marked as nondeterministic because InQL's registry has no separate statement-stable determinism category. Backends may evaluate them statement-stably, but InQL authors must not rely on them as deterministic pure functions. +Current date, time, and timestamp helpers are registry-marked as nondeterministic because IncQL's registry has no separate statement-stable determinism category. Backends may evaluate them statement-stably, but IncQL authors must not rely on them as deterministic pure functions. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces The same scalar catalog must be usable in filters, computed projections, grouping keys where allowed, aggregate arguments, and query-block expressions. Function availability must not depend on whether the author used dataframe methods or query blocks. @@ -97,8 +97,8 @@ Existing arithmetic helpers should be treated as the start of this catalog but s ## Alternatives considered -- **Add all Spark scalar functions.** Rejected because many Spark functions are format-specific, engine-specific, physical-execution-specific, or require types InQL has not standardized. -- **Only expose DataFusion functions.** Rejected because backend availability should inform lowering, not define the portable InQL author contract. +- **Add all Spark scalar functions.** Rejected because many Spark functions are format-specific, engine-specific, physical-execution-specific, or require types IncQL has not standardized. +- **Only expose DataFusion functions.** Rejected because backend availability should inform lowering, not define the portable IncQL author contract. - **Delay scalar breadth until every edge case is settled.** Rejected because practical data cleaning needs a common catalog, but unresolved families can remain Draft until semantics are precise. ## Drawbacks @@ -109,8 +109,8 @@ Existing arithmetic helpers should be treated as the start of this catalog but s ## Layers affected -- **InQL specification** — the common scalar catalog must extend the registry without contradicting core scalar semantics. -- **InQL library package** — public helpers should expose canonical names and selected aliases. +- **IncQL specification** — the common scalar catalog must extend the registry without contradicting core scalar semantics. +- **IncQL library package** — public helpers should expose canonical names and selected aliases. - **Incan compiler** — query-block operator and keyword forms should lower to registry entries where applicable. - **Execution / interchange** — Prism and Substrait lowering must either preserve semantics, use registered extensions, or diagnose unsupported functions. - **Documentation** — function reference docs should group scalar functions by family and show aliases. @@ -125,7 +125,7 @@ Existing arithmetic helpers should be treated as the start of this catalog but s ### Phase 2: Complete common scalar catalog - Add the remaining math helpers, string helpers, text encoding helpers, regex helpers, date/time helpers, and compatibility aliases. -- Preserve InQL semantics through registry metadata and Substrait extension mappings, with adapter-owned DataFusion UDFs only where DataFusion has no direct API-level equivalent. +- Preserve IncQL semantics through registry metadata and Substrait extension mappings, with adapter-owned DataFusion UDFs only where DataFusion has no direct API-level equivalent. - Cover registry metadata, Substrait lowering, expression shape, and concrete DataFusion-backed session execution. ## Progress Checklist @@ -152,5 +152,5 @@ Existing arithmetic helpers should be treated as the start of this catalog but s - **Regex flavor:** RFC 018 regex helpers use Rust-regex-compatible semantics. The portable contract does not require lookaround or backreference support. - **Regex captures:** `regexp_extract(expr, pattern)` defaults to capture group `1`; group `0` returns the full match. - **Current time determinism:** `current_date`, `current_time`, and `current_timestamp` are registry-marked nondeterministic until the registry has a statement-stable category. -- **Encoding scope:** RFC 018 encoding helpers are text-to-text helpers. Binary and semi-structured logical value surfaces belong to InQL RFC 026 rather than hidden DataFusion-specific behavior. -- **DataFusion boundary:** DataFusion native functions are used where they preserve InQL semantics directly. Adapter UDFs are DataFusion execution details and do not own the InQL semantic contract. +- **Encoding scope:** RFC 018 encoding helpers are text-to-text helpers. Binary and semi-structured logical value surfaces belong to IncQL RFC 026 rather than hidden DataFusion-specific behavior. +- **DataFusion boundary:** DataFusion native functions are used where they preserve IncQL semantics directly. Adapter UDFs are DataFusion execution details and do not own the IncQL semantic contract. diff --git a/docs/rfcs/019_window_functions.md b/docs/rfcs/019_window_functions.md index 72d2aa96..60b9c604 100644 --- a/docs/rfcs/019_window_functions.md +++ b/docs/rfcs/019_window_functions.md @@ -1,26 +1,26 @@ -# InQL RFC 019: Window functions +# IncQL RFC 019: Window functions - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 003 (`query {}` blocks and relational authoring) - - InQL RFC 012 (scalar expressions and aggregate measures) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 016 (core aggregate functions) -- **Issue:** [InQL #36](https://github.com/encero-systems/InQL/issues/36) -- **RFC PR:** [InQL #48](https://github.com/encero-systems/InQL/pull/48) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 003 (`query {}` blocks and relational authoring) + - IncQL RFC 012 (scalar expressions and aggregate measures) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 016 (core aggregate functions) +- **Issue:** [IncQL #36](https://github.com/encero-systems/IncQL/issues/36) +- **RFC PR:** [IncQL #48](https://github.com/encero-systems/IncQL/pull/48) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines InQL window functions and window specifications: partitioning, ordering, frames, ranking functions, offset functions, and value functions. Window functions are explicitly not ordinary aggregates; they produce one value per input row while seeing a related set of rows defined by the window specification. +This RFC defines IncQL window functions and window specifications: partitioning, ordering, frames, ranking functions, offset functions, and value functions. Window functions are explicitly not ordinary aggregates; they produce one value per input row while seeing a related set of rows defined by the window specification. ## Motivation -Analytic dataframe work needs ranking, lag/lead comparisons, running totals, and first/last value access. Spark and SQL systems expose these through window functions, and DataFusion distinguishes ordered-set and aggregate behavior from window behavior. InQL should preserve that distinction instead of modeling window functions as ordinary aggregates or scalar helpers. +Analytic dataframe work needs ranking, lag/lead comparisons, running totals, and first/last value access. Spark and SQL systems expose these through window functions, and DataFusion distinguishes ordered-set and aggregate behavior from window behavior. IncQL should preserve that distinction instead of modeling window functions as ordinary aggregates or scalar helpers. Window functions also force a clearer relation between row-level expressions and group-level aggregates. A windowed `sum` may produce one value per row, but it still has aggregate-like input semantics within a window frame. @@ -43,7 +43,7 @@ Window functions also force a clearer relation between row-level expressions and Authors can rank rows within a partition using the builder surface: ```incan -from pub::inql.functions import col, current_row, desc, lag, rank, sum, unbounded_preceding, window +from pub::incql.functions import col, current_row, desc, lag, rank, sum, unbounded_preceding, window ranked = ( orders @@ -65,13 +65,13 @@ The exact query-block syntax may evolve, but authors should understand that a wi ## Reference-level explanation (precise rules) -InQL must define a window specification containing partition expressions, ordering expressions, and an optional frame. Partition expressions and ordering expressions must be scalar expressions. +IncQL must define a window specification containing partition expressions, ordering expressions, and an optional frame. Partition expressions and ordering expressions must be scalar expressions. -InQL must define ranking functions `row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, and `ntile`. Ranking functions must require an ordering unless a function's registry entry explicitly permits unordered use. +IncQL must define ranking functions `row_number`, `rank`, `dense_rank`, `percent_rank`, `cume_dist`, and `ntile`. Ranking functions must require an ordering unless a function's registry entry explicitly permits unordered use. -InQL must define offset functions `lag` and `lead`. Offset functions must accept a scalar input expression, an optional positive integer offset, and an optional default value whose type is compatible with the input expression. +IncQL must define offset functions `lag` and `lead`. Offset functions must accept a scalar input expression, an optional positive integer offset, and an optional default value whose type is compatible with the input expression. -InQL must define value functions `first_value`, `last_value`, and `nth_value`. Value and offset calls support explicit `RESPECT NULLS` and `IGNORE NULLS` metadata through method modifiers on the unplaced window call. +IncQL must define value functions `first_value`, `last_value`, and `nth_value`. Value and offset calls support explicit `RESPECT NULLS` and `IGNORE NULLS` metadata through method modifiers on the unplaced window call. Windowed aggregate calls may reuse aggregate functions over a window specification. They must still obey aggregate input type rules, but their result is a row-level value in the surrounding projection. @@ -89,13 +89,13 @@ Window frames may be row-based or range-based. Frame start and end must be expli Ordering null placement must follow the ordering expression rules defined by the scalar function catalog. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Query blocks may expose SQL-style window syntax. Dataframe methods may expose builder-style window specs. Both must use the same function registry entries and window specification semantics. ### Compatibility / migration -No current InQL function should be reclassified silently as a window function. Aggregate names reused in window contexts must be position-sensitive and diagnosable. +No current IncQL function should be reclassified silently as a window function. Aggregate names reused in window contexts must be position-sensitive and diagnosable. ## Alternatives considered @@ -111,9 +111,9 @@ No current InQL function should be reclassified silently as a window function. A ## Layers affected -- **InQL specification** — window functions must be distinguished from scalar and aggregate functions. -- **InQL library package** — public helpers should expose window function and window specification builders. -- **Incan compiler / InQL authoring surfaces** — checked call signatures and future query syntax must enforce window function placement, partition expressions, ordering expressions, and frame bounds. +- **IncQL specification** — window functions must be distinguished from scalar and aggregate functions. +- **IncQL library package** — public helpers should expose window function and window specification builders. +- **Incan compiler / IncQL authoring surfaces** — checked call signatures and future query syntax must enforce window function placement, partition expressions, ordering expressions, and frame bounds. - **Execution / interchange** — Prism and Substrait lowering must preserve window partitioning, ordering, frames, and function identity. - **Documentation** — docs should clearly separate aggregate functions from window functions. @@ -122,7 +122,7 @@ No current InQL function should be reclassified silently as a window function. A ### Resolved - The implemented package surface exposes explicit `with_window_column(...)` projection-like placement rather than accepting window functions in arbitrary scalar-expression positions. -- Ranking helpers require explicit `order_by(...)` in the window spec. InQL does not invent a silent default ordering. +- Ranking helpers require explicit `order_by(...)` in the window spec. IncQL does not invent a silent default ordering. - Distribution, offset, and value helpers use the same relation-aware placement model as ranking helpers. - Aggregate helpers may be placed over windows through `AggregateMeasure.over(...)`; invalid aggregate modifier combinations are rejected before backend execution. - Window specs use a documented whole-partition default row frame, and explicit `rows_between(...)` / `range_between(...)` calls preserve frame bounds through Substrait lowering. diff --git a/docs/rfcs/020_nested_data_functions.md b/docs/rfcs/020_nested_data_functions.md index d15b841d..d29096bb 100644 --- a/docs/rfcs/020_nested_data_functions.md +++ b/docs/rfcs/020_nested_data_functions.md @@ -1,26 +1,26 @@ -# InQL RFC 020: Nested data functions +# IncQL RFC 020: Nested data functions - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 000 (schema shapes and model-driven typing) - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 021 (generator and table-valued functions) -- **Issue:** [InQL #37](https://github.com/encero-systems/InQL/issues/37) -- **RFC PR:** [InQL #46](https://github.com/encero-systems/InQL/pull/46) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 000 (schema shapes and model-driven typing) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 021 (generator and table-valued functions) +- **Issue:** [IncQL #37](https://github.com/encero-systems/IncQL/issues/37) +- **RFC PR:** [IncQL #46](https://github.com/encero-systems/IncQL/pull/46) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines InQL functions for nested scalar values: arrays, maps, and structs. It covers construction, element access, cardinality, containment, overlap checks, sorting, set-like array operations, scalar array flattening, map entry access, and higher-order collection functions as a later extension point. Nested functions remain scalar when they produce one value per input row; cardinality-changing operations such as `explode` belong to a separate generator RFC. +This RFC defines IncQL functions for nested scalar values: arrays, maps, and structs. It covers construction, element access, cardinality, containment, overlap checks, sorting, set-like array operations, scalar array flattening, map entry access, and higher-order collection functions as a later extension point. Nested functions remain scalar when they produce one value per input row; cardinality-changing operations such as `explode` belong to a separate generator RFC. ## Motivation -Modern dataframe and warehouse systems routinely handle nested data. Spark has a large array/map/struct catalog, Snowflake has ARRAY/OBJECT/MAP and semi-structured VARIANT-oriented functions, Arrow and DataFusion support nested physical types, and Beam schemas support nested rows and collections. InQL needs nested data functions for realistic semi-structured data without confusing scalar nested values with relation-shaping generators. +Modern dataframe and warehouse systems routinely handle nested data. Spark has a large array/map/struct catalog, Snowflake has ARRAY/OBJECT/MAP and semi-structured VARIANT-oriented functions, Arrow and DataFusion support nested physical types, and Beam schemas support nested rows and collections. IncQL needs nested data functions for realistic semi-structured data without confusing scalar nested values with relation-shaping generators. The split matters. `array_contains(.items, "x")` is a row-level scalar predicate. `explode(.items)` changes the number of rows and must be modeled differently. @@ -44,7 +44,7 @@ The split matters. `array_contains(.items, "x")` is a row-level scalar predicate Authors can inspect and manipulate nested values without changing relation cardinality: ```incan -from pub::inql.functions import array_contains, cardinality, col, element_at, lit, map_keys +from pub::incql.functions import array_contains, cardinality, col, element_at, lit, map_keys enriched = ( events @@ -59,19 +59,19 @@ If an author wants one output row per item, that is a generator/table-valued ope ## Reference-level explanation (precise rules) -InQL defines array construction with `array`, struct construction with `named_struct`, and map construction with `map_from_arrays`. +IncQL defines array construction with `array`, struct construction with `named_struct`, and map construction with `map_from_arrays`. -InQL defines `cardinality` as the canonical size function for arrays and maps. Compatibility aliases such as `size`, `array_size`, and `array_length` may resolve to `cardinality` where semantics match, but the initial implemented surface keeps the canonical spelling. +IncQL defines `cardinality` as the canonical size function for arrays and maps. Compatibility aliases such as `size`, `array_size`, and `array_length` may resolve to `cardinality` where semantics match, but the initial implemented surface keeps the canonical spelling. -InQL defines array element access with `element_at(array_expr, index)`. Indexes are one-based. Current lowering maps to the portable array-element adapter path and uses the backend adapter's recoverable out-of-range behavior until InQL has a richer static/runtime error-policy split for strict versus try-style element access. +IncQL defines array element access with `element_at(array_expr, index)`. Indexes are one-based. Current lowering maps to the portable array-element adapter path and uses the backend adapter's recoverable out-of-range behavior until IncQL has a richer static/runtime error-policy split for strict versus try-style element access. -InQL defines array predicates and transforms including `array_contains`, `array_position`, `array_sort`, `array_distinct`, `array_except`, `array_intersect`, `array_union`, `array_join`, `arrays_overlap`, `array_flatten`, `array_slice`, and `array_reverse` where type and null semantics are specified by the registry and backend adapter boundary. The scalar array-flattening helper is named `array_flatten` so table-valued or generator `flatten` remains available for RFC 021. +IncQL defines array predicates and transforms including `array_contains`, `array_position`, `array_sort`, `array_distinct`, `array_except`, `array_intersect`, `array_union`, `array_join`, `arrays_overlap`, `array_flatten`, `array_slice`, and `array_reverse` where type and null semantics are specified by the registry and backend adapter boundary. The scalar array-flattening helper is named `array_flatten` so table-valued or generator `flatten` remains available for RFC 021. -InQL defines map functions including `map_contains_key`, `map_entries`, `map_extract`, `map_from_arrays`, `map_keys`, and `map_values`. +IncQL defines map functions including `map_contains_key`, `map_entries`, `map_extract`, `map_from_arrays`, `map_keys`, and `map_values`. Object-style warehouse functions such as `object_construct`, `object_construct_keep_null`, `object_delete`, `object_insert`, `object_keys`, and `object_pick` are accounted for as semi-structured and dynamic-object concerns. They should be modeled through typed object/map semantics where possible and through the RFC 022 semi-structured family only when dynamic value semantics are required. -Higher-order functions such as `transform`, `filter`, `exists`, `forall`, `aggregate`, `reduce`, `zip_with`, `map_filter`, `transform_keys`, and `transform_values` must not reach Planned status until lambda or equivalent callback semantics are specified for InQL expressions. +Higher-order functions such as `transform`, `filter`, `exists`, `forall`, `aggregate`, `reduce`, `zip_with`, `map_filter`, `transform_keys`, and `transform_values` must not reach Planned status until lambda or equivalent callback semantics are specified for IncQL expressions. ## Design details @@ -81,17 +81,17 @@ This RFC requires importable function forms. Literal syntax for arrays, maps, or ### Semantics -Nested scalar functions produce one value per input row. A function that changes row count, expands fields into multiple columns, or returns a relation belongs to InQL RFC 021. +Nested scalar functions produce one value per input row. A function that changes row count, expands fields into multiple columns, or returns a relation belongs to IncQL RFC 021. Index origin, invalid-index behavior, null container behavior, null element behavior, and duplicate-map-key behavior must be specified before the corresponding functions reach Planned status. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Nested functions may appear wherever scalar expressions of their result type are valid. Grouping by nested values is not documented as portable until equality and ordering semantics for nested values are fully specified. ### Compatibility / migration -No current InQL APIs are expected to break. Nested functions should be additive and gated by type support. +No current IncQL APIs are expected to break. Nested functions should be additive and gated by type support. ## Alternatives considered @@ -107,8 +107,8 @@ No current InQL APIs are expected to break. Nested functions should be additive ## Layers affected -- **InQL specification** — nested scalar functions must fit the scalar expression model without changing relation cardinality. -- **InQL library package** — public helpers should expose nested constructors, accessors, and predicates. +- **IncQL specification** — nested scalar functions must fit the scalar expression model without changing relation cardinality. +- **IncQL library package** — public helpers should expose nested constructors, accessors, and predicates. - **Incan compiler** — typechecking must understand nested collection, map, and struct result types where functions are used. - **Execution / interchange** — Prism and Substrait lowering must preserve nested value semantics or diagnose unsupported operations. - **Documentation** — docs should separate nested scalar operations from generator functions. @@ -121,4 +121,4 @@ No current InQL APIs are expected to break. Nested functions should be additive - `element_at(...)` uses the current adapter's recoverable array-element behavior for out-of-range indexes. A separate strict/try split is deferred until registry error policy can distinguish static validation failures from runtime recoverable results. - Grouping and ordering over arrays, maps, and structs are not documented as portable in the initial implementation. - Scalar `array_flatten(...)` is separate from RFC 021 table-valued or generator flattening. -- Higher-order collection functions remain deferred until InQL expression callback or lambda semantics are specified. +- Higher-order collection functions remain deferred until IncQL expression callback or lambda semantics are specified. diff --git a/docs/rfcs/021_generator_table_functions.md b/docs/rfcs/021_generator_table_functions.md index 8e030702..95fb8a51 100644 --- a/docs/rfcs/021_generator_table_functions.md +++ b/docs/rfcs/021_generator_table_functions.md @@ -1,29 +1,29 @@ -# InQL RFC 021: Generator and table-valued functions +# IncQL RFC 021: Generator and table-valued functions - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 001 (dataset carriers and relation operations) - - InQL RFC 003 (`query {}` clause inventory) - - InQL RFC 006 (unnest/explode Substrait lowering) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 020 (nested data functions) -- **Issue:** [InQL #38](https://github.com/encero-systems/InQL/issues/38) -- **RFC PR:** [InQL #47](https://github.com/encero-systems/InQL/pull/47) + - IncQL RFC 001 (dataset carriers and relation operations) + - IncQL RFC 003 (`query {}` clause inventory) + - IncQL RFC 006 (unnest/explode Substrait lowering) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 020 (nested data functions) +- **Issue:** [IncQL #38](https://github.com/encero-systems/IncQL/issues/38) +- **RFC PR:** [IncQL #47](https://github.com/encero-systems/IncQL/pull/47) - **Written against:** Incan v0.2 - **Shipped in:** v0.1 ## Summary -This RFC defines generator and table-valued functions for InQL, including `explode`, `explode_outer`, `posexplode`, `posexplode_outer`, `inline`, `inline_outer`, `flatten`, `stack`, and selected tuple-producing extraction helpers. These functions change relation shape or cardinality and therefore must be modeled as relation operations, not scalar expressions. +This RFC defines generator and table-valued functions for IncQL, including `explode`, `explode_outer`, `posexplode`, `posexplode_outer`, `inline`, `inline_outer`, `flatten`, `stack`, and selected tuple-producing extraction helpers. These functions change relation shape or cardinality and therefore must be modeled as relation operations, not scalar expressions. ## Motivation Spark exposes generators near functions, and Snowflake exposes `FLATTEN` as a table function, but their semantics are fundamentally different from scalar functions. `explode` and `flatten` turn one input row into zero or more output rows. `inline` can turn nested fields into multiple output columns. `stack` constructs multiple rows. Treating these as scalar functions would make planning, typing, and lowering unsound. -InQL already has an unnest/explode design direction through its Substrait work. This RFC gives the broader generator family a clear semantic home. +IncQL already has an unnest/explode design direction through its Substrait work. This RFC gives the broader generator family a clear semantic home. ## Goals @@ -45,7 +45,7 @@ InQL already has an unnest/explode design direction through its Substrait work. Authors should use generators when one input row may become multiple output rows. In the current builder surface, generators are constructed as explicit applications and then applied to a relation: ```incan -from pub::inql.functions import col, explode +from pub::incql.functions import col, explode items = ( orders @@ -64,7 +64,7 @@ Generator functions must be registry entries with function class `generator` or `explode_outer(array_expr)` must preserve the input row when the input array is null or empty and must produce a null generated value according to its output schema. -`posexplode(array_expr)` and `posexplode_outer(array_expr)` must include a positional output column in addition to the generated element. Positional output is zero-based because `posexplode` follows the Spark-compatible naming convention rather than InQL's one-based scalar collection indexing rule. +`posexplode(array_expr)` and `posexplode_outer(array_expr)` must include a positional output column in addition to the generated element. Positional output is zero-based because `posexplode` follows the Spark-compatible naming convention rather than IncQL's one-based scalar collection indexing rule. `inline(array_of_struct_expr)` must expand each struct element into output columns. `inline_outer` must preserve outer rows for null or empty input according to the outer generator rule. @@ -84,7 +84,7 @@ Generators may appear as dataframe relation methods, query-block clauses, or tab Generator output schema is part of the relation schema after the generator operation. The initial portable generator applications preserve all input columns and append generated output columns in declaration order. Generated aliases are required, must be non-empty, and must not collide with existing columns. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces `query {}` may expose an `EXPLODE` clause or table-valued function syntax when the query surface is available. Dataframe APIs expose the same semantic target through `generate(...)` and registry-backed generator helpers. Both use the same generator semantics. @@ -106,8 +106,8 @@ Existing unnest/explode behavior should align with this RFC. If current behavior ## Layers affected -- **InQL specification** — generator functions must be a relation-shaping class distinct from scalar functions. -- **InQL library package** — public APIs should expose generator operations with explicit output aliases. +- **IncQL specification** — generator functions must be a relation-shaping class distinct from scalar functions. +- **IncQL library package** — public APIs should expose generator operations with explicit output aliases. - **Incan compiler** — query syntax must constrain generator placement and update relation schemas. - **Execution / interchange** — Prism and Substrait lowering must represent cardinality changes and output schemas faithfully. - **Documentation** — generator docs should explain cardinality and schema effects before listing helper names. diff --git a/docs/rfcs/022_semi_structured_format_functions.md b/docs/rfcs/022_semi_structured_format_functions.md index 5204145e..5a7708be 100644 --- a/docs/rfcs/022_semi_structured_format_functions.md +++ b/docs/rfcs/022_semi_structured_format_functions.md @@ -1,28 +1,28 @@ -# InQL RFC 022: Semi-structured and format functions +# IncQL RFC 022: Semi-structured and format functions - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 009 (session format handler registry) - - InQL RFC 010 (CSV dialect and interpretation contract) - - InQL RFC 011 (source discovery and parse-unit expansion) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 020 (nested data functions) - - InQL RFC 026 (semi-structured variant logical values) -- **Issue:** [InQL #39](https://github.com/encero-systems/InQL/issues/39) -- **RFC PR:** [InQL #49](https://github.com/encero-systems/InQL/pull/49) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 009 (session format handler registry) + - IncQL RFC 010 (CSV dialect and interpretation contract) + - IncQL RFC 011 (source discovery and parse-unit expansion) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 020 (nested data functions) + - IncQL RFC 026 (semi-structured variant logical values) +- **Issue:** [IncQL #39](https://github.com/encero-systems/IncQL/issues/39) +- **RFC PR:** [IncQL #49](https://github.com/encero-systems/IncQL/pull/49) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines InQL's semi-structured and format-oriented function families: JSON value functions, CSV value functions, schema inference helpers, URL helpers, and hashing functions. These functions are practical data-engineering tools, but they should live in explicit format families rather than the core scalar catalog. Semi-structured variant values and their type predicates are defined separately by InQL RFC 026. +This RFC defines IncQL's semi-structured and format-oriented function families: JSON value functions, CSV value functions, schema inference helpers, URL helpers, and hashing functions. These functions are practical data-engineering tools, but they should live in explicit format families rather than the core scalar catalog. Semi-structured variant values and their type predicates are defined separately by IncQL RFC 026. ## Motivation -Real data pipelines frequently parse JSON strings, emit JSON values, inspect CSV-shaped payloads, hash identifiers, and normalize URLs. Spark and Snowflake expose many of these as functions, while InQL already has separate source-format RFCs. The design should preserve that boundary: reading a CSV file is an I/O concern, but parsing a CSV-encoded scalar value is a scalar format function. +Real data pipelines frequently parse JSON strings, emit JSON values, inspect CSV-shaped payloads, hash identifiers, and normalize URLs. Spark and Snowflake expose many of these as functions, while IncQL already has separate source-format RFCs. The design should preserve that boundary: reading a CSV file is an I/O concern, but parsing a CSV-encoded scalar value is a scalar format function. Without a separate RFC, format helpers risk leaking ingestion policy into the scalar catalog or duplicating schema inference semantics from the session/source-discovery layer. @@ -30,7 +30,7 @@ Without a separate RFC, format helpers risk leaking ingestion policy into the sc - Define JSON scalar and schema helper functions. - Define CSV scalar and schema helper functions. -- Keep string-backed format helpers compatible with the variant value model defined by InQL RFC 026. +- Keep string-backed format helpers compatible with the variant value model defined by IncQL RFC 026. - Define URL parse/encode/decode helpers. - Define deterministic hash functions for data engineering. - Keep format functions separate from source reading and writing contracts. @@ -39,7 +39,7 @@ Without a separate RFC, format helpers risk leaking ingestion policy into the sc - Defining source discovery, file scanning, or format handler registration. - Defining XML, geospatial, crypto, or sketch functions. -- Defining semi-structured variant logical values or variant predicates; see InQL RFC 026. +- Defining semi-structured variant logical values or variant predicates; see IncQL RFC 026. - Defining nested array/map/struct functions except as return values of parsing functions. - Defining physical input-file metadata functions. @@ -48,7 +48,7 @@ Without a separate RFC, format helpers risk leaking ingestion policy into the sc Authors should be able to parse and produce semi-structured scalar values inside relational transformations: ```incan -from pub::inql.functions import col, from_json, get_json_object, sha2, to_json +from pub::incql.functions import col, from_json, get_json_object, sha2, to_json model EventPayload: type_ as "type": str @@ -66,15 +66,15 @@ Reading a JSON file into a dataset remains a session/source operation, not a sca ## Reference-level explanation (precise rules) -InQL should define JSON functions including `from_json`, `to_json`, `get_json_object`, `json_array_length`, `json_object_keys`, `schema_of_json`, `parse_json`, `check_json`, `json_extract_path_text`, and `try_from_json` where recoverable parse behavior is desired. Explicit-schema JSON helpers derive schema descriptions from Incan model type parameters. +IncQL should define JSON functions including `from_json`, `to_json`, `get_json_object`, `json_array_length`, `json_object_keys`, `schema_of_json`, `parse_json`, `check_json`, `json_extract_path_text`, and `try_from_json` where recoverable parse behavior is desired. Explicit-schema JSON helpers derive schema descriptions from Incan model type parameters. -InQL should define CSV value functions including `from_csv`, `to_csv`, and `schema_of_csv` only insofar as they operate on scalar values or schema descriptions. `from_csv[Model](...)` returns a logical map keyed by fields from the supplied Incan model type. These functions must not replace the session CSV read/write contract. +IncQL should define CSV value functions including `from_csv`, `to_csv`, and `schema_of_csv` only insofar as they operate on scalar values or schema descriptions. `from_csv[Model](...)` returns a logical map keyed by fields from the supplied Incan model type. These functions must not replace the session CSV read/write contract. -InQL should define URL functions including `parse_url`, `url_encode`, `url_decode`, and `try_url_decode`, with exact invalid-input behavior recorded in the registry. `parse_url` extracts query parameter values by key. +IncQL should define URL functions including `parse_url`, `url_encode`, `url_decode`, and `try_url_decode`, with exact invalid-input behavior recorded in the registry. `parse_url` extracts query parameter values by key. -InQL should define hash functions including `crc32`, `md5`, `sha1`, `sha2`, and `xxhash64`, with input encoding and output representation specified. +IncQL should define hash functions including `crc32`, `md5`, `sha1`, `sha2`, and `xxhash64`, with input encoding and output representation specified. -InQL RFC 026 defines semi-structured variant logical values and their type inspection predicates. Parser helpers in this RFC return validated, normalized payload text. RFC 026 must not silently change the meaning of the RFC 022 string-backed payload helpers. +IncQL RFC 026 defines semi-structured variant logical values and their type inspection predicates. Parser helpers in this RFC return validated, normalized payload text. RFC 026 must not silently change the meaning of the RFC 022 string-backed payload helpers. Schema inference helper functions must be deterministic for the same input values and options. They must not inspect external files or session state unless explicitly defined as source-discovery functions outside this RFC. @@ -90,7 +90,7 @@ Strict parsing functions must fail on invalid input according to their registry Hash functions must define whether they operate on UTF-8 string bytes, binary bytes, or typed value encodings. A hash over a typed value must not silently change encoding by backend. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Format scalar functions may be used anywhere scalar expressions of their result type are valid. Source reading and writing remain governed by the session and format handler RFCs. @@ -102,7 +102,7 @@ This RFC is additive. It should not change existing CSV ingestion behavior. - **Place all format helpers in the common scalar catalog.** Rejected because format parsing has option, schema, and I/O-adjacent concerns that deserve a separate boundary. - **Make JSON and CSV functions source-only.** Rejected because scalar payload parsing is common inside already-loaded datasets. -- **Add full XML and variant support in the same RFC.** Rejected because XML and semi-structured variant values need their own type and compatibility discussion. InQL RFC 026 owns the variant value model. +- **Add full XML and variant support in the same RFC.** Rejected because XML and semi-structured variant values need their own type and compatibility discussion. IncQL RFC 026 owns the variant value model. ## Drawbacks @@ -112,8 +112,8 @@ This RFC is additive. It should not change existing CSV ingestion behavior. ## Layers affected -- **InQL specification** — format functions must stay distinct from source and sink contracts. -- **InQL library package** — public helpers should expose JSON, CSV scalar, URL, and hash functions with explicit scalar argument contracts. +- **IncQL specification** — format functions must stay distinct from source and sink contracts. +- **IncQL library package** — public helpers should expose JSON, CSV scalar, URL, and hash functions with explicit scalar argument contracts. - **Incan compiler** — typechecking must validate current scalar argument shapes; no new compiler syntax is required by this RFC. - **Execution / interchange** — Prism and Substrait lowering must preserve parser options, hash encodings, and scalar payload contracts or diagnose unsupported functions. - **Documentation** — docs should distinguish scalar format functions from session read/write APIs. @@ -128,7 +128,7 @@ This RFC is additive. It should not change existing CSV ingestion behavior. - URL helpers accept scalar URL or component strings. `parse_url(...)` returns the first query parameter value for a literal key. `url_decode(...)` is strict and fails malformed percent escapes; `try_url_decode(...)` returns null for malformed percent escapes. - JSON helpers accept scalar JSON payload strings. Strict helpers fail invalid JSON; `try_from_json[Model](...)` returns null for invalid JSON; schema and path helpers are deterministic over the provided payload, model type parameter, and literal path arguments. - CSV helpers accept scalar row strings and explicit Incan model type parameters. Parsed CSV rows are logical maps rather than JSON text. They operate on payload values only and do not replace the session CSV read/write contract. -- Semi-structured variant predicates such as `typeof`, `is_array`, `is_object`, `is_integer`, `is_timestamp`, and `is_null_value` belong to InQL RFC 026. RFC 022 does not accept them as JSON-text parser helpers. +- Semi-structured variant predicates such as `typeof`, `is_array`, `is_object`, `is_integer`, `is_timestamp`, and `is_null_value` belong to IncQL RFC 026. RFC 022 does not accept them as JSON-text parser helpers. ## Implementation Plan @@ -138,7 +138,7 @@ This RFC is additive. It should not change existing CSV ingestion behavior. 4. Keep `sha2(...)` as a compatibility rewrite over concrete helpers rather than a second mapping. 5. Add focused helper, registry, Substrait lowering, and DataFusion-backed session tests with concrete output values across the full helper set. 6. Add user-facing format-function docs and release notes. -7. Record semi-structured variant values as InQL RFC 026 rather than accepting fake JSON-text predicates in RFC 022. +7. Record semi-structured variant values as IncQL RFC 026 rather than accepting fake JSON-text predicates in RFC 022. ## Progress Checklist @@ -149,4 +149,4 @@ This RFC is additive. It should not change existing CSV ingestion behavior. - [x] `sha2(...)` implemented as a literal-bit-length rewrite with invalid-input diagnostics. - [x] Focused helper, registry, Substrait lowering, and DataFusion-backed session tests added. - [x] User-facing format-function docs and release notes added. -- [x] Semi-structured variant predicates delegated to InQL RFC 026 instead of being accepted as JSON-text parser helpers. +- [x] Semi-structured variant predicates delegated to IncQL RFC 026 instead of being accepted as JSON-text parser helpers. diff --git a/docs/rfcs/023_approximate_sketch_functions.md b/docs/rfcs/023_approximate_sketch_functions.md index b42104b3..5f94b5ef 100644 --- a/docs/rfcs/023_approximate_sketch_functions.md +++ b/docs/rfcs/023_approximate_sketch_functions.md @@ -1,42 +1,42 @@ -# InQL RFC 023: Approximate and sketch functions +# IncQL RFC 023: Approximate and sketch functions - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 016 (core aggregate functions) - - InQL RFC 017 (aggregate modifiers) - - InQL RFC 024 (function extension policy) - - InQL RFC 025 (typed sketch logical values) -- **Issue:** [InQL #40](https://github.com/encero-systems/InQL/issues/40) -- **RFC PR:** [InQL #53](https://github.com/encero-systems/InQL/pull/53) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 016 (core aggregate functions) + - IncQL RFC 017 (aggregate modifiers) + - IncQL RFC 024 (function extension policy) + - IncQL RFC 025 (typed sketch logical values) +- **Issue:** [IncQL #40](https://github.com/encero-systems/IncQL/issues/40) +- **RFC PR:** [IncQL #53](https://github.com/encero-systems/IncQL/pull/53) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines the portable approximate aggregate boundary for InQL and records the sketch-state policy decision. InQL exposes explicit approximate aggregates for distinct counts and percentiles. It delegates sketch-state construction, merge, estimate, serialization, and deserialization helpers to InQL RFC 025 because those helpers require typed sketch logical values rather than ordinary string or binary payloads. +This RFC defines the portable approximate aggregate boundary for IncQL and records the sketch-state policy decision. IncQL exposes explicit approximate aggregates for distinct counts and percentiles. It delegates sketch-state construction, merge, estimate, serialization, and deserialization helpers to IncQL RFC 025 because those helpers require typed sketch logical values rather than ordinary string or binary payloads. ## Motivation -Spark exposes many approximate and sketch functions because large-scale analytics often trades exactness for bounded memory or faster execution. InQL should support the portable part of that direction, but sketch functions require more than names: they need accuracy parameters, merge semantics, serialization formats, determinism rules, and typed opaque state values. +Spark exposes many approximate and sketch functions because large-scale analytics often trades exactness for bounded memory or faster execution. IncQL should support the portable part of that direction, but sketch functions require more than names: they need accuracy parameters, merge semantics, serialization formats, determinism rules, and typed opaque state values. -If sketches are added as ordinary functions returning untyped bytes, InQL will not be able to reason about compatibility, aggregation state, or cross-backend behavior. +If sketches are added as ordinary functions returning untyped bytes, IncQL will not be able to reason about compatibility, aggregation state, or cross-backend behavior. ## Goals - Define approximate aggregates as distinct from exact aggregates. - Implement portable `approx_count_distinct` and `approx_percentile` aggregate helpers. - Require accuracy/error parameters to be part of function signatures when a portable helper supports them. -- Preserve InQL-owned Substrait extension names and keep backend implementation names at the adapter boundary. -- Reserve sketch-state function names for InQL RFC 025, where sketch state is modeled as an explicit typed value family. +- Preserve IncQL-owned Substrait extension names and keep backend implementation names at the adapter boundary. +- Reserve sketch-state function names for IncQL RFC 025, where sketch state is modeled as an explicit typed value family. ## Non-Goals - Making approximate functions part of the exact core aggregate contract. -- Exposing sketch state as string or binary payloads instead of the typed sketch logical values defined by InQL RFC 025. +- Exposing sketch state as string or binary payloads instead of the typed sketch logical values defined by IncQL RFC 025. - Guaranteeing bit-for-bit compatibility with Spark or any other engine. - Defining geospatial, cryptographic, or physical execution metadata functions. @@ -45,7 +45,7 @@ If sketches are added as ordinary functions returning untyped bytes, InQL will n Authors should see approximate functions as explicit approximate choices: ```incan -from pub::inql.functions import approx_count_distinct, approx_percentile, col +from pub::incql.functions import approx_count_distinct, approx_percentile, col summary = ( events @@ -63,13 +63,13 @@ The function names and arguments should make it clear that results are approxima Approximate aggregate functions must be registered as approximate. Their registry entries must declare accuracy parameters, deterministic behavior for fixed inputs and parameters, mergeability, and result interpretation. -`approx_count_distinct(expr)` returns an approximate cardinality estimate over non-null expression values. It is a HyperLogLog-family aggregate. The portable helper intentionally has no relative-error parameter because the registered InQL Substrait extension mapping is unary; backend-specific precision controls must not be smuggled into the portable helper contract. +`approx_count_distinct(expr)` returns an approximate cardinality estimate over non-null expression values. It is a HyperLogLog-family aggregate. The portable helper intentionally has no relative-error parameter because the registered IncQL Substrait extension mapping is unary; backend-specific precision controls must not be smuggled into the portable helper contract. `approx_percentile(expr, percentile, accuracy=10000)` returns an approximate percentile estimate over numeric non-null expression values. `percentile` is a literal fraction in the inclusive range `[0.0, 1.0]`. `accuracy` is a positive integer approximation hint carried as a normal aggregate argument. The portable contract is an approximate percentile estimate, not a bit-for-bit promise of one backend's interpolation or sketch representation. -Sketch-construction functions are reserved for InQL RFC 025 and are not lowerable in this RFC. When they are introduced, they must return typed sketch values, not untyped binary blobs. Sketch values may have opaque runtime representation, but their logical type must identify the sketch family and value domain. +Sketch-construction functions are reserved for IncQL RFC 025 and are not lowerable in this RFC. When they are introduced, they must return typed sketch values, not untyped binary blobs. Sketch values may have opaque runtime representation, but their logical type must identify the sketch family and value domain. -Sketch union, intersection, estimation, serialization, and deserialization functions are likewise reserved until InQL can accept only compatible sketch types and reject incompatible sketch-family or value-domain combinations before execution. +Sketch union, intersection, estimation, serialization, and deserialization functions are likewise reserved until IncQL can accept only compatible sketch types and reject incompatible sketch-family or value-domain combinations before execution. If serialized sketch formats are exposed, format versioning and cross-version compatibility must be specified. @@ -77,15 +77,15 @@ If serialized sketch formats are exposed, format versioning and cross-version co ### Syntax -This RFC permits ordinary function-call syntax for approximate aggregate functions. Reserved sketch helpers will use the same call style if InQL RFC 025 admits them. RFC 023 does not require special query syntax. +This RFC permits ordinary function-call syntax for approximate aggregate functions. Reserved sketch helpers will use the same call style if IncQL RFC 025 admits them. RFC 023 does not require special query syntax. ### Semantics -Approximate functions must be opt-in by name or explicit option. InQL must not silently replace an exact aggregate with an approximate aggregate because a backend prefers it. +Approximate functions must be opt-in by name or explicit option. IncQL must not silently replace an exact aggregate with an approximate aggregate because a backend prefers it. Sketch merge functions must define whether they are associative, commutative, idempotent, or order-sensitive. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Approximate aggregates may appear anywhere aggregate measures are valid if their registry entry supports the position. Sketch scalar helpers are not exposed until sketch expressions have typed logical values. @@ -107,8 +107,8 @@ This RFC is additive. Existing exact aggregates must not change semantics when a ## Layers affected -- **InQL specification** — approximate and sketch functions must be separate from exact aggregate semantics. -- **InQL library package** — public helpers should expose approximate aggregate and sketch-state types only when contracts are explicit. +- **IncQL specification** — approximate and sketch functions must be separate from exact aggregate semantics. +- **IncQL library package** — public helpers should expose approximate aggregate and sketch-state types only when contracts are explicit. - **Incan compiler** — typechecking must validate sketch family compatibility and aggregate positions. - **Execution / interchange** — Prism and Substrait lowering must preserve approximate parameters, sketch state types, and merge semantics or reject unsupported functions. - **Documentation** — docs must label approximate functions clearly and explain accuracy parameters. @@ -119,29 +119,29 @@ This RFC is additive. Existing exact aggregates must not change semantics when a - `approx_count_distinct(expr)` is an aggregate measure, not a scalar expression, and its helper name makes approximate execution an explicit author choice. - `approx_count_distinct` is registered as approximate metadata with HyperLogLog-family semantics, mergeability, and an approximate cardinality-result interpretation. -- `approx_count_distinct` follows InQL's registered unary Substrait extension mapping. It does not expose a user-tunable relative-error parameter because the portable mapping does not carry one. +- `approx_count_distinct` follows IncQL's registered unary Substrait extension mapping. It does not expose a user-tunable relative-error parameter because the portable mapping does not carry one. - `approx_percentile(expr, percentile, accuracy=10000)` is an aggregate measure with t-digest-family approximation metadata. The helper validates literal percentile and accuracy arguments before building the measure. - `approx_percentile` output names include both percentile and accuracy parameters, so multiple percentile estimates over the same input expression remain distinct through Prism and Substrait inspection. -- DataFusion's implementation is named `approx_distinct`; InQL keeps the InQL Substrait function name in emitted function metadata and rewrites only the DataFusion consumer declaration at the backend adapter boundary. -- DataFusion's approximate percentile implementation is named `approx_percentile_cont`; InQL uses the same adapter-only declaration rewrite and keeps `approx_percentile` as the portable Substrait extension name. +- DataFusion's implementation is named `approx_distinct`; IncQL keeps the IncQL Substrait function name in emitted function metadata and rewrites only the DataFusion consumer declaration at the backend adapter boundary. +- DataFusion's approximate percentile implementation is named `approx_percentile_cont`; IncQL uses the same adapter-only declaration rewrite and keeps `approx_percentile` as the portable Substrait extension name. - `approx_count_distinct` allows aggregate-local filters and rejects an extra `distinct()` modifier because distinct estimation is already the helper's semantics. - `approx_percentile` allows aggregate-local filters and rejects `distinct()` and ordered input because those modifiers are not part of the portable percentile aggregate contract. -- Sketch-state construction, merge, estimate, serialization, and deserialization helpers are delegated to InQL RFC 025. They are not exposed as lowerable RFC 023 functions because exposing those helpers as ordinary strings or binary values would violate the compatibility rules this RFC is meant to protect. +- Sketch-state construction, merge, estimate, serialization, and deserialization helpers are delegated to IncQL RFC 025. They are not exposed as lowerable RFC 023 functions because exposing those helpers as ordinary strings or binary values would violate the compatibility rules this RFC is meant to protect. ### Remaining -- InQL RFC 025 defines the follow-up design space for typed sketch state, portable serialization formats, and named merge/estimate helpers. That work must not retrofit RFC 023 by treating untyped binary payloads as sketch values. +- IncQL RFC 025 defines the follow-up design space for typed sketch state, portable serialization formats, and named merge/estimate helpers. That work must not retrofit RFC 023 by treating untyped binary payloads as sketch values. - A future backend-capability layer may expose backend-specific approximation knobs as engine-specific functions or options when they cannot be represented by the portable helper signatures. ## Implementation Plan 1. Add registry approximation metadata with exact-helper defaults. 2. Add `approx_count_distinct(expr)` and `approx_percentile(expr, percentile, accuracy=10000)` under a logical approximate function family. -3. Add stable Substrait anchors and keep emitted function metadata on InQL extension names. +3. Add stable Substrait anchors and keep emitted function metadata on IncQL extension names. 4. Add DataFusion adapter-local declaration rewrites to the first backend's implementation names. 5. Add focused helper, registry, Substrait lowering, Prism, and DataFusion-backed session tests with materialized output. 6. Add user-facing approximate-function docs, aggregate-builder docs, and release notes. -7. Record sketch-state helper names as reserved for InQL RFC 025. +7. Record sketch-state helper names as reserved for IncQL RFC 025. ## Progress Checklist @@ -149,8 +149,8 @@ This RFC is additive. Existing exact aggregates must not change semantics when a - [x] Registry approximation metadata added for intentionally approximate functions. - [x] `approx_count_distinct` helper added under the function catalog. - [x] `approx_percentile` helper added under the function catalog. -- [x] Stable InQL Substrait approximate aggregate extension metadata added. +- [x] Stable IncQL Substrait approximate aggregate extension metadata added. - [x] DataFusion adapter-local approximate aggregate mappings added. - [x] Focused helper, registry, Substrait lowering, Prism, and DataFusion-backed session tests added. - [x] User-facing approximate-function docs, aggregate-builder docs, and release notes added. -- [x] Sketch-state logical types and sketch merge/estimate/serialization helpers delegated to InQL RFC 025 rather than exposed as untyped lowerable functions. +- [x] Sketch-state logical types and sketch merge/estimate/serialization helpers delegated to IncQL RFC 025 rather than exposed as untyped lowerable functions. diff --git a/docs/rfcs/024_function_extension_policy.md b/docs/rfcs/024_function_extension_policy.md index 25d9c592..5cdca52b 100644 --- a/docs/rfcs/024_function_extension_policy.md +++ b/docs/rfcs/024_function_extension_policy.md @@ -1,28 +1,28 @@ -# InQL RFC 024: Function extension policy +# IncQL RFC 024: Function extension policy - **Status:** Implemented - **Created:** 2026-04-27 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Substrait lowering and extension policy) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 022 (semi-structured and format functions) - - InQL RFC 023 (approximate and sketch functions) -- **Issue:** [InQL #41](https://github.com/encero-systems/InQL/issues/41) -- **RFC PR:** [InQL #44](https://github.com/encero-systems/InQL/pull/44) + - IncQL RFC 002 (Substrait lowering and extension policy) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 022 (semi-structured and format functions) + - IncQL RFC 023 (approximate and sketch functions) +- **Issue:** [IncQL #41](https://github.com/encero-systems/IncQL/issues/41) +- **RFC PR:** [IncQL #44](https://github.com/encero-systems/IncQL/pull/44) - **Written against:** Incan v0.2 - **Shipped in:** v0.1 ## Summary -This RFC defines how InQL handles functions that should not live in the portable core catalog: geospatial functions, cryptographic functions, engine-specific physical metadata, UDF and UDTF hooks, JVM/reflection-style escape hatches, partition transforms, and dialect-specific compatibility families. It establishes extension registration, namespacing, capability reporting, and rejection rules. +This RFC defines how IncQL handles functions that should not live in the portable core catalog: geospatial functions, cryptographic functions, engine-specific physical metadata, UDF and UDTF hooks, JVM/reflection-style escape hatches, partition transforms, and dialect-specific compatibility families. It establishes extension registration, namespacing, capability reporting, and rejection rules. ## Motivation -Spark's and Snowflake's function catalogs include useful portable functions, but they also include APIs tied to a specific runtime, physical execution model, warehouse type system, or specialist domain. dbt adds a different kind of pressure: cross-database names that need adapter-specific rendering. InQL needs a deliberate answer for those functions. Some should become registered extensions. Some should be table metadata transforms rather than scalar functions. Some should be rejected from portable InQL core. +Spark's and Snowflake's function catalogs include useful portable functions, but they also include APIs tied to a specific runtime, physical execution model, warehouse type system, or specialist domain. dbt adds a different kind of pressure: cross-database names that need adapter-specific rendering. IncQL needs a deliberate answer for those functions. Some should become registered extensions. Some should be table metadata transforms rather than scalar functions. Some should be rejected from portable IncQL core. -Without an extension policy, compatibility pressure will push too much into the core catalog and weaken InQL's backend-neutral contract. +Without an extension policy, compatibility pressure will push too much into the core catalog and weaken IncQL's backend-neutral contract. ## Goals @@ -42,22 +42,22 @@ Without an extension policy, compatibility pressure will push too much into the ## Guide-level explanation (how authors think about it) -Portable InQL functions should work across supported execution paths when backend support exists. Extension functions are explicit: +Portable IncQL functions should work across supported execution paths when backend support exists. Extension functions are explicit: ```incan -from pub::inql.functions import col -from pub::inql_ext.geo import st_srid +from pub::incql.functions import col +from pub::incql_ext.geo import st_srid places_with_srid = places.with_column("srid", st_srid(col("geometry"))) ``` -If an author asks for a function that depends on physical Spark partitions, JVM reflection, or a specific engine runtime, InQL should say no in the portable core rather than pretending it is a normal data logic function. +If an author asks for a function that depends on physical Spark partitions, JVM reflection, or a specific engine runtime, IncQL should say no in the portable core rather than pretending it is a normal data logic function. ## Reference-level explanation (precise rules) The function registry must classify each function as core, extension, compatibility alias, engine-specific, or rejected. -Core functions must have portable InQL semantics independent of one execution engine. +Core functions must have portable IncQL semantics independent of one execution engine. Extension functions must be namespaced. They must declare ownership, function class, type rules, null behavior, determinism, backend support, and interchange behavior. Extension functions may lower through Substrait extensions when they cannot be represented in core interchange. @@ -65,7 +65,7 @@ Engine-specific functions must not be imported into the portable core namespace. Rejected functions must be documented when they are likely compatibility requests. JVM reflection functions, physical partition identifiers, input file block metadata, and backend-specific execution counters should be rejected from portable core semantics. -UDF and UDTF hooks must register function metadata before use in InQL planning. A UDF without type, null, determinism, and backend support metadata must not be treated as a fully typed portable function. +UDF and UDTF hooks must register function metadata before use in IncQL planning. A UDF without type, null, determinism, and backend support metadata must not be treated as a fully typed portable function. Partition transforms such as year, month, day, hour, and bucket transforms may be supported as table metadata or layout expressions, but they must not be confused with ordinary scalar date functions unless their semantics are identical. @@ -79,7 +79,7 @@ This RFC does not require new syntax. Extension functions may be imported throug Extension functions are allowed to be non-portable, but their non-portability must be visible. A plan containing extension functions must carry enough identity for downstream tooling to reject, route, or execute it intentionally. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces All authoring surfaces must observe the same extension policy. A function rejected from portable core must not become available simply because it is written in a query block instead of a dataframe method chain. @@ -87,12 +87,12 @@ All authoring surfaces must observe the same extension policy. A function reject Compatibility aliases for portable functions may live in core or dialect modules. Compatibility names for non-portable functions should require explicit extension imports. -dbt-style adapter-dispatched names should be modeled as portability aliases or adapter-rendered functions only when their semantics are stable enough to be typed by InQL. They must not introduce templated SQL generation as a separate authoring model. +dbt-style adapter-dispatched names should be modeled as portability aliases or adapter-rendered functions only when their semantics are stable enough to be typed by IncQL. They must not introduce templated SQL generation as a separate authoring model. ## Alternatives considered -- **Import every Spark or Snowflake function into core.** Rejected because runtime-specific, physical, warehouse-specific, and dynamic-type functions would undermine InQL's backend-neutral model. -- **Adopt dbt-style templated SQL as the function system.** Rejected because InQL needs typed expression semantics; dbt is a useful portability reference, not the authoring model. +- **Import every Spark or Snowflake function into core.** Rejected because runtime-specific, physical, warehouse-specific, and dynamic-type functions would undermine IncQL's backend-neutral model. +- **Adopt dbt-style templated SQL as the function system.** Rejected because IncQL needs typed expression semantics; dbt is a useful portability reference, not the authoring model. - **Ban all extensions.** Rejected because geospatial, crypto, custom UDF, and backend-specific work are legitimate needs when made explicit. - **Allow untyped UDFs anywhere.** Rejected because it breaks typed planning and diagnostics. @@ -104,8 +104,8 @@ dbt-style adapter-dispatched names should be modeled as portability aliases or a ## Layers affected -- **InQL specification** — function categories and rejection policy must be clear enough to prevent core catalog sprawl. -- **InQL library package** — extension modules and compatibility aliases should register metadata through the same registry model. +- **IncQL specification** — function categories and rejection policy must be clear enough to prevent core catalog sprawl. +- **IncQL library package** — extension modules and compatibility aliases should register metadata through the same registry model. - **Incan compiler** — function resolution must preserve namespace and extension identity for diagnostics. - **Execution / interchange** — Prism and Substrait lowering must carry extension identity or reject unsupported extension functions. - **Documentation** — docs should list rejected compatibility requests and point to explicit extension alternatives where they exist. @@ -120,7 +120,7 @@ dbt-style adapter-dispatched names should be modeled as portability aliases or a ## Design Decisions -- **Namespace convention:** portable core functions use `inql.functions`. Extension packages should use explicit namespaces such as `inql_ext.` or another package-owned namespace that cannot be confused with core. +- **Namespace convention:** portable core functions use `incql.functions`. Extension packages should use explicit namespaces such as `incql_ext.` or another package-owned namespace that cannot be confused with core. - **Dialect compatibility:** dialect compatibility should be modeled as ordinary opt-in package/module surface unless a later RFC proves a built-in dialect module is necessary. Compatibility aliases for portable functions remain metadata-visible and opt-in. -- **UDF metadata floor:** a UDF or UDTF must provide the same minimum registry facts as any extension function before it participates in typed InQL planning: namespace, function class, lifecycle, determinism, null behavior, error behavior, and interchange/backend support metadata. +- **UDF metadata floor:** a UDF or UDTF must provide the same minimum registry facts as any extension function before it participates in typed IncQL planning: namespace, function class, lifecycle, determinism, null behavior, error behavior, and interchange/backend support metadata. - **Rejected requests:** likely compatibility requests that cannot be represented as portable data logic should be represented as rejection metadata, not fake lowerable functions. diff --git a/docs/rfcs/025_typed_sketch_logical_values.md b/docs/rfcs/025_typed_sketch_logical_values.md index 6ac3930e..b6a68df5 100644 --- a/docs/rfcs/025_typed_sketch_logical_values.md +++ b/docs/rfcs/025_typed_sketch_logical_values.md @@ -1,26 +1,26 @@ -# InQL RFC 025: Typed sketch logical values +# IncQL RFC 025: Typed sketch logical values - **Status:** Implemented - **Created:** 2026-05-28 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 013 (function catalog program) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 023 (approximate and sketch functions) - - InQL RFC 024 (function extension policy) -- **Issue:** [InQL #51](https://github.com/encero-systems/InQL/issues/51) -- **RFC PR:** [InQL #55](https://github.com/encero-systems/InQL/pull/55) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 023 (approximate and sketch functions) + - IncQL RFC 024 (function extension policy) +- **Issue:** [IncQL #51](https://github.com/encero-systems/IncQL/issues/51) +- **RFC PR:** [IncQL #55](https://github.com/encero-systems/IncQL/pull/55) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines typed sketch logical values for InQL. Sketch helpers must not be modeled as ordinary strings or binary blobs; they must produce and consume logical sketch values that record sketch family, input value domain, parameterization, merge compatibility, and serialization format identity. +This RFC defines typed sketch logical values for IncQL. Sketch helpers must not be modeled as ordinary strings or binary blobs; they must produce and consume logical sketch values that record sketch family, input value domain, parameterization, merge compatibility, and serialization format identity. ## Core model -1. A sketch value has a logical type, even if its runtime representation is opaque to InQL. +1. A sketch value has a logical type, even if its runtime representation is opaque to IncQL. 2. Sketch construction is aggregate-shaped when it summarizes many input rows into one sketch state. 3. Sketch merge is valid only for compatible sketch values. 4. Sketch estimate, quantile, serialization, and deserialization helpers must preserve sketch-family semantics instead of treating sketch payloads as generic bytes. @@ -30,14 +30,14 @@ This RFC defines typed sketch logical values for InQL. Sketch helpers must not b Approximate aggregates such as `approx_count_distinct(...)` and `approx_percentile(...)` are useful when authors only need a scalar result. Sketch state is different: authors may want to materialize a sketch, merge sketches across partitions or files, estimate from a stored sketch later, or serialize sketch state for transport. Those operations require compatibility rules that cannot be represented by `bytes` or `str` alone. -If InQL accepts untyped sketch blobs, it cannot reject invalid operations such as merging a HyperLogLog sketch with a KLL sketch, merging sketches over different value domains, or deserializing a payload with an incompatible format version. That would push semantic validation into backend-specific runtime failures and weaken the Substrait boundary. +If IncQL accepts untyped sketch blobs, it cannot reject invalid operations such as merging a HyperLogLog sketch with a KLL sketch, merging sketches over different value domains, or deserializing a payload with an incompatible format version. That would push semantic validation into backend-specific runtime failures and weaken the Substrait boundary. ## Goals - Define sketch logical values as first-class typed values. - Define the metadata required to compare sketch compatibility before execution. - Define how sketch construction, merge, estimate, serialization, and deserialization helpers interact with the function registry. -- Keep sketch state backend-neutral in InQL semantics while allowing backend-specific execution support. +- Keep sketch state backend-neutral in IncQL semantics while allowing backend-specific execution support. - Provide a design home for HyperLogLog, KLL, theta, count-min, and bitmap-style sketch families without forcing all families into the first implementation. ## Non-Goals @@ -45,7 +45,7 @@ If InQL accepts untyped sketch blobs, it cannot reject invalid operations such a - Defining exact binary serialization formats for every sketch family. - Guaranteeing bit-for-bit sketch compatibility with Spark, DataSketches, DataFusion, or any other runtime. - Making sketch values ordinary scalar strings or binary columns. -- Replacing the scalar approximate aggregates defined by InQL RFC 023. +- Replacing the scalar approximate aggregates defined by IncQL RFC 023. - Defining geospatial, cryptographic, or physical execution metadata functions. ## Guide-level explanation (how authors think about it) @@ -53,8 +53,8 @@ If InQL accepts untyped sketch blobs, it cannot reject invalid operations such a Authors should think of a sketch as a typed summary value. It can be produced by an aggregate, stored as a column when the carrier supports it, merged with compatible sketches, and estimated later. RFC 025 ships the first concrete family: HyperLogLog. ```incan -from pub::inql.functions import col, hll_sketch -from pub::inql.sketches import hll_estimate, hll_merge, hll_type, sketch_col +from pub::incql.functions import col, hll_sketch +from pub::incql.sketches import hll_estimate, hll_merge, hll_type, sketch_col daily = events.group_by([col("event_date")]).agg([ hll_sketch(col("user_id"), precision=14), @@ -80,17 +80,17 @@ A sketch logical value must carry at least: - input value domain, such as string identifiers, integer identifiers, numeric values, or categorical values; - family parameters that affect merge compatibility, such as precision, accuracy, nominal entries, width/depth, seed, or ordering policy; - format identity and version when the value can be serialized; -- nullability and ordinary column-position metadata needed by existing InQL expression and relation surfaces. +- nullability and ordinary column-position metadata needed by existing IncQL expression and relation surfaces. Sketch construction helpers that summarize rows must be aggregate measures. They must declare approximate-result metadata and sketch-output metadata in the function registry. They must not appear where row-level scalar expressions are required unless the helper is explicitly a scalar transformation over an existing sketch value. -Sketch merge helpers must validate sketch family compatibility before lowering. InQL must reject merges between different families, incompatible value domains, incompatible parameter sets, or incompatible format versions unless the specific family declares a safe coercion or union rule. +Sketch merge helpers must validate sketch family compatibility before lowering. IncQL must reject merges between different families, incompatible value domains, incompatible parameter sets, or incompatible format versions unless the specific family declares a safe coercion or union rule. Sketch estimate helpers must declare the scalar result they produce. HyperLogLog-style estimates should return approximate cardinality results. KLL-style quantile helpers must require explicit percentile or rank arguments. Count-min-style lookup helpers must require an item expression whose domain is compatible with the sketch domain. -Sketch serialization helpers must be explicit. InQL must not implicitly coerce a sketch value to `str` or `bytes`. Deserialization must require enough type metadata to identify family, domain, parameters, and format version before the value can participate in merge or estimate operations. +Sketch serialization helpers must be explicit. IncQL must not implicitly coerce a sketch value to `str` or `bytes`. Deserialization must require enough type metadata to identify family, domain, parameters, and format version before the value can participate in merge or estimate operations. -Substrait lowering must preserve sketch logical type identity through extension type metadata or must reject the operation before execution. A backend adapter may map the sketch operation to a native implementation only when it can preserve the InQL sketch contract. +Substrait lowering must preserve sketch logical type identity through extension type metadata or must reject the operation before execution. A backend adapter may map the sketch operation to a native implementation only when it can preserve the IncQL sketch contract. ## Design details @@ -104,13 +104,13 @@ Sketch values are opaque to ordinary scalar operators. Equality, ordering, strin Sketch families must define whether merge is associative, commutative, idempotent, or order-sensitive. Families must define which parameters are part of merge compatibility and which parameters are execution hints. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Dataframe method chains and future query-block syntax must resolve to the same sketch logical value model. A sketch helper that is rejected in one authoring surface must not become valid in another. Prism may preserve sketch type metadata and may use it for validation, projection pruning, and rewrite safety. Prism must not rewrite sketch operations in ways that drop family, domain, parameter, or serialization metadata. -The Substrait boundary must remain between InQL semantics and backend execution. DataFusion or any other backend may be the first implementation target, but backend-native sketch names and payload formats do not define the portable InQL type. +The Substrait boundary must remain between IncQL semantics and backend execution. DataFusion or any other backend may be the first implementation target, but backend-native sketch names and payload formats do not define the portable IncQL type. ### Implementation @@ -127,7 +127,7 @@ The implemented first family is HyperLogLog: - The public `SketchFamily` API exposes HyperLogLog in this implementation; additional families should add their own family-specific type builders, serialization formats, registry policies, and tests rather than sharing HLL metadata. - Function registry entries expose typed sketch policy metadata and Substrait extension mappings. - Substrait lowering carries sketch family, value domain, precision, and format in function options. -- The DataFusion adapter rejects typed sketch execution with a backend planning diagnostic. This is an adapter capability boundary, not an InQL semantic limitation. +- The DataFusion adapter rejects typed sketch execution with a backend planning diagnostic. This is an adapter capability boundary, not an IncQL semantic limitation. ### Compatibility / migration @@ -137,7 +137,7 @@ This RFC is additive. RFC 023 approximate scalar-result aggregates remain valid. - **Treat sketches as bytes.** Rejected because it prevents typechecking merge compatibility and moves semantic errors into backend runtime failures. - **Expose only scalar approximate aggregates.** Rejected as a complete long-term answer because stored and mergeable sketches are a legitimate analytics need, especially for pre-aggregated data. -- **Copy one backend's sketch catalog directly.** Rejected because InQL needs backend-neutral semantics and capability reporting. +- **Copy one backend's sketch catalog directly.** Rejected because IncQL needs backend-neutral semantics and capability reporting. - **Make sketch values ordinary structs.** Rejected unless the struct carries a distinct logical type; ordinary structs do not by themselves encode family-specific compatibility rules. ## Drawbacks @@ -149,8 +149,8 @@ This RFC is additive. RFC 023 approximate scalar-result aggregates remain valid. ## Layers affected -- **InQL specification** — sketch values must be distinguished from ordinary scalar, binary, string, map, and struct values. -- **InQL library package** — public sketch helpers must register family, domain, parameter, merge, estimate, and serialization metadata. +- **IncQL specification** — sketch values must be distinguished from ordinary scalar, binary, string, map, and struct values. +- **IncQL library package** — public sketch helpers must register family, domain, parameter, merge, estimate, and serialization metadata. - **Incan compiler** — typechecking and diagnostics may need enough type information to represent sketch-valued expressions, reject invalid operations, and preserve metadata through public helper signatures. - **Execution / interchange** — Substrait lowering and backend adapters must preserve sketch logical type identity or reject unsupported sketch operations before execution. - **Documentation** — function references and RFCs must present sketch helpers as typed approximate state, not as backend-specific blobs. @@ -163,5 +163,5 @@ This RFC is additive. RFC 023 approximate scalar-result aggregates remain valid. - Public sketch helpers use the same typed value-or-column input conventions as the post-RFC018 scalar catalog: source values are accepted as primitive values or scalar expressions, while serialized sketch payloads use the string value-or-column surface. - HyperLogLog is the first implemented sketch family because it cleanly extends the distinct-count approximation surface. - HyperLogLog merge compatibility is defined by family, value domain, precision, and serialization format. -- Serialized sketch format identity is explicit and portable at the InQL logical layer. RFC 025 defines `inql_hll_v1` as the first format identity without promising bit-for-bit compatibility with every backend runtime. +- Serialized sketch format identity is explicit and portable at the IncQL logical layer. RFC 025 defines `incql_hll_v1` as the first format identity without promising bit-for-bit compatibility with every backend runtime. - Sketch values may be represented in authoring expressions today through `SketchExpr`. Broader table-schema logical typing is left to RFC 026 and later schema work rather than hiding sketch state as strings or bytes. diff --git a/docs/rfcs/026_semi_structured_variant_values.md b/docs/rfcs/026_semi_structured_variant_values.md index b0952149..f52505d8 100644 --- a/docs/rfcs/026_semi_structured_variant_values.md +++ b/docs/rfcs/026_semi_structured_variant_values.md @@ -1,22 +1,22 @@ -# InQL RFC 026: Semi-structured variant logical values +# IncQL RFC 026: Semi-structured variant logical values - **Status:** Implemented - **Created:** 2026-05-28 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 020 (nested data functions) - - InQL RFC 022 (semi-structured and format functions) - - InQL RFC 024 (function extension policy) -- **Issue:** [InQL #52](https://github.com/encero-systems/InQL/issues/52) -- **RFC PR:** [InQL #56](https://github.com/encero-systems/InQL/pull/56) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 020 (nested data functions) + - IncQL RFC 022 (semi-structured and format functions) + - IncQL RFC 024 (function extension policy) +- **Issue:** [IncQL #52](https://github.com/encero-systems/IncQL/issues/52) +- **RFC PR:** [IncQL #56](https://github.com/encero-systems/IncQL/pull/56) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines semi-structured variant logical values for InQL. A variant value is distinct from ordinary `str` and `bytes` payloads: it carries a logical kind such as null, boolean, integer, floating point, string, timestamp, array, or object, and InQL predicates inspect that logical value rather than reparsing arbitrary JSON text. +This RFC defines semi-structured variant logical values for IncQL. A variant value is distinct from ordinary `str` and `bytes` payloads: it carries a logical kind such as null, boolean, integer, floating point, string, timestamp, array, or object, and IncQL predicates inspect that logical value rather than reparsing arbitrary JSON text. ## Core model @@ -24,13 +24,13 @@ This RFC defines semi-structured variant logical values for InQL. A variant valu 2. Variant null and relation-level SQL null are distinct. Variant predicates must not erase that distinction. 3. Type predicates such as `typeof`, `is_array`, `is_object`, `is_integer`, `is_timestamp`, and `is_null_value` operate on variant expressions, not raw strings. 4. RFC 022 string-backed JSON and CSV payload helpers remain stable; this RFC may add variant-returning parse helpers, but it must not silently change existing helper return types. -5. Backend adapters may implement, emulate, or reject variant operations, but backend-native variant semantics do not define the portable InQL contract. +5. Backend adapters may implement, emulate, or reject variant operations, but backend-native variant semantics do not define the portable IncQL contract. ## Motivation -JSON and semi-structured payloads are common in data pipelines, but predicates such as `is_array(...)` and `typeof(...)` are ambiguous if InQL only has string payloads. `is_array(col("payload"))` could mean "parse this string as JSON and inspect the root value", or it could mean "inspect a typed semi-structured value that was already parsed by the logical plan." Those are different contracts with different null behavior, error behavior, and backend portability. +JSON and semi-structured payloads are common in data pipelines, but predicates such as `is_array(...)` and `typeof(...)` are ambiguous if IncQL only has string payloads. `is_array(col("payload"))` could mean "parse this string as JSON and inspect the root value", or it could mean "inspect a typed semi-structured value that was already parsed by the logical plan." Those are different contracts with different null behavior, error behavior, and backend portability. -If InQL accepts variant predicate names before defining variant values, it either squats on better names with string-parser semantics or leaves backend adapters to decide meaning at execution time. This RFC records the missing logical value model so those predicates have a precise home. +If IncQL accepts variant predicate names before defining variant values, it either squats on better names with string-parser semantics or leaves backend adapters to decide meaning at execution time. This RFC records the missing logical value model so those predicates have a precise home. ## Goals @@ -53,8 +53,8 @@ If InQL accepts variant predicate names before defining variant values, it eithe Authors should parse payload text into a variant value before using variant predicates: ```incan -from pub::inql.functions import col, is_array, is_null_value, parse_variant_json, typeof, variant_get -from pub::inql.variants import variant_col +from pub::incql.functions import col, is_array, is_null_value, parse_variant_json, typeof, variant_get +from pub::incql.variants import variant_col events_with_payload = events.with_column("payload_value", parse_variant_json(col("payload"))) @@ -70,7 +70,7 @@ Authors who only need text validation or normalized JSON strings should keep usi ## Reference-level explanation (precise rules) -InQL defines `VariantLogicalType` and `VariantExpr`. A variant logical type records a `VariantKind` and `VariantEncoding`. The implemented portable kind set is `any`, `null`, `boolean`, `integer`, `float`, `string`, `timestamp`, `array`, and `object`. The first implemented encoding is JSON. +IncQL defines `VariantLogicalType` and `VariantExpr`. A variant logical type records a `VariantKind` and `VariantEncoding`. The implemented portable kind set is `any`, `null`, `boolean`, `integer`, `float`, `string`, `timestamp`, `array`, and `object`. The first implemented encoding is JSON. Variant predicates must accept variant expressions. They must not accept ordinary `str` expressions as an implicit parse-and-inspect shortcut. Authors must use an explicit variant parse or cast helper when starting from JSON text. @@ -78,13 +78,13 @@ Variant predicates must accept variant expressions. They must not accept ordinar `is_null_value(expr)`, `is_boolean(expr)`, `is_integer(expr)`, `is_float(expr)`, `is_string(expr)`, `is_timestamp(expr)`, `is_array(expr)`, and `is_object(expr)` must inspect the variant kind. `is_integer(...)` must be true only for integer variant values, not floating point values whose runtime value happens to have no fractional component. `is_null_value(...)` must be true only for semi-structured null values. -SQL null must remain distinct from variant null. If a predicate input is SQL null rather than a present variant value, the predicate result must follow InQL's scalar null behavior for missing inputs rather than returning true for `is_null_value(...)`. +SQL null must remain distinct from variant null. If a predicate input is SQL null rather than a present variant value, the predicate result must follow IncQL's scalar null behavior for missing inputs rather than returning true for `is_null_value(...)`. `parse_variant_json(payload)` is the strict JSON-to-variant helper. `try_parse_variant_json(payload)` is the recoverable form. Their payload parameter accepts `StrValueOrColumn`, so authors may pass a string literal, a string column reference, or a string-producing expression without wrapping primitive values in `lit(...)`. Strict parse helpers must fail malformed payloads according to registry error metadata. Recoverable parse helpers must return SQL null or another explicitly documented recoverable result for malformed payloads. A JSON `null` payload must produce a present variant null, not SQL null. `variant_get(expr, path)` accesses a variant path. Literal path strings are validated as beginning with `$`. String columns or string-producing expressions are accepted as dynamic paths and are validated at execution time by the implementation that evaluates the expression. The current path spelling matches the RFC 022 JSON path helper spelling so authors do not learn two root-marker conventions, but it remains a variant operation rather than a JSON-text extraction shortcut. Variant field/path access must preserve whether a missing path produced SQL null, variant null, or a present value. If a backend cannot preserve that distinction, the adapter must reject the operation or require an explicit compatibility mode. -Substrait lowering preserves variant logical type identity in registry metadata and scalar function options. A backend adapter may map variant values and predicates to native functions only when it preserves the InQL variant contract. The DataFusion adapter currently reports a backend planning diagnostic for typed variant execution because it has no variant runtime implementation. +Substrait lowering preserves variant logical type identity in registry metadata and scalar function options. A backend adapter may map variant values and predicates to native functions only when it preserves the IncQL variant contract. The DataFusion adapter currently reports a backend planning diagnostic for typed variant execution because it has no variant runtime implementation. ## Design details @@ -94,11 +94,11 @@ This RFC does not require new language syntax. Variant values and predicates may ### Semantics -Variant arrays and objects are semi-structured values, not InQL relation shapes. They may be projected, inspected, and passed to variant-aware helpers. They must not change relation cardinality unless used with a generator or table-valued function that explicitly defines such a change. +Variant arrays and objects are semi-structured values, not IncQL relation shapes. They may be projected, inspected, and passed to variant-aware helpers. They must not change relation cardinality unless used with a generator or table-valued function that explicitly defines such a change. -Variant ordering, equality, grouping, and serialization are not implicit. If InQL supports those operations for variants, the operation must define kind ordering, null behavior, and backend compatibility rather than inheriting an arbitrary backend default. +Variant ordering, equality, grouping, and serialization are not implicit. If IncQL supports those operations for variants, the operation must define kind ordering, null behavior, and backend compatibility rather than inheriting an arbitrary backend default. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces RFC 020 nested data functions operate on typed array, map, and struct expressions. Variant arrays and objects may interoperate with those functions only through explicit conversion rules. @@ -106,7 +106,7 @@ RFC 022 format helpers that return normalized JSON or CSV text remain string-bac Prism may use variant kind metadata for validation, projection pruning, and rewrite safety. Prism must not rewrite variant operations in ways that collapse SQL null and variant null, drop schema-directed typed scalar information, or turn variant predicates into text parser calls. -The Substrait boundary remains between InQL semantics and backend execution. DataFusion or any other backend may be an implementation target, but backend-native variant names, path syntaxes, and null rules do not define the portable InQL semantics. +The Substrait boundary remains between IncQL semantics and backend execution. DataFusion or any other backend may be an implementation target, but backend-native variant names, path syntaxes, and null rules do not define the portable IncQL semantics. ### Compatibility / migration @@ -121,7 +121,7 @@ The implemented public model is: - Parse/access helpers: `parse_variant_json(...)`, `try_parse_variant_json(...)`, and `variant_get(...)`. - Inspection helpers: `typeof(...)` returns `StringColumnExpr`; predicates such as `is_null_value(...)`, `is_boolean(...)`, `is_integer(...)`, `is_float(...)`, `is_string(...)`, `is_timestamp(...)`, `is_array(...)`, and `is_object(...)` return `BoolColumnExpr`. -Each public helper is registry-backed with explicit variant policy metadata. Variant helpers lower through InQL-owned Substrait extension mappings and carry variant kind, encoding, and parse mode as scalar function options where needed. +Each public helper is registry-backed with explicit variant policy metadata. Variant helpers lower through IncQL-owned Substrait extension mappings and carry variant kind, encoding, and parse mode as scalar function options where needed. ## Alternatives considered @@ -139,8 +139,8 @@ Each public helper is registry-backed with explicit variant policy metadata. Var ## Layers affected -- **InQL specification** — variant values must be distinct from strings, bytes, typed nested values, and sketch values. -- **InQL library package** — public helpers must expose variant parse, path access, and predicate functions only with explicit registry metadata. +- **IncQL specification** — variant values must be distinct from strings, bytes, typed nested values, and sketch values. +- **IncQL library package** — public helpers must expose variant parse, path access, and predicate functions only with explicit registry metadata. - **Incan compiler** — typechecking may need enough helper metadata to reject string expressions where variant expressions are required. - **Execution / interchange** — Prism, Substrait lowering, and backend adapters must preserve variant type identity and SQL-null versus variant-null behavior or reject unsupported operations. - **Documentation** — function references must present variant predicates as variant operations, not as JSON-text parser shortcuts. diff --git a/docs/rfcs/027_relational_evidence_program.md b/docs/rfcs/027_relational_evidence_program.md index 045e8395..76dc0a80 100644 --- a/docs/rfcs/027_relational_evidence_program.md +++ b/docs/rfcs/027_relational_evidence_program.md @@ -1,68 +1,68 @@ -# InQL RFC 027: Relational evidence program +# IncQL RFC 027: Relational evidence program - **Status:** In Progress - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 000 (core language model and layer boundaries) - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 004 (execution context) - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 013 (function catalog program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 034 (quality assertions and observations) - - InQL RFC 035 (governed attributes and policy checkpoints) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 037 (plan diff and blast-radius inputs) - - InQL RFC 038 (evidence exchange bridges) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 041 (Prism plan ingress and external client frontends) - - InQL RFC 042 (async verification evidence) - - InQL RFC 043 (canonical equality and digest profiles) - - InQL RFC 044 (verifier statements and proof artifacts) - - InQL RFC 045 (constraint evidence and verification-aware planning) - - InQL RFC 046 (data contract ingress and product topology) - - InQL RFC 047 (semantic evidence graph and agent query surface) -- **Issue:** [InQL #61](https://github.com/encero-systems/InQL/issues/61) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60); [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 000 (core language model and layer boundaries) + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 004 (execution context) + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 034 (quality assertions and observations) + - IncQL RFC 035 (governed attributes and policy checkpoints) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 037 (plan diff and blast-radius inputs) + - IncQL RFC 038 (evidence exchange bridges) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 041 (Prism plan ingress and external client frontends) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 043 (canonical equality and digest profiles) + - IncQL RFC 044 (verifier statements and proof artifacts) + - IncQL RFC 045 (constraint evidence and verification-aware planning) + - IncQL RFC 046 (data contract ingress and product topology) + - IncQL RFC 047 (semantic evidence graph and agent query surface) +- **Issue:** [IncQL #61](https://github.com/encero-systems/IncQL/issues/61) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60); [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC is the umbrella tracking RFC for InQL's relational evidence program. The program defines the local, open semantic evidence contracts that make typed relational computation inspectable before execution and reviewable after execution: stable semantic targets, metadata attachments, Prism lineage, inspection artifacts, execution observations, adapter coverage, quality observations, governed attributes, plan bundles, plan diffs, evidence exchange bridges, interoperability semantic profiles, Prism plan ingress, async verification evidence, canonical equality profiles, verifier statements, proof artifacts, constraint evidence, verification-aware planning, data contract ingress, product topology, semantic evidence graph projections, and agent query surfaces. This RFC is complete only when the child RFCs are implemented, rejected, or explicitly superseded by design decision. +This RFC is the umbrella tracking RFC for IncQL's relational evidence program. The program defines the local, open semantic evidence contracts that make typed relational computation inspectable before execution and reviewable after execution: stable semantic targets, metadata attachments, Prism lineage, inspection artifacts, execution observations, adapter coverage, quality observations, governed attributes, plan bundles, plan diffs, evidence exchange bridges, interoperability semantic profiles, Prism plan ingress, async verification evidence, canonical equality profiles, verifier statements, proof artifacts, constraint evidence, verification-aware planning, data contract ingress, product topology, semantic evidence graph projections, and agent query surfaces. This RFC is complete only when the child RFCs are implemented, rejected, or explicitly superseded by design decision. ## Core model -1. InQL owns typed relational evidence, not enterprise governance operations. +1. IncQL owns typed relational evidence, not enterprise governance operations. 2. Prism is the semantic checkpoint for authored and rewritten relational meaning. 3. Substrait remains a portable interchange boundary, not the only semantic evidence store. 4. Session and adapter execution may attach observations to semantic targets, but they must not redefine authored relational meaning. 5. Local evidence must be useful without any hosted control plane, catalog service, approval workflow, or proprietary governance product. -6. Downstream systems may consume InQL evidence, but those systems are outside the InQL contract. -7. Interoperability profiles provide evidence context for target environments, not alternate semantic owners for InQL. +6. Downstream systems may consume IncQL evidence, but those systems are outside the IncQL contract. +7. Interoperability profiles provide evidence context for target environments, not alternate semantic owners for IncQL. ## Motivation -InQL already has the pieces of a stronger relational evidence layer: typed carriers, Prism planning, Substrait lowering, registry-backed expressions, aggregate/window/generator semantics, and a session boundary. What is missing is a coherent contract for the evidence that tools need to answer questions such as which source fields produced an output field, which plan rewrite changed a relation, which backend capability was required, which quality assertion failed, and which execution attempt produced a result. +IncQL already has the pieces of a stronger relational evidence layer: typed carriers, Prism planning, Substrait lowering, registry-backed expressions, aggregate/window/generator semantics, and a session boundary. What is missing is a coherent contract for the evidence that tools need to answer questions such as which source fields produced an output field, which plan rewrite changed a relation, which backend capability was required, which quality assertion failed, and which execution attempt produced a result. -Without this program, lineage, governance, quality, observability, and change-impact work will grow as disconnected features. Some tools will reconstruct meaning from Substrait, some from backend plans, some from session logs, and some from user-facing helper names. That would repeat the same failure InQL exists to avoid: typed relational meaning would be present during authoring, then weakened or reinterpreted at the next boundary. +Without this program, lineage, governance, quality, observability, and change-impact work will grow as disconnected features. Some tools will reconstruct meaning from Substrait, some from backend plans, some from session logs, and some from user-facing helper names. That would repeat the same failure IncQL exists to avoid: typed relational meaning would be present during authoring, then weakened or reinterpreted at the next boundary. ## Goals -- Establish relational evidence as one coordinated InQL program. +- Establish relational evidence as one coordinated IncQL program. - Define the child RFC set required for semantic identity, lineage, inspection, observations, coverage, quality, governed attributes, plan bundles, plan diffs, evidence exchange, interoperability profiles, verification evidence, canonical equality, verifier statements, proof artifacts, constraint evidence, verification-aware planning, data contract ingress, product topology, semantic evidence graph projections, and agent query surfaces. - Keep the program open, local, and backend-neutral. - Make Prism-authored relational meaning the source of local lineage and schema-flow evidence. - Define target-environment profile evidence without making any external engine, dialect, or interchange format the semantic owner. - Ensure execution observations and adapter coverage attach to semantic targets without redefining semantics. -- Allow higher-level governance, catalog, orchestration, audit, and approval systems to consume InQL evidence without becoming part of the InQL contract. +- Allow higher-level governance, catalog, orchestration, audit, and approval systems to consume IncQL evidence without becoming part of the IncQL contract. ## Non-Goals @@ -75,10 +75,10 @@ Without this program, lineage, governance, quality, observability, and change-im ## Guide-level explanation (how authors think about it) -Authors and tools should be able to inspect an InQL plan as structured evidence rather than formatted prose: +Authors and tools should be able to inspect an IncQL plan as structured evidence rather than formatted prose: ```incan -from pub::inql.inspect import inspect_lineage +from pub::incql.inspect import inspect_lineage summary = ( orders @@ -91,9 +91,9 @@ lineage = inspect_lineage(summary) total = lineage.field("total_amount") ``` -The exact API is defined in the child RFCs. The important user model is stable: InQL can explain typed relational computation locally, before a backend runs it and without requiring an external governance service. +The exact API is defined in the child RFCs. The important user model is stable: IncQL can explain typed relational computation locally, before a backend runs it and without requiring an external governance service. -The same evidence model should also support migration and modernization workbenches. A tool can ingest source-system metadata, target-environment profiles, transformation project artifacts, catalog metadata, and orchestration metadata; attach them to InQL semantic targets; assess compatibility gaps; and export reviewable suggestions back into the transformation stack. Representative ecosystems include legacy and operational SQL systems such as Oracle, PostgreSQL, SQL Server, and MySQL; cloud and lakehouse targets such as Athena, Presto, Trino, Spark, Snowflake, BigQuery, Redshift, and Databricks; catalogs such as Glue Data Catalog and Hive Metastore; transformation projects such as dbt; and orchestrators such as Airflow, MWAA, Dagster, and Prefect: +The same evidence model should also support migration and modernization workbenches. A tool can ingest source-system metadata, target-environment profiles, transformation project artifacts, catalog metadata, and orchestration metadata; attach them to IncQL semantic targets; assess compatibility gaps; and export reviewable suggestions back into the transformation stack. Representative ecosystems include legacy and operational SQL systems such as Oracle, PostgreSQL, SQL Server, and MySQL; cloud and lakehouse targets such as Athena, Presto, Trino, Spark, Snowflake, BigQuery, Redshift, and Databricks; catalogs such as Glue Data Catalog and Hive Metastore; transformation projects such as dbt; and orchestrators such as Airflow, MWAA, Dagster, and Prefect: ```incan brief = migration_evidence_brief( @@ -107,37 +107,37 @@ risk = inspection.profile_gaps() suggestions = inspection.export_transformation_suggestions() ``` -The names are illustrative. The important boundary is not the exact migration stack. InQL owns semantic targets, profile assessments, lineage, and evidence; external projects, catalogs, and orchestrators remain consumers or evidence sources. +The names are illustrative. The important boundary is not the exact migration stack. IncQL owns semantic targets, profile assessments, lineage, and evidence; external projects, catalogs, and orchestrators remain consumers or evidence sources. ## Reference-level explanation (precise rules) The relational evidence program must consist of the following child RFCs unless this RFC is amended or superseded: -- InQL RFC 028 (semantic identity and target model) -- InQL RFC 029 (typed metadata attachments) -- InQL RFC 030 (Prism lineage graph) -- InQL RFC 031 (local inspection APIs and artifacts) -- InQL RFC 032 (execution observations) -- InQL RFC 033 (adapter requirements and coverage) -- InQL RFC 034 (quality assertions and observations) -- InQL RFC 035 (governed attributes and policy checkpoints) -- InQL RFC 036 (governed plan bundle) -- InQL RFC 037 (plan diff and blast-radius inputs) -- InQL RFC 038 (evidence exchange bridges) -- InQL RFC 040 (interoperability semantic profiles) -- InQL RFC 041 (Prism plan ingress and external client frontends) -- InQL RFC 042 (async verification evidence) -- InQL RFC 043 (canonical equality and digest profiles) -- InQL RFC 044 (verifier statements and proof artifacts) -- InQL RFC 045 (constraint evidence and verification-aware planning) -- InQL RFC 046 (data contract ingress and product topology) -- InQL RFC 047 (semantic evidence graph and agent query surface) +- IncQL RFC 028 (semantic identity and target model) +- IncQL RFC 029 (typed metadata attachments) +- IncQL RFC 030 (Prism lineage graph) +- IncQL RFC 031 (local inspection APIs and artifacts) +- IncQL RFC 032 (execution observations) +- IncQL RFC 033 (adapter requirements and coverage) +- IncQL RFC 034 (quality assertions and observations) +- IncQL RFC 035 (governed attributes and policy checkpoints) +- IncQL RFC 036 (governed plan bundle) +- IncQL RFC 037 (plan diff and blast-radius inputs) +- IncQL RFC 038 (evidence exchange bridges) +- IncQL RFC 040 (interoperability semantic profiles) +- IncQL RFC 041 (Prism plan ingress and external client frontends) +- IncQL RFC 042 (async verification evidence) +- IncQL RFC 043 (canonical equality and digest profiles) +- IncQL RFC 044 (verifier statements and proof artifacts) +- IncQL RFC 045 (constraint evidence and verification-aware planning) +- IncQL RFC 046 (data contract ingress and product topology) +- IncQL RFC 047 (semantic evidence graph and agent query surface) This umbrella RFC must not be marked Implemented while any required child RFC remains Draft, Planned, In Progress, Blocked, or otherwise unresolved. A child RFC may be removed from the required completion set only by a design decision recorded in this RFC or by a superseding RFC. -Child RFCs must preserve the layer boundary established by this RFC. They may define local InQL evidence contracts and generic exchange shapes. They must not define proprietary product behavior, hosted storage behavior, managed approval semantics, or organization-wide policy lifecycle rules. +Child RFCs must preserve the layer boundary established by this RFC. They may define local IncQL evidence contracts and generic exchange shapes. They must not define proprietary product behavior, hosted storage behavior, managed approval semantics, or organization-wide policy lifecycle rules. -Relational evidence must derive from InQL semantic sources where possible. Prism-authored and Prism-rewritten plans are the authoritative source for local relational lineage. Session and backend adapter observations may report execution facts, diagnostics, and capability coverage, but they must not decide that an authored lineage edge exists or does not exist. +Relational evidence must derive from IncQL semantic sources where possible. Prism-authored and Prism-rewritten plans are the authoritative source for local relational lineage. Session and backend adapter observations may report execution facts, diagnostics, and capability coverage, but they must not decide that an authored lineage edge exists or does not exist. Evidence that affects correctness must not be encoded only as ignorable interchange metadata. If a downstream consumer must understand evidence for correctness, the plan must require a real supported capability, reject execution, or report unknown/uncovered coverage. @@ -151,7 +151,7 @@ This umbrella RFC introduces no new syntax. Child RFCs should prefer APIs and ar This RFC is normative for program structure, lifecycle, and layer boundaries. Individual semantic contracts are normative only in the child RFC that owns the corresponding evidence family. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Evidence must be independent of authoring surface. Equivalent method chains, `query {}` blocks, and future relational surfaces should produce equivalent semantic targets and lineage where they express equivalent relational intent. @@ -167,7 +167,7 @@ This program should be additive. Existing plans may lack evidence artifacts unti - **One giant governance RFC.** Rejected because governance, lineage, quality, execution evidence, adapter coverage, and exports are too broad to specify responsibly in one normative document. - **One RFC per artifact file.** Rejected because artifacts are downstream views of semantic contracts; the RFC boundary should be the concept, not the filename. -- **Use Substrait metadata as the evidence store.** Rejected because Substrait consumers may ignore extension metadata and because InQL needs richer local semantic targets than portable interchange can guarantee. +- **Use Substrait metadata as the evidence store.** Rejected because Substrait consumers may ignore extension metadata and because IncQL needs richer local semantic targets than portable interchange can guarantee. - **Let each downstream integration reconstruct evidence.** Rejected because it would make lineage and quality inconsistent across tools. ## Drawbacks @@ -179,11 +179,11 @@ This program should be additive. Existing plans may lack evidence artifacts unti ## Layers affected -- **InQL specification** — the RFC set must define a coherent relational evidence model across existing expression, planning, execution, and function-catalog RFCs. -- **InQL library package** — public inspection, quality, and artifact APIs must follow the child RFCs rather than growing as unrelated helpers. +- **IncQL specification** — the RFC set must define a coherent relational evidence model across existing expression, planning, execution, and function-catalog RFCs. +- **IncQL library package** — public inspection, quality, and artifact APIs must follow the child RFCs rather than growing as unrelated helpers. - **Incan compiler** — compiler-facing support is affected only where child RFCs require typed metadata, stable symbols, or package inspection. - **Execution / interchange** — Session, Substrait lowering, and adapters must attach execution evidence and capability coverage without owning relational semantics. -- **Documentation** — public docs must distinguish InQL local evidence contracts from downstream governance, catalog, and orchestration products. +- **Documentation** — public docs must distinguish IncQL local evidence contracts from downstream governance, catalog, and orchestration products. ## Unresolved questions diff --git a/docs/rfcs/028_semantic_identity_targets.md b/docs/rfcs/028_semantic_identity_targets.md index 579a03d3..24ad30cd 100644 --- a/docs/rfcs/028_semantic_identity_targets.md +++ b/docs/rfcs/028_semantic_identity_targets.md @@ -1,30 +1,30 @@ -# InQL RFC 028: Semantic identity and target model +# IncQL RFC 028: Semantic identity and target model - **Status:** In Progress - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 000 (core language model and layer boundaries) - - InQL RFC 004 (execution context) - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 027 (relational evidence program) - - InQL RFC 041 (Prism plan ingress and external client frontends) -- **Issue:** [InQL #62](https://github.com/encero-systems/InQL/issues/62) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 000 (core language model and layer boundaries) + - IncQL RFC 004 (execution context) + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 041 (Prism plan ingress and external client frontends) +- **Issue:** [IncQL #62](https://github.com/encero-systems/IncQL/issues/62) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines the semantic identity and target model required by InQL's relational evidence program. It establishes stable, typed targets for plans, Prism nodes, plan ingress requests, client sessions, relation outputs, fields, scalar expressions, aggregate measures, window expressions, generator outputs, read roots, quality assertions, policy decisions, adapter requirements, coverage records, and execution attempts. +This RFC defines the semantic identity and target model required by IncQL's relational evidence program. It establishes stable, typed targets for plans, Prism nodes, plan ingress requests, client sessions, relation outputs, fields, scalar expressions, aggregate measures, window expressions, generator outputs, read roots, quality assertions, policy decisions, adapter requirements, coverage records, and execution attempts. ## Motivation -Lineage, metadata attachments, quality observations, policy decisions, execution observations, adapter coverage, and plan diffs all need something precise to attach to. If evidence attaches only to display names or backend plan fragments, it becomes fragile under aliases, rewrites, projections, and execution adapter differences. InQL needs a semantic target model before it can produce trustworthy evidence. +Lineage, metadata attachments, quality observations, policy decisions, execution observations, adapter coverage, and plan diffs all need something precise to attach to. If evidence attaches only to display names or backend plan fragments, it becomes fragile under aliases, rewrites, projections, and execution adapter differences. IncQL needs a semantic target model before it can produce trustworthy evidence. ## Goals -- Define the required semantic target categories for local InQL evidence. +- Define the required semantic target categories for local IncQL evidence. - Define which identities are plan-local, artifact-stable, and execution-local. - Require deterministic identities within one plan, ingress, or profile snapshot where possible. - Distinguish authored Prism targets from rewritten Prism targets and execution observations. @@ -39,7 +39,7 @@ Lineage, metadata attachments, quality observations, policy decisions, execution ## Guide-level explanation (how authors think about it) -An InQL author should not usually construct semantic IDs manually. They appear when tools inspect a plan: +An IncQL author should not usually construct semantic IDs manually. They appear when tools inspect a plan: ```incan lineage = inspect_lineage(summary) @@ -53,7 +53,7 @@ The target lets tools attach evidence to the field even if the field was produce ## Reference-level explanation (precise rules) -InQL must define semantic targets for at least the following categories: +IncQL must define semantic targets for at least the following categories: - plan - ingress request @@ -83,15 +83,15 @@ A semantic target must identify its category, its containing plan, ingress reque Plan, ingress request, ingress node, ingress expression, analyzer binding, authored node, rewritten node, relation output, field, scalar expression, aggregate measure, window expression, generator output, read-root, semantic profile, and profile assessment identities must be deterministic within one plan, ingress request, or profile snapshot where possible. Client sessions, execution attempts, quality observations, and runtime coverage records may be unique per frontend, session, or execution lifecycle. -InQL must distinguish authored targets from rewritten targets. If an optimizer rewrite removes, fuses, or replaces a target, the rewritten target must preserve an authored-origin relationship rather than reusing an authored identity for a different structure. +IncQL must distinguish authored targets from rewritten targets. If an optimizer rewrite removes, fuses, or replaces a target, the rewritten target must preserve an authored-origin relationship rather than reusing an authored identity for a different structure. Field identities must not be based only on output display names. Renames, aliases, generated names, and duplicate field names must still produce unambiguous field targets. -Backend adapter identifiers may be attached as metadata or observations, but they must not replace InQL semantic targets. +Backend adapter identifiers may be attached as metadata or observations, but they must not replace IncQL semantic targets. -External client protocol identifiers may be attached as ingress-origin evidence, but they must not replace InQL semantic targets. +External client protocol identifiers may be attached as ingress-origin evidence, but they must not replace IncQL semantic targets. -A client session target identifies external client session state that can affect plan ingress analysis, such as current catalog or namespace, session configuration, temporary relation names, function registrations, profile selection, case-sensitivity mode, or dialect flags. Client session targets are not execution session identities unless an RFC or implementation explicitly binds the external client session to an InQL execution session. +A client session target identifies external client session state that can affect plan ingress analysis, such as current catalog or namespace, session configuration, temporary relation names, function registrations, profile selection, case-sensitivity mode, or dialect flags. Client session targets are not execution session identities unless an RFC or implementation explicitly binds the external client session to an IncQL execution session. ## Design details @@ -103,7 +103,7 @@ This RFC introduces no authoring syntax. Semantic targets are evidence anchors. They do not by themselves define lineage, policy, quality, or execution behavior. Those contracts belong to child RFCs that reference this target model. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Method chains and `query {}` blocks that express equivalent relational intent should produce equivalent target categories. They are not required to produce byte-identical IDs if their authored syntax differs, but the resulting semantic graph must preserve comparable targets for downstream tools. @@ -125,8 +125,8 @@ Existing plans without semantic targets remain valid for execution. Tools that r ## Layers affected -- **InQL specification** — semantic target categories must become shared terminology for relational evidence RFCs. -- **InQL library package** — inspection and artifact APIs must expose targets rather than unstructured strings. +- **IncQL specification** — semantic target categories must become shared terminology for relational evidence RFCs. +- **IncQL library package** — inspection and artifact APIs must expose targets rather than unstructured strings. - **Execution / interchange** — Session, Substrait lowering, and adapters must preserve references to semantic targets where they emit related evidence. - **Documentation** — docs must explain local plan identity versus global asset identity. diff --git a/docs/rfcs/029_metadata_attachments.md b/docs/rfcs/029_metadata_attachments.md index 50ff61d4..d7ed04f7 100644 --- a/docs/rfcs/029_metadata_attachments.md +++ b/docs/rfcs/029_metadata_attachments.md @@ -1,20 +1,20 @@ -# InQL RFC 029: Typed metadata attachments +# IncQL RFC 029: Typed metadata attachments - **Status:** In Progress - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) -- **Issue:** [InQL #63](https://github.com/encero-systems/InQL/issues/63) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) +- **Issue:** [IncQL #63](https://github.com/encero-systems/IncQL/issues/63) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines typed metadata attachments for InQL semantic targets. Attachments provide a common way to associate lifecycle, source, visibility, typed payloads, provenance, and evidence references with plans, fields, expressions, requirements, observations, and other semantic targets without hardcoding every evidence family into one model. +This RFC defines typed metadata attachments for IncQL semantic targets. Attachments provide a common way to associate lifecycle, source, visibility, typed payloads, provenance, and evidence references with plans, fields, expressions, requirements, observations, and other semantic targets without hardcoding every evidence family into one model. ## Motivation @@ -51,7 +51,7 @@ The attachment shape lets tools distinguish a user-authored label, a planner-der ## Reference-level explanation (precise rules) -An InQL metadata attachment must include: +An IncQL metadata attachment must include: - target semantic identity - namespace @@ -64,7 +64,7 @@ An InQL metadata attachment must include: Attachment lifecycle must distinguish at least authored, planned, analyzed, rewritten, lowered, bound, executed, exported, and imported states. -Attachment source must distinguish at least InQL, user, Prism, Session, adapter, function registry, quality engine, policy engine, external catalog, and imported artifact. +Attachment source must distinguish at least IncQL, user, Prism, Session, adapter, function registry, quality engine, policy engine, external catalog, and imported artifact. Attachment visibility must distinguish at least public, internal, sensitive, and redacted. Sensitive attachments must not be emitted into portable artifacts by default. Redacted attachments may preserve the existence, target, and reason code of a hidden fact without exposing the payload. @@ -82,7 +82,7 @@ This RFC introduces no syntax. Future helper APIs may expose attachments, but au Attachments are evidence records, not semantic authority by default. A child RFC may define an authoritative attachment kind only when the authority, lifecycle, and conflict behavior are explicit. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Function registry metadata may produce attachments when functions affect lineage, adapter requirements, null behavior, determinism, or extension support. Those attachments must derive from registry facts rather than duplicating function names or signatures in a separate evidence catalog. @@ -104,14 +104,14 @@ Existing APIs may continue returning simple inspection data. New evidence APIs s ## Layers affected -- **InQL specification** — metadata attachments must become the shared extension point for evidence families. -- **InQL library package** — inspection and artifact APIs must preserve typed attachment payloads and visibility. +- **IncQL specification** — metadata attachments must become the shared extension point for evidence families. +- **IncQL library package** — inspection and artifact APIs must preserve typed attachment payloads and visibility. - **Execution / interchange** — lowering and adapters may carry attachment references but must not leak sensitive payloads by default. - **Documentation** — docs must show which attachment namespaces are stable public contracts. ## Unresolved questions -- Which attachment namespaces should be reserved by InQL core? +- Which attachment namespaces should be reserved by IncQL core? - Should users be able to author arbitrary attachments directly, or only through typed helper APIs? - What is the first serialized payload schema format for attachments? diff --git a/docs/rfcs/030_prism_lineage_graph.md b/docs/rfcs/030_prism_lineage_graph.md index f1f2553f..ff3e4f2a 100644 --- a/docs/rfcs/030_prism_lineage_graph.md +++ b/docs/rfcs/030_prism_lineage_graph.md @@ -1,27 +1,27 @@ -# InQL RFC 030: Prism lineage graph +# IncQL RFC 030: Prism lineage graph - **Status:** In Progress - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 019 (window functions) - - InQL RFC 020 (nested data functions) - - InQL RFC 021 (generator and table-valued functions) - - InQL RFC 022 (semi-structured and format functions) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 041 (Prism plan ingress and external client frontends) -- **Issue:** [InQL #64](https://github.com/encero-systems/InQL/issues/64) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 019 (window functions) + - IncQL RFC 020 (nested data functions) + - IncQL RFC 021 (generator and table-valued functions) + - IncQL RFC 022 (semi-structured and format functions) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 041 (Prism plan ingress and external client frontends) +- **Issue:** [IncQL #64](https://github.com/encero-systems/IncQL/issues/64) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines the Prism lineage graph for InQL. The graph records relation-level, field-level, and expression-level dependencies over authored and rewritten Prism plans, including reads, projections, filters, joins, aggregates, windows, generators, ordering, limits, nested data access, and semi-structured or format parsing. +This RFC defines the Prism lineage graph for IncQL. The graph records relation-level, field-level, and expression-level dependencies over authored and rewritten Prism plans, including reads, projections, filters, joins, aggregates, windows, generators, ordering, limits, nested data access, and semi-structured or format parsing. ## Motivation @@ -29,7 +29,7 @@ Lineage reconstructed from backend SQL, backend plans, or Substrait alone is too ## Goals -- Define native InQL lineage edges over semantic targets. +- Define native IncQL lineage edges over semantic targets. - Distinguish value, control, grouping, join, sort, policy, and quality dependencies. - Preserve authored-origin and ingress-origin relationships through rewrites. - Represent exact, conservative, and unknown lineage confidence. @@ -40,7 +40,7 @@ Lineage reconstructed from backend SQL, backend plans, or Substrait alone is too - Defining global cross-workspace lineage storage. - Defining business meaning, certification, or ownership. - Replacing Substrait relation lowering. -- Guaranteeing that every external lineage tool can represent every native InQL edge kind. +- Guaranteeing that every external lineage tool can represent every native IncQL edge kind. ## Guide-level explanation (how authors think about it) @@ -55,7 +55,7 @@ summary = ( ) ``` -InQL should be able to explain that `total_amount` has value lineage from `orders.amount`, grouping lineage from `orders.customer_id`, and control lineage from `orders.status`. +IncQL should be able to explain that `total_amount` has value lineage from `orders.amount`, grouping lineage from `orders.customer_id`, and control lineage from `orders.status`. ## Reference-level explanation (precise rules) @@ -73,7 +73,7 @@ Relationship kind must distinguish at least value, control, grouping, join, sort Transformation kind must distinguish at least identity, expression, aggregate, window, generator, filter, join, mask, format_parse, nested_access, semi_structured_access, opaque, and unknown transformations. -Confidence must distinguish exact, conservative, and unknown lineage. Exact lineage means InQL can identify the relevant source target set. Conservative lineage means InQL can identify a safe over-approximation. Unknown lineage means InQL cannot determine the dependency and must report that uncertainty explicitly. +Confidence must distinguish exact, conservative, and unknown lineage. Exact lineage means IncQL can identify the relevant source target set. Conservative lineage means IncQL can identify a safe over-approximation. Unknown lineage means IncQL cannot determine the dependency and must report that uncertainty explicitly. Projection of an input field without transformation must produce value lineage from the input field to the output field. Projection of an expression must produce value lineage from every scalar input expression dependency to the output field. @@ -103,7 +103,7 @@ This RFC introduces no syntax. Lineage is a semantic graph over Prism targets. It is not an execution log and not a formatted explanation string. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Function registry metadata must provide lineage-relevant facts for functions whose behavior is not derivable from argument structure alone. Opaque user-defined or extension functions must produce conservative or unknown lineage unless they provide explicit metadata. @@ -113,7 +113,7 @@ Plans without lineage remain executable. Inspection APIs must distinguish unsupp ## Alternatives considered -- **Use OpenLineage column lineage as the internal model.** Rejected because InQL needs richer relationship kinds than common post-hoc run events. +- **Use OpenLineage column lineage as the internal model.** Rejected because IncQL needs richer relationship kinds than common post-hoc run events. - **Infer lineage from Substrait only.** Rejected because Prism has authored-origin and type information that may be lost during interchange lowering. - **Only expose field-level direct lineage.** Rejected because governance, quality, and blast-radius analysis need control, grouping, join, and sort relationships. @@ -125,8 +125,8 @@ Plans without lineage remain executable. Inspection APIs must distinguish unsupp ## Layers affected -- **InQL specification** — lineage relationship and transformation kinds become normative vocabulary. -- **InQL library package** — inspection APIs must expose typed lineage edges. +- **IncQL specification** — lineage relationship and transformation kinds become normative vocabulary. +- **IncQL library package** — inspection APIs must expose typed lineage edges. - **Execution / interchange** — Substrait and adapters may carry references to lineage targets but must not redefine them. - **Documentation** — docs must explain the difference between value, control, grouping, join, and sort lineage. @@ -134,6 +134,6 @@ Plans without lineage remain executable. Inspection APIs must distinguish unsupp - Should sort lineage attach to relation outputs, fields, or a separate ordering target? - Which nested and semi-structured operations can claim exact lineage in the first release? -- How should lineage represent set operations once InQL adds them? +- How should lineage represent set operations once IncQL adds them? diff --git a/docs/rfcs/031_inspection_artifacts.md b/docs/rfcs/031_inspection_artifacts.md index e47fde66..f673ba7a 100644 --- a/docs/rfcs/031_inspection_artifacts.md +++ b/docs/rfcs/031_inspection_artifacts.md @@ -1,32 +1,32 @@ -# InQL RFC 031: Local inspection APIs and artifacts +# IncQL RFC 031: Local inspection APIs and artifacts - **Status:** In Progress - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 041 (Prism plan ingress and external client frontends) -- **Issue:** [InQL #65](https://github.com/encero-systems/InQL/issues/65) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 041 (Prism plan ingress and external client frontends) +- **Issue:** [IncQL #65](https://github.com/encero-systems/IncQL/issues/65) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines local inspection APIs and deterministic evidence artifacts for InQL plans. The APIs expose plan structure, schema flow, lineage, metadata attachments, semantic profile evidence, ingress evidence, and diagnostics as typed records, while artifacts provide versioned serialized views suitable for CI, IDEs, agents, documentation, and downstream integrations. +This RFC defines local inspection APIs and deterministic evidence artifacts for IncQL plans. The APIs expose plan structure, schema flow, lineage, metadata attachments, semantic profile evidence, ingress evidence, and diagnostics as typed records, while artifacts provide versioned serialized views suitable for CI, IDEs, agents, documentation, and downstream integrations. ## Motivation -Relational evidence is only useful if authors and tools can inspect it without scraping logs or formatted explanations. InQL needs local APIs and artifacts that work without a hosted service, without a catalog product, and without executing the plan. This keeps plan inspection open, reproducible, and testable. +Relational evidence is only useful if authors and tools can inspect it without scraping logs or formatted explanations. IncQL needs local APIs and artifacts that work without a hosted service, without a catalog product, and without executing the plan. This keeps plan inspection open, reproducible, and testable. ## Goals -- Define local inspection APIs over InQL plans. +- Define local inspection APIs over IncQL plans. - Define deterministic artifact families for plan graph, lineage graph, schema flow, metadata attachments, semantic profiles, ingress mappings, client session context, and diagnostics. - Require artifact versioning and unsupported-evidence markers. - Keep human reports as projections from structured artifacts. @@ -43,7 +43,7 @@ Relational evidence is only useful if authors and tools can inspect it without s An author can inspect a lazy plan locally: ```incan -from pub::inql.inspect import inspect_plan +from pub::incql.inspect import inspect_plan inspection = inspect_plan(summary) inspection.output_schema() @@ -53,18 +53,18 @@ inspection.lineage().field("total_amount") The same inspection data can be written as artifacts for CI or downstream tools: ```incan -inspection.write_artifacts("target/inql") +inspection.write_artifacts("target/incql") ``` The exact helper names are illustrative; the contract is that structured inspection exists before execution. ## Reference-level explanation (precise rules) -InQL must expose a local inspection capability for plans that returns typed records, not only formatted strings. +IncQL must expose a local inspection capability for plans that returns typed records, not only formatted strings. Inspection records must include semantic targets, output schema information, relation structure, lineage when available, metadata attachments when available, semantic profile records or assessments when available, ingress origin mappings, client session context, and frontend coverage when available, diagnostics, and evidence-version metadata. -InQL must define deterministic serialized artifacts for at least: +IncQL must define deterministic serialized artifacts for at least: - plan graph - lineage graph @@ -75,11 +75,11 @@ InQL must define deterministic serialized artifacts for at least: - client session context - diagnostics -Artifacts must include schema version, InQL version, relevant rule versions, target identifiers, and unsupported-evidence markers. An empty lineage graph must be distinguishable from lineage that was not computed or is not supported. +Artifacts must include schema version, IncQL version, relevant rule versions, target identifiers, and unsupported-evidence markers. An empty lineage graph must be distinguishable from lineage that was not computed or is not supported. Human-readable reports may exist, but they must be generated from structured inspection records or artifacts. -Sensitive attachments must be redacted or omitted according to the visibility rules from InQL RFC 029. +Sensitive attachments must be redacted or omitted according to the visibility rules from IncQL RFC 029. ## Design details @@ -91,7 +91,7 @@ This RFC introduces no language syntax. Inspection is read-only. It must not execute a plan, bind physical sources, mutate Prism-authored meaning, or make policy decisions. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Method-chain, query-block, and future authoring surfaces should be inspectable through the same API once they lower to Prism. @@ -103,7 +103,7 @@ Existing code remains valid. New tooling should prefer structured inspection ove - **Only expose formatted explanations.** Rejected because tools need structured data. - **Only emit files, no API.** Rejected because IDEs and tests need in-memory inspection. -- **Wait for a higher-level catalog.** Rejected because local InQL users need inspection without external services. +- **Wait for a higher-level catalog.** Rejected because local IncQL users need inspection without external services. ## Drawbacks @@ -113,8 +113,8 @@ Existing code remains valid. New tooling should prefer structured inspection ove ## Layers affected -- **InQL specification** — local inspection becomes part of the relational evidence contract. -- **InQL library package** — inspection APIs and artifact writers must expose structured records. +- **IncQL specification** — local inspection becomes part of the relational evidence contract. +- **IncQL library package** — inspection APIs and artifact writers must expose structured records. - **Execution / interchange** — no execution is required, but artifacts may reference Substrait lowering status. - **Documentation** — docs must present artifacts as primary evidence and reports as derived views. diff --git a/docs/rfcs/032_execution_observations.md b/docs/rfcs/032_execution_observations.md index d29eb72c..3780009d 100644 --- a/docs/rfcs/032_execution_observations.md +++ b/docs/rfcs/032_execution_observations.md @@ -1,26 +1,26 @@ -# InQL RFC 032: Execution observations +# IncQL RFC 032: Execution observations - **Status:** Implemented - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 004 (execution context) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 040 (interoperability semantic profiles) -- **Issue:** [InQL #66](https://github.com/encero-systems/InQL/issues/66) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60); [InQL #85](https://github.com/encero-systems/InQL/pull/85); [InQL #87](https://github.com/encero-systems/InQL/pull/87) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 004 (execution context) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 040 (interoperability semantic profiles) +- **Issue:** [IncQL #66](https://github.com/encero-systems/IncQL/issues/66) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60); [IncQL #85](https://github.com/encero-systems/IncQL/pull/85); [IncQL #87](https://github.com/encero-systems/IncQL/pull/87) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines execution observations for InQL sessions. Execution observations correlate runtime attempts with semantic plan targets and record backend, adapter, semantic profile context, status, timing, diagnostics, row counts, and optional trace identifiers without making runtime logs the source of relational semantics. +This RFC defines execution observations for IncQL sessions. Execution observations correlate runtime attempts with semantic plan targets and record backend, adapter, semantic profile context, status, timing, diagnostics, row counts, and optional trace identifiers without making runtime logs the source of relational semantics. ## Motivation -After a plan executes, users and tools need evidence about what was attempted and what happened. A result table alone cannot explain which plan version ran, which adapter executed it, which diagnostics occurred, or how runtime observations attach to semantic targets. InQL needs a lightweight execution observation model that is structural, redacted by default, and independent of any particular telemetry backend. +After a plan executes, users and tools need evidence about what was attempted and what happened. A result table alone cannot explain which plan version ran, which adapter executed it, which diagnostics occurred, or how runtime observations attach to semantic targets. IncQL needs a lightweight execution observation model that is structural, redacted by default, and independent of any particular telemetry backend. ## Goals @@ -61,7 +61,7 @@ Diagnostics must be structured records. Sensitive values, row samples, query pay An execution observation may reference local inspection artifacts or semantic targets produced before execution. It must not mutate authored Prism targets or claim lineage that was not present in the plan evidence model. -Telemetry integrations may emit equivalent spans, events, logs, or metrics, but the InQL observation model must remain usable when no telemetry backend is configured. +Telemetry integrations may emit equivalent spans, events, logs, or metrics, but the IncQL observation model must remain usable when no telemetry backend is configured. ## Design details @@ -73,7 +73,7 @@ This RFC introduces no syntax. Execution observations are runtime evidence. They describe an attempt to execute a semantic plan through a session and adapter. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Quality observations, adapter coverage records, semantic profile records, and evidence exchange bridges may refer to execution observations. Pipeline layers may consume them, but orchestration behavior remains outside this RFC. @@ -100,7 +100,7 @@ The implemented scope adds typed execution observation records, observed `Sessio ## Alternatives considered - **Use backend logs only.** Rejected because logs are not stable semantic evidence and may be sensitive. -- **Require OpenTelemetry for all observations.** Rejected because local InQL evidence should work without provider configuration. +- **Require OpenTelemetry for all observations.** Rejected because local IncQL evidence should work without provider configuration. - **Attach execution data directly to plan nodes.** Rejected because runtime attempts are lifecycle events, not authored plan structure. ## Drawbacks @@ -111,8 +111,8 @@ The implemented scope adds typed execution observation records, observed `Sessio ## Layers affected -- **InQL specification** — execution observation fields and status values become normative. -- **InQL library package** — Session results must expose observations. +- **IncQL specification** — execution observation fields and status values become normative. +- **IncQL library package** — Session results must expose observations. - **Execution / interchange** — adapters must report execution facts without redefining semantics. - **Documentation** — docs must explain observation redaction and telemetry independence. @@ -124,4 +124,4 @@ Every adapter-backed observation must provide the semantic attempt target, plan Failed planning, binding, and lowering attempts use the same execution observation model when they occur through a session operation and can be attached to a plan target or the explicit unavailable-plan target. Their diagnostics identify which stage failed. This keeps runtime attempt evidence uniform without claiming that an authored Prism plan exists when planning did not complete. -Trace identifiers are represented as `list[str]`. InQL does not interpret the values or require a telemetry provider. Multiple telemetry systems can add multiple opaque identifiers, and local execution remains valid when the list is empty. +Trace identifiers are represented as `list[str]`. IncQL does not interpret the values or require a telemetry provider. Multiple telemetry systems can add multiple opaque identifiers, and local execution remains valid when the list is empty. diff --git a/docs/rfcs/033_adapter_requirements_coverage.md b/docs/rfcs/033_adapter_requirements_coverage.md index 810de134..edb73d6c 100644 --- a/docs/rfcs/033_adapter_requirements_coverage.md +++ b/docs/rfcs/033_adapter_requirements_coverage.md @@ -1,47 +1,47 @@ -# InQL RFC 033: Adapter requirements and coverage +# IncQL RFC 033: Adapter requirements and coverage - **Status:** Implemented - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 004 (execution context) - - InQL RFC 024 (function extension policy) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 032 (execution observations) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 041 (Prism plan ingress and external client frontends) - - InQL RFC 042 (async verification evidence) - - InQL RFC 043 (canonical equality and digest profiles) - - InQL RFC 044 (verifier statements and proof artifacts) - - InQL RFC 045 (constraint evidence and verification-aware planning) -- **Issue:** [InQL #67](https://github.com/encero-systems/InQL/issues/67) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60); [InQL #83](https://github.com/encero-systems/InQL/pull/83); [InQL #86](https://github.com/encero-systems/InQL/pull/86); [InQL #87](https://github.com/encero-systems/InQL/pull/87) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 004 (execution context) + - IncQL RFC 024 (function extension policy) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 032 (execution observations) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 041 (Prism plan ingress and external client frontends) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 043 (canonical equality and digest profiles) + - IncQL RFC 044 (verifier statements and proof artifacts) + - IncQL RFC 045 (constraint evidence and verification-aware planning) +- **Issue:** [IncQL #67](https://github.com/encero-systems/IncQL/issues/67) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60); [IncQL #83](https://github.com/encero-systems/IncQL/pull/83); [IncQL #86](https://github.com/encero-systems/IncQL/pull/86); [IncQL #87](https://github.com/encero-systems/IncQL/pull/87) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines adapter requirements and coverage states for InQL. Requirements describe backend capabilities needed by a plan or evidence contract, while coverage states report whether a specific adapter can satisfy those requirements under the relevant binding and semantic profile. Unknown coverage is not enforcement. +This RFC defines adapter requirements and coverage states for IncQL. Requirements describe backend capabilities needed by a plan or evidence contract, while coverage states report whether a specific adapter can satisfy those requirements under the relevant binding and semantic profile. Unknown coverage is not enforcement. ## Motivation -Backend neutrality only works when backend limits are visible. A plan may require extension functions, precise decimal behavior, variant semantics, lineage preservation, audit emission, masking, aggregation thresholding, or other capabilities. If InQL hides adapter uncertainty, downstream systems may assume a guarantee that the selected backend cannot provide. +Backend neutrality only works when backend limits are visible. A plan may require extension functions, precise decimal behavior, variant semantics, lineage preservation, audit emission, masking, aggregation thresholding, or other capabilities. If IncQL hides adapter uncertainty, downstream systems may assume a guarantee that the selected backend cannot provide. ## Goals - Define adapter requirements as semantic evidence targets. - Define coverage states: covered, partially_covered, uncovered, and unknown. - Require coverage records to name the adapter, semantic profile when relevant, and evidence. -- Keep backend inability distinct from unsupported InQL semantics. +- Keep backend inability distinct from unsupported IncQL semantics. - Make capability uncertainty explicit before execution when possible. ## Non-Goals - Defining every possible backend capability. - Defining physical execution strategies. -- Making any one adapter the semantic owner of InQL behavior. +- Making any one adapter the semantic owner of IncQL behavior. - Defining organization-wide enforcement policy. ## Guide-level explanation (how authors think about it) @@ -78,11 +78,11 @@ Coverage state must distinguish: - covered: the adapter can satisfy the requirement under the current binding - partially_covered: the adapter can satisfy part of the requirement or only under restrictions - uncovered: the adapter cannot satisfy the requirement -- unknown: InQL cannot determine whether the adapter can satisfy the requirement +- unknown: IncQL cannot determine whether the adapter can satisfy the requirement Unknown coverage must not be treated as covered. If a requirement whose guarantee level is required is unknown or uncovered, execution must reject, route, rewrite, require approval, or report non-enforcing behavior according to the higher-level policy using the coverage record. -Backend inability must be reported as adapter coverage or execution failure. It must not be encoded as a normal Substrait-level state for core InQL semantics. +Backend inability must be reported as adapter coverage or execution failure. It must not be encoded as a normal Substrait-level state for core IncQL semantics. Ingress coverage belongs to plan ingress frontends. It must not be reported as backend adapter coverage unless the same feature also creates an execution requirement for the selected adapter. @@ -96,7 +96,7 @@ This RFC introduces no syntax. Adapter requirements are evidence about what a plan needs. Coverage records are evidence about what a selected adapter can provide. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Function registry entries, semi-structured functions, extensions, quality assertions, governed attribute constraints, and semantic profile assessments may all create adapter requirements. Execution observations may reference coverage records. @@ -104,7 +104,7 @@ Function registry entries, semi-structured functions, extensions, quality assert Existing adapters may initially report unknown coverage for capabilities they do not declare. Consumers must distinguish unknown from covered. -The first implementation provides the adapter requirement and coverage record vocabulary plus `Session.check_coverage(requirements)` for caller-provided requirements. The RFC 033 completion slice adds inspection-inferred requirements for plan evidence that InQL can observe directly, including baseline null semantics, row filters, ordered execution, extension functions, variant semantics, and lineage-preservation evidence. `Session.check_inspection_coverage(inspection)` and `Session.check_plan_coverage(data)` evaluate those inferred requirements through the same adapter coverage model. Policy requirements that are not visible in plan evidence, such as masking, audit emission, region binding, waiver recording, and cryptographic proofs, still need explicit requirement records or their owning future surfaces. +The first implementation provides the adapter requirement and coverage record vocabulary plus `Session.check_coverage(requirements)` for caller-provided requirements. The RFC 033 completion slice adds inspection-inferred requirements for plan evidence that IncQL can observe directly, including baseline null semantics, row filters, ordered execution, extension functions, variant semantics, and lineage-preservation evidence. `Session.check_inspection_coverage(inspection)` and `Session.check_plan_coverage(data)` evaluate those inferred requirements through the same adapter coverage model. Policy requirements that are not visible in plan evidence, such as masking, audit emission, region binding, waiver recording, and cryptographic proofs, still need explicit requirement records or their owning future surfaces. ## Implementation plan @@ -115,7 +115,7 @@ The implemented scope adds the adapter requirement and coverage vocabulary, expl - [x] Define adapter requirement identity, target, capability, guarantee, reason references, and diagnostic fields. - [x] Define adapter coverage state, adapter identity, semantic profile, evidence references, and diagnostic fields. - [x] Add explicit `Session.check_coverage(requirements)` evaluation for caller-provided requirements. -- [x] Infer requirements from local plan inspection evidence where InQL owns the semantics. +- [x] Infer requirements from local plan inspection evidence where IncQL owns the semantics. - [x] Add `Session.check_inspection_coverage(inspection)` and `Session.check_plan_coverage(data)`. - [x] Keep unknown and uncovered coverage distinct from covered behavior. - [x] Keep backend inability in coverage or execution evidence, not as a normal Substrait-level state. @@ -125,7 +125,7 @@ The implemented scope adds the adapter requirement and coverage vocabulary, expl ## Alternatives considered - **Fail only at backend runtime.** Rejected because users need pre-execution visibility when possible. -- **Treat unsupported backend features as unsupported InQL semantics.** Rejected because backend inability is not the same as invalid InQL. +- **Treat unsupported backend features as unsupported IncQL semantics.** Rejected because backend inability is not the same as invalid IncQL. - **Use boolean supports flags.** Rejected because partial and unknown coverage are important operational states. ## Drawbacks @@ -136,8 +136,8 @@ The implemented scope adds the adapter requirement and coverage vocabulary, expl ## Layers affected -- **InQL specification** — adapter requirement and coverage vocabulary becomes normative. -- **InQL library package** — inspection and session APIs must expose requirements and coverage. +- **IncQL specification** — adapter requirement and coverage vocabulary becomes normative. +- **IncQL library package** — inspection and session APIs must expose requirements and coverage. - **Execution / interchange** — adapters must report capability evidence honestly. - **Documentation** — docs must explain that unknown coverage is not enforcement. @@ -145,7 +145,7 @@ The implemented scope adds the adapter requirement and coverage vocabulary, expl ### Resolved -The first implementation must cover the capability families that are directly visible from current InQL evidence: null semantics, row filters, ordered execution, extension functions, variant semantics, and lineage preservation. The broader public vocabulary remains available for explicit requirements and future owning surfaces, but InQL must not infer masking, audit emission, regional binding, waiver recording, cryptographic proof, or similar governance requirements until those surfaces produce evidence. +The first implementation must cover the capability families that are directly visible from current IncQL evidence: null semantics, row filters, ordered execution, extension functions, variant semantics, and lineage preservation. The broader public vocabulary remains available for explicit requirements and future owning surfaces, but IncQL must not infer masking, audit emission, regional binding, waiver recording, cryptographic proof, or similar governance requirements until those surfaces produce evidence. Coverage checks are available without binding physical sources when the requirement evidence is already present in a `PlanInspection` or caller-provided requirement list. Execution still validates bindings separately. This keeps pre-execution coverage useful without pretending that plan inspection can prove physical source availability. diff --git a/docs/rfcs/034_quality_assertions_observations.md b/docs/rfcs/034_quality_assertions_observations.md index 2b77b237..73d092e5 100644 --- a/docs/rfcs/034_quality_assertions_observations.md +++ b/docs/rfcs/034_quality_assertions_observations.md @@ -1,30 +1,30 @@ -# InQL RFC 034: Quality assertions and observations +# IncQL RFC 034: Quality assertions and observations - **Status:** Implemented - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 004 (execution context) - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 016 (core aggregate functions) - - InQL RFC 017 (aggregate modifiers) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 042 (async verification evidence) -- **Issue:** [InQL #68](https://github.com/encero-systems/InQL/issues/68) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60); [InQL #83](https://github.com/encero-systems/InQL/pull/83); [InQL #88](https://github.com/encero-systems/InQL/pull/88) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 004 (execution context) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 016 (core aggregate functions) + - IncQL RFC 017 (aggregate modifiers) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 042 (async verification evidence) +- **Issue:** [IncQL #68](https://github.com/encero-systems/IncQL/issues/68) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60); [IncQL #83](https://github.com/encero-systems/IncQL/pull/83); [IncQL #88](https://github.com/encero-systems/IncQL/pull/88) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines InQL quality assertions, quality assertion syntax, and quality observations. Quality assertions are typed relational checks over datasets, fields, groups, or explicit multi-relation inputs. Quality observations are runtime results produced by executing those assertions. A quality assertion is not an ordinary filter unless the author explicitly asks to filter rows. +This RFC defines IncQL quality assertions, quality assertion syntax, and quality observations. Quality assertions are typed relational checks over datasets, fields, groups, or explicit multi-relation inputs. Quality observations are runtime results produced by executing those assertions. A quality assertion is not an ordinary filter unless the author explicitly asks to filter rows. ## Motivation -Data quality needs to participate in typed relational planning without collapsing into ad hoc post-run tests or silent filters. InQL can express many quality checks as relational work: row counts, null rates, accepted values, uniqueness, ranges, group thresholds, and aggregate conditions. Those checks should produce observations that sessions, CI, and pipeline layers can consume. +Data quality needs to participate in typed relational planning without collapsing into ad hoc post-run tests or silent filters. IncQL can express many quality checks as relational work: row counts, null rates, accepted values, uniqueness, ranges, group thresholds, and aggregate conditions. Those checks should produce observations that sessions, CI, and pipeline layers can consume. ## Goals @@ -82,17 +82,17 @@ A quality observation must include observation identity, assertion identity, exe Observation status must distinguish passed, failed, errored, skipped, and unsupported. -Quality observation status describes the predicate outcome. When a quality observation is used as verification evidence, the verification observation defined by InQL RFC 042 carries the separate assurance label describing whether the predicate result was proven, verified, attested, sampled, waived, or unknown. +Quality observation status describes the predicate outcome. When a quality observation is used as verification evidence, the verification observation defined by IncQL RFC 042 carries the separate assurance label describing whether the predicate result was proven, verified, attested, sampled, waived, or unknown. Quality assertions may be planned as relational work. They must not change the cardinality or contents of the checked relation unless represented as an explicit transformation requested by the author. -Quality expressions must use ordinary InQL scalar, aggregate, and grouping semantics. Invalid expression context must be diagnosed before execution where possible. +Quality expressions must use ordinary IncQL scalar, aggregate, and grouping semantics. Invalid expression context must be diagnosed before execution where possible. ## Design details ### Syntax -This RFC introduces `quality { ... }` and expression-position `quality:` syntax for declaring a `list[QualityAssertion]`. The syntax is activated by importing `pub::inql`, like `query { ... }`. +This RFC introduces `quality { ... }` and expression-position `quality:` syntax for declaring a `list[QualityAssertion]`. The syntax is activated by importing `pub::incql`, like `query { ... }`. ```incan checks = quality: @@ -107,7 +107,7 @@ Each body item must be an assertion expression. Brace syntax uses newline-separa Quality assertions produce observations. They may inform session failure, warnings, CI status, or pipeline gates, but those policies are outside the assertion semantics. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces `quality` syntax lowers to the same assertion model as helper calls. It may be used next to `query {}` blocks by shaping a relation with `query {}` and declaring checks with `quality {}` before passing both to a `Session` observation API. Future pipeline syntax may lower to the same assertion model, but the assertion model must not become method-chain, query-block, or quality-block specific. @@ -119,7 +119,7 @@ The first implementation adds relation, field, group, and explicit cross-relatio ## Implementation plan -The implemented scope adds typed `QualityAssertion`, `QualityObservation`, `QualityMetric`, and quality enum records; helper APIs for the first assertion families; `quality` vocab syntax for assertion-list declarations; policy-neutral assertion mode/severity metadata; session evaluation APIs; concrete DataFusion-backed execution tests; reference documentation; and a task-oriented how-to guide. The evaluator uses ordinary InQL relation plans and structured materialization row counts, including aggregate/filter plans for field and group checks. It does not scrape preview text and does not push enforcement behavior into `Session`. +The implemented scope adds typed `QualityAssertion`, `QualityObservation`, `QualityMetric`, and quality enum records; helper APIs for the first assertion families; `quality` vocab syntax for assertion-list declarations; policy-neutral assertion mode/severity metadata; session evaluation APIs; concrete DataFusion-backed execution tests; reference documentation; and a task-oriented how-to guide. The evaluator uses ordinary IncQL relation plans and structured materialization row counts, including aggregate/filter plans for field and group checks. It does not scrape preview text and does not push enforcement behavior into `Session`. ## Progress checklist @@ -137,7 +137,7 @@ The implemented scope adds typed `QualityAssertion`, `QualityObservation`, `Qual ## Alternatives considered - **Treat every quality check as a filter.** Rejected because observations and transformations are different semantics. -- **Leave quality entirely to external tools.** Rejected because typed relational checks need InQL schema and expression semantics. +- **Leave quality entirely to external tools.** Rejected because typed relational checks need IncQL schema and expression semantics. - **Make `quality` syntax execute checks directly.** Rejected because declaration and observation are separate semantics. The syntax produces assertion values; session APIs produce runtime observations. ## Drawbacks @@ -148,8 +148,8 @@ The implemented scope adds typed `QualityAssertion`, `QualityObservation`, `Qual ## Layers affected -- **InQL specification** — quality assertion and observation semantics become normative. -- **InQL library package** — public helper APIs, quality vocab syntax, and session observation APIs are affected. +- **IncQL specification** — quality assertion and observation semantics become normative. +- **IncQL library package** — public helper APIs, quality vocab syntax, and session observation APIs are affected. - **Execution / interchange** — quality plans may lower to backend-executable relational work. - **Documentation** — docs must distinguish checks, filters, and pipeline gates. diff --git a/docs/rfcs/035_governed_attributes_policy_checkpoints.md b/docs/rfcs/035_governed_attributes_policy_checkpoints.md index 5614cb6d..c67b45c0 100644 --- a/docs/rfcs/035_governed_attributes_policy_checkpoints.md +++ b/docs/rfcs/035_governed_attributes_policy_checkpoints.md @@ -1,35 +1,35 @@ -# InQL RFC 035: Governed attributes and policy checkpoints +# IncQL RFC 035: Governed attributes and policy checkpoints - **Status:** Implemented - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 004 (execution context) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 033 (adapter requirements and coverage) -- **Issue:** [InQL #69](https://github.com/encero-systems/InQL/issues/69) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60); [InQL #89](https://github.com/encero-systems/InQL/pull/89) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 004 (execution context) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 033 (adapter requirements and coverage) +- **Issue:** [IncQL #69](https://github.com/encero-systems/IncQL/issues/69) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60); [IncQL #89](https://github.com/encero-systems/IncQL/pull/89) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** v0.1 ## Summary -This RFC defines how InQL carries governed attributes and records policy checkpoints as local relational evidence. Governed attributes are typed facts attached to semantic targets with provenance, confidence, authority, and lifetime. Policy checkpoints are decision records attached at authoring, planning, binding, or execution boundaries. InQL carries and propagates evidence; it does not define an organization-wide policy engine. +This RFC defines how IncQL carries governed attributes and records policy checkpoints as local relational evidence. Governed attributes are typed facts attached to semantic targets with provenance, confidence, authority, and lifetime. Policy checkpoints are decision records attached at authoring, planning, binding, or execution boundaries. IncQL carries and propagates evidence; it does not define an organization-wide policy engine. ## Motivation -Relational plans often need to carry facts such as classification, origin, purpose, jurisdiction, derivation, masking status, or coverage state. Those facts may be supplied by model metadata, user declarations, imported artifacts, catalogs, policy engines, or prior plans. InQL should preserve and propagate them through relational semantics without pretending that inferred attributes are automatically authoritative policy truth. +Relational plans often need to carry facts such as classification, origin, purpose, jurisdiction, derivation, masking status, or coverage state. Those facts may be supplied by model metadata, user declarations, imported artifacts, catalogs, policy engines, or prior plans. IncQL should preserve and propagate them through relational semantics without pretending that inferred attributes are automatically authoritative policy truth. ## Goals - Define governed attributes as typed evidence attached to semantic targets. - Preserve source, confidence, authority, status, observed time, expiration, and evidence references. - Define policy checkpoint records at authoring, planning, binding, and execution. -- Allow InQL to explain how attributes move through relational transformations. -- Keep policy authoring, approval, and global enforcement outside InQL. +- Allow IncQL to explain how attributes move through relational transformations. +- Keep policy authoring, approval, and global enforcement outside IncQL. ## Non-Goals @@ -65,7 +65,7 @@ Confidence must distinguish exact, high, medium, low, and unknown or equivalent Status must distinguish asserted, inferred, accepted, rejected, overridden, stale, and pending_review or equivalent review states. -InQL may propagate attributes through relational transformations when transformation semantics are known. It must preserve provenance and confidence. It must report conservative or unknown propagation when exact propagation is not available. +IncQL may propagate attributes through relational transformations when transformation semantics are known. It must preserve provenance and confidence. It must report conservative or unknown propagation when exact propagation is not available. A policy checkpoint record must include decision identity, target, checkpoint, action, policy reference, reason code, evidence references, visibility, and optional diagnostics. @@ -81,9 +81,9 @@ This RFC introduces no policy syntax. ### Semantics -InQL owns attribute carriage, propagation evidence, and checkpoint records. It does not own organizational policy meaning. +IncQL owns attribute carriage, propagation evidence, and checkpoint records. It does not own organizational policy meaning. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Lineage edges explain how attributes may propagate. Adapter requirements may be created when a policy checkpoint requires backend capabilities such as masking or row filtering. @@ -93,7 +93,7 @@ Existing plans without governed attributes remain valid. Consumers must treat ab ## Alternatives considered -- **Make InQL a policy engine.** Rejected because policy authoring and approval are outside typed relational semantics. +- **Make IncQL a policy engine.** Rejected because policy authoring and approval are outside typed relational semantics. - **Use plain metadata tags only.** Rejected because provenance, confidence, authority, and status are required for trustworthy evidence. - **Drop uncertain attributes.** Rejected because uncertainty is meaningful evidence. @@ -105,8 +105,8 @@ Existing plans without governed attributes remain valid. Consumers must treat ab ## Layers affected -- **InQL specification** — governed attribute and checkpoint record semantics become normative. -- **InQL library package** — inspection APIs must expose attributes and decisions as typed records. +- **IncQL specification** — governed attribute and checkpoint record semantics become normative. +- **IncQL library package** — inspection APIs must expose attributes and decisions as typed records. - **Execution / interchange** — Session and adapters may attach binding and execution checkpoint records. - **Documentation** — docs must distinguish attribute evidence from policy authority. @@ -114,11 +114,11 @@ Existing plans without governed attributes remain valid. Consumers must treat ab ### Core attribute keys -The first core governed attribute key is `schema.substrait_primitive_kind`, emitted from local inspection when an output field has a known primitive kind. Additional core keys must use an `inql.` or similarly explicit namespace when they describe InQL-owned evidence. User, catalog, policy, and imported artifact keys may use their own namespaces, but InQL must preserve their source and authority rather than silently treating them as core facts. +The first core governed attribute key is `schema.substrait_primitive_kind`, emitted from local inspection when an output field has a known primitive kind. Additional core keys must use an `incql.` or similarly explicit namespace when they describe IncQL-owned evidence. User, catalog, policy, and imported artifact keys may use their own namespaces, but IncQL must preserve their source and authority rather than silently treating them as core facts. ### First propagation rules -The first propagation rule is conservative schema evidence propagation from inspection metadata into field-scoped governed attributes. The emitted attributes use `GovernedAttributeSource.Lineage`, `GovernedAttributeConfidence.Exact`, and `GovernedAttributeStatus.Inferred` because they come from local plan/schema evidence rather than external authority. InQL must not fabricate policy, catalog, masking, jurisdiction, or classification attributes from name heuristics alone. +The first propagation rule is conservative schema evidence propagation from inspection metadata into field-scoped governed attributes. The emitted attributes use `GovernedAttributeSource.Lineage`, `GovernedAttributeConfidence.Exact`, and `GovernedAttributeStatus.Inferred` because they come from local plan/schema evidence rather than external authority. IncQL must not fabricate policy, catalog, masking, jurisdiction, or classification attributes from name heuristics alone. ### Serialization boundary @@ -158,7 +158,7 @@ Governed attributes and policy checkpoints are local evidence records in the fir - [x] Add governed attribute enums and model. - [x] Add policy checkpoint enums and model. - [x] Add helper constructors and modifier methods. -- [x] Export public governance records and helpers through `pub::inql`. +- [x] Export public governance records and helpers through `pub::incql`. ### Inspection diff --git a/docs/rfcs/036_governed_plan_bundle.md b/docs/rfcs/036_governed_plan_bundle.md index e9c4f29e..b44d4245 100644 --- a/docs/rfcs/036_governed_plan_bundle.md +++ b/docs/rfcs/036_governed_plan_bundle.md @@ -1,44 +1,44 @@ -# InQL RFC 036: Governed plan bundle +# IncQL RFC 036: Governed plan bundle - **Status:** Draft - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 034 (quality assertions and observations) - - InQL RFC 035 (governed attributes and policy checkpoints) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 041 (Prism plan ingress and external client frontends) - - InQL RFC 042 (async verification evidence) - - InQL RFC 043 (canonical equality and digest profiles) - - InQL RFC 044 (verifier statements and proof artifacts) - - InQL RFC 045 (constraint evidence and verification-aware planning) - - InQL RFC 046 (data contract ingress and product topology) - - InQL RFC 047 (semantic evidence graph and agent query surface) -- **Issue:** [InQL #70](https://github.com/encero-systems/InQL/issues/70) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60); [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 034 (quality assertions and observations) + - IncQL RFC 035 (governed attributes and policy checkpoints) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 041 (Prism plan ingress and external client frontends) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 043 (canonical equality and digest profiles) + - IncQL RFC 044 (verifier statements and proof artifacts) + - IncQL RFC 045 (constraint evidence and verification-aware planning) + - IncQL RFC 046 (data contract ingress and product topology) + - IncQL RFC 047 (semantic evidence graph and agent query surface) +- **Issue:** [IncQL #70](https://github.com/encero-systems/IncQL/issues/70) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60); [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines the governed plan bundle as the local InQL artifact that keeps relational computation and evidence together. A bundle contains a plan reference, schemas, lineage, governed attributes, policy checkpoints, quality assertions, verification evidence, canonical equality profiles, verifier statements, proof artifacts, constraint evidence, data contract evidence, product topology, semantic evidence graph projections, semantic profiles, ingress evidence, client session context, adapter requirements, coverage records, evidence references, and version metadata for the InQL-owned parts of governed relational computation. +This RFC defines the governed plan bundle as the local IncQL artifact that keeps relational computation and evidence together. A bundle contains a plan reference, schemas, lineage, governed attributes, policy checkpoints, quality assertions, verification evidence, canonical equality profiles, verifier statements, proof artifacts, constraint evidence, data contract evidence, product topology, semantic evidence graph projections, semantic profiles, ingress evidence, client session context, adapter requirements, coverage records, evidence references, and version metadata for the IncQL-owned parts of governed relational computation. ## Motivation -Individual evidence artifacts are useful, but many consumers need a coherent handoff unit. A plan without its evidence can be executed without understanding requirements. Evidence without the plan cannot explain what computation it describes. The governed plan bundle gives InQL a portable local package for the facts it owns while leaving hosted storage, global policy, approvals, and cross-system reasoning outside the contract. +Individual evidence artifacts are useful, but many consumers need a coherent handoff unit. A plan without its evidence can be executed without understanding requirements. Evidence without the plan cannot explain what computation it describes. The governed plan bundle gives IncQL a portable local package for the facts it owns while leaving hosted storage, global policy, approvals, and cross-system reasoning outside the contract. ## Goals -- Define a bundle shape for InQL-owned relational evidence. +- Define a bundle shape for IncQL-owned relational evidence. - Keep plan, schema, lineage, attributes, policy checkpoints, quality assertions, verification evidence, canonical equality profiles, verifier statements, proof artifacts, constraint evidence, data contract evidence, product topology, semantic evidence graph projections, semantic profiles, ingress evidence, client session context, adapter requirements, coverage, and versions together. - Support local tooling and downstream generic consumers. -- Avoid proprietary hosted behavior in the InQL contract. +- Avoid proprietary hosted behavior in the IncQL contract. ## Non-Goals @@ -53,7 +53,7 @@ An author or CI job can produce one bundle for a planned relation: ```incan bundle = governed_plan_bundle(summary) -bundle.write("target/inql/summary.bundle.json") +bundle.write("target/incql/summary.bundle.json") ``` The bundle can be inspected locally or consumed by other tools. It does not require a hosted service. @@ -63,7 +63,7 @@ The bundle can be inspected locally or consumed by other tools. It does not requ A governed plan bundle must include: - bundle schema version -- InQL version and relevant rule versions +- IncQL version and relevant rule versions - plan target - input schema references - output schema reference @@ -89,7 +89,7 @@ A governed plan bundle must include: The bundle must distinguish required, optional, unavailable, and unsupported evidence sections. A missing evidence section must not be treated as an empty evidence section. -The bundle may include a Substrait plan or a reference to a Substrait artifact, but Substrait must not be the only source of InQL evidence in the bundle. +The bundle may include a Substrait plan or a reference to a Substrait artifact, but Substrait must not be the only source of IncQL evidence in the bundle. Sensitive or redacted evidence must follow attachment visibility rules. @@ -105,7 +105,7 @@ This RFC introduces no authoring syntax. The bundle is an evidence package. It does not make policy decisions by itself. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Inspection artifacts, execution observations, quality observations, ingress frontends, and evidence exchange bridges may all read from or write to bundle-compatible records. @@ -116,8 +116,8 @@ Bundles must be versioned from the start. Early bundles may contain fewer eviden ## Alternatives considered - **Only emit separate artifacts.** Rejected because consumers often need a coherent handoff unit. -- **Make the bundle a hosted-service protocol.** Rejected because local InQL evidence must remain open and service-independent. -- **Embed all evidence directly into Substrait.** Rejected because Substrait is not a complete InQL evidence model. +- **Make the bundle a hosted-service protocol.** Rejected because local IncQL evidence must remain open and service-independent. +- **Embed all evidence directly into Substrait.** Rejected because Substrait is not a complete IncQL evidence model. ## Drawbacks @@ -127,8 +127,8 @@ Bundles must be versioned from the start. Early bundles may contain fewer eviden ## Layers affected -- **InQL specification** — bundle contents and required distinctions become normative. -- **InQL library package** — APIs must produce bundle-compatible records. +- **IncQL specification** — bundle contents and required distinctions become normative. +- **IncQL library package** — APIs must produce bundle-compatible records. - **Execution / interchange** — Substrait may be included or referenced, but not treated as the sole evidence store. - **Documentation** — docs must define local bundle use without implying hosted-service requirements. diff --git a/docs/rfcs/037_plan_diff_blast_radius_inputs.md b/docs/rfcs/037_plan_diff_blast_radius_inputs.md index e3a2cbc3..5c3046a5 100644 --- a/docs/rfcs/037_plan_diff_blast_radius_inputs.md +++ b/docs/rfcs/037_plan_diff_blast_radius_inputs.md @@ -1,28 +1,28 @@ -# InQL RFC 037: Plan diff and blast-radius inputs +# IncQL RFC 037: Plan diff and blast-radius inputs - **Status:** Draft - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 040 (interoperability semantic profiles) -- **Issue:** [InQL #71](https://github.com/encero-systems/InQL/issues/71) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 040 (interoperability semantic profiles) +- **Issue:** [IncQL #71](https://github.com/encero-systems/IncQL/issues/71) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines local InQL plan diffs and blast-radius input artifacts. A plan diff compares two InQL evidence artifacts and classifies changes in output schema, field identity, lineage, joins, filters, aggregates, windows, generators, quality assertions, semantic profiles, adapter requirements, and coverage. The result is a local input to downstream impact analysis, not an organization-wide blast-radius service. +This RFC defines local IncQL plan diffs and blast-radius input artifacts. A plan diff compares two IncQL evidence artifacts and classifies changes in output schema, field identity, lineage, joins, filters, aggregates, windows, generators, quality assertions, semantic profiles, adapter requirements, and coverage. The result is a local input to downstream impact analysis, not an organization-wide blast-radius service. ## Motivation -Typed relational evidence should help users understand change before it reaches production. If a query changes, tools should know whether output fields changed, dependencies changed, adapter requirements changed, or quality checks changed. InQL can produce accurate local diff evidence because it owns the plan and lineage, but it should not claim to know every downstream consumer in every organization. +Typed relational evidence should help users understand change before it reaches production. If a query changes, tools should know whether output fields changed, dependencies changed, adapter requirements changed, or quality checks changed. IncQL can produce accurate local diff evidence because it owns the plan and lineage, but it should not claim to know every downstream consumer in every organization. ## Goals @@ -51,11 +51,11 @@ diff.changed_output_fields() diff.changed_adapter_requirements() ``` -The diff tells InQL-local facts. A higher-level system may combine those facts with dependency indexes and approvals. +The diff tells IncQL-local facts. A higher-level system may combine those facts with dependency indexes and approvals. ## Reference-level explanation (precise rules) -A plan diff must compare two compatible InQL evidence artifacts or bundles. If artifacts are incompatible, stale, or missing required identity information, the diff must report unsupported or unknown impact. +A plan diff must compare two compatible IncQL evidence artifacts or bundles. If artifacts are incompatible, stale, or missing required identity information, the diff must report unsupported or unknown impact. The diff must classify at least: @@ -80,9 +80,9 @@ The diff must classify at least: Diff records must include affected semantic targets when available, change kind, severity or compatibility classification when known, evidence references, and confidence. -Unknown impact must be explicit. If InQL cannot determine whether a change affects a target, it must not omit the target silently. +Unknown impact must be explicit. If IncQL cannot determine whether a change affects a target, it must not omit the target silently. -Plan diffs may produce blast-radius input artifacts. Those artifacts describe local affected targets and requirement changes. They do not claim to enumerate all downstream consumers outside InQL's local artifact set. +Plan diffs may produce blast-radius input artifacts. Those artifacts describe local affected targets and requirement changes. They do not claim to enumerate all downstream consumers outside IncQL's local artifact set. ## Design details @@ -94,7 +94,7 @@ This RFC introduces no syntax. Diffs operate over evidence artifacts, not raw source text. Text diffs may be useful but are not sufficient for semantic change classification. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Plan diffs depend on stable semantic identity, lineage, inspection artifacts, semantic profiles, and governed plan bundles. They should not be implemented before those contracts exist. @@ -105,7 +105,7 @@ Older artifacts may not contain enough identity or lineage for precise diffs. Di ## Alternatives considered - **Use textual diffs only.** Rejected because text diffs cannot reliably classify relational meaning changes. -- **Make InQL own full blast radius.** Rejected because downstream consumers, deployments, dashboards, and global dependency indexes are outside InQL. +- **Make IncQL own full blast radius.** Rejected because downstream consumers, deployments, dashboards, and global dependency indexes are outside IncQL. - **Ignore unknown impact.** Rejected because unknown impact is a real result. ## Drawbacks @@ -116,15 +116,15 @@ Older artifacts may not contain enough identity or lineage for precise diffs. Di ## Layers affected -- **InQL specification** — change kinds and local diff semantics become normative. -- **InQL library package** — diff APIs must compare structured artifacts. +- **IncQL specification** — change kinds and local diff semantics become normative. +- **IncQL library package** — diff APIs must compare structured artifacts. - **Execution / interchange** — adapter requirement changes must be included where execution behavior changes. - **Documentation** — docs must distinguish local blast-radius inputs from global impact analysis. ## Unresolved questions - Which change kinds should be compatibility-classified in the first release? -- Should diff severity be part of InQL or left entirely to downstream policy? +- Should diff severity be part of IncQL or left entirely to downstream policy? - How should diffs handle generated or unstable field names? diff --git a/docs/rfcs/038_evidence_exchange_bridges.md b/docs/rfcs/038_evidence_exchange_bridges.md index 5328c372..b534acf0 100644 --- a/docs/rfcs/038_evidence_exchange_bridges.md +++ b/docs/rfcs/038_evidence_exchange_bridges.md @@ -1,53 +1,53 @@ -# InQL RFC 038: Evidence exchange bridges +# IncQL RFC 038: Evidence exchange bridges - **Status:** Draft - **Created:** 2026-05-29 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 027 (relational evidence program) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 032 (execution observations) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 042 (async verification evidence) - - InQL RFC 043 (canonical equality and digest profiles) - - InQL RFC 044 (verifier statements and proof artifacts) - - InQL RFC 045 (constraint evidence and verification-aware planning) - - InQL RFC 046 (data contract ingress and product topology) - - InQL RFC 047 (semantic evidence graph and agent query surface) -- **Issue:** [InQL #72](https://github.com/encero-systems/InQL/issues/72) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60); [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 032 (execution observations) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 043 (canonical equality and digest profiles) + - IncQL RFC 044 (verifier statements and proof artifacts) + - IncQL RFC 045 (constraint evidence and verification-aware planning) + - IncQL RFC 046 (data contract ingress and product topology) + - IncQL RFC 047 (semantic evidence graph and agent query surface) +- **Issue:** [IncQL #72](https://github.com/encero-systems/IncQL/issues/72) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60); [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines evidence exchange bridges between InQL's internal evidence model and external or adjacent formats. Exchange bridges map InQL plan, lineage, schema-flow, execution, quality, verification, canonical equality, proof artifact, constraint, contract, product topology, evidence graph, coverage, semantic profile, and bundle records into downstream views such as OpenLineage events, telemetry signals, semantic inspection fragments, transformation-project artifacts, data-contract artifacts, product-topology artifacts, graph projections, and catalog/governance integration artifacts. They may also ingest external evidence artifacts such as transformation manifests, source catalogs, schema catalogs, run results, verification results, proof results, constraint metadata, contract artifacts, product artifacts, runtime lineage events, and orchestration metadata. Representative artifact families include dbt manifests and run results, Open Data Contract Standard artifacts, Open Data Product Standard artifacts, legacy Data Contract Specification artifacts, Glue Data Catalog or Hive Metastore snapshots, Airflow or MWAA DAG metadata, Dagster assets, Prefect deployment metadata, OpenLineage events, DataHub or OpenMetadata catalog records, and Great Expectations-style quality results. Inbound artifacts and outbound projections are evidence exchange records, not the internal source of truth. +This RFC defines evidence exchange bridges between IncQL's internal evidence model and external or adjacent formats. Exchange bridges map IncQL plan, lineage, schema-flow, execution, quality, verification, canonical equality, proof artifact, constraint, contract, product topology, evidence graph, coverage, semantic profile, and bundle records into downstream views such as OpenLineage events, telemetry signals, semantic inspection fragments, transformation-project artifacts, data-contract artifacts, product-topology artifacts, graph projections, and catalog/governance integration artifacts. They may also ingest external evidence artifacts such as transformation manifests, source catalogs, schema catalogs, run results, verification results, proof results, constraint metadata, contract artifacts, product artifacts, runtime lineage events, and orchestration metadata. Representative artifact families include dbt manifests and run results, Open Data Contract Standard artifacts, Open Data Product Standard artifacts, legacy Data Contract Specification artifacts, Glue Data Catalog or Hive Metastore snapshots, Airflow or MWAA DAG metadata, Dagster assets, Prefect deployment metadata, OpenLineage events, DataHub or OpenMetadata catalog records, and Great Expectations-style quality results. Inbound artifacts and outbound projections are evidence exchange records, not the internal source of truth. ## Motivation -InQL evidence should be useful outside InQL, and external project artifacts should be usable as evidence inputs when they are explicit about their source and scope. CI systems, lineage tools, telemetry pipelines, catalogs, notebooks, transformation frameworks, orchestrators, and agents may all consume or produce different formats. Systems such as dbt, Airflow, MWAA, Dagster, Prefect, Glue Data Catalog, Hive Metastore, DataHub, OpenMetadata, OpenLineage, and Great Expectations are useful ecosystem examples, but none of them should become InQL's internal evidence model. If each integration reconstructs evidence independently, semantics will drift. InQL should provide exchange bridges that preserve its local evidence model while acknowledging that external formats may be less expressive or may represent facts at a different semantic layer. +IncQL evidence should be useful outside IncQL, and external project artifacts should be usable as evidence inputs when they are explicit about their source and scope. CI systems, lineage tools, telemetry pipelines, catalogs, notebooks, transformation frameworks, orchestrators, and agents may all consume or produce different formats. Systems such as dbt, Airflow, MWAA, Dagster, Prefect, Glue Data Catalog, Hive Metastore, DataHub, OpenMetadata, OpenLineage, and Great Expectations are useful ecosystem examples, but none of them should become IncQL's internal evidence model. If each integration reconstructs evidence independently, semantics will drift. IncQL should provide exchange bridges that preserve its local evidence model while acknowledging that external formats may be less expressive or may represent facts at a different semantic layer. ## Goals -- Define exchange bridges as inbound and outbound mappings around InQL evidence. +- Define exchange bridges as inbound and outbound mappings around IncQL evidence. - Preserve semantic target references and evidence versions where possible. - Allow lossy external mappings only when loss is explicit. -- Allow external artifacts to seed metadata, lineage hints, quality observations, verification observations, proof artifact references, constraint evidence, contract evidence, product topology, run observations, graph projections, and target mappings without becoming authoritative InQL semantics. +- Allow external artifacts to seed metadata, lineage hints, quality observations, verification observations, proof artifact references, constraint evidence, contract evidence, product topology, run observations, graph projections, and target mappings without becoming authoritative IncQL semantics. - Prefer public standards and open ecosystem specifications for external projections when they fit the evidence being exchanged. - Support transformation-framework artifacts such as manifests, catalogs, run results, source definitions, model metadata, tests, tags, and documentation scaffolds, including dbt-shaped artifacts where a bridge supports that profile. -- Keep provider configuration and hosted ingestion outside InQL core. +- Keep provider configuration and hosted ingestion outside IncQL core. - Support local exchange without requiring a specific external service. ## Non-Goals -- Making any external format the internal InQL evidence model. +- Making any external format the internal IncQL evidence model. - Defining hosted ingestion, storage, dashboards, or managed governance behavior. - Defining a telemetry provider, collector, exporter, or sampling policy. -- Guaranteeing that every external tool can represent every InQL evidence feature. +- Guaranteeing that every external tool can represent every IncQL evidence feature. - Guaranteeing that imported transformation, catalog, or orchestration artifacts are complete or semantically authoritative. - Defining a full migration product, transformation runtime, or orchestration engine. @@ -57,11 +57,11 @@ An author or CI job can exchange evidence with local artifacts: ```incan bundle = governed_plan_bundle(summary) -bundle.export_openlineage("target/inql/openlineage.json") -bundle.export_telemetry("target/inql/telemetry.json") +bundle.export_openlineage("target/incql/openlineage.json") +bundle.export_telemetry("target/incql/telemetry.json") ``` -The names are illustrative. The key contract is that outbound exports are generated from InQL evidence artifacts, not from backend logs or reconstructed SQL strings. +The names are illustrative. The key contract is that outbound exports are generated from IncQL evidence artifacts, not from backend logs or reconstructed SQL strings. For transformation-project workflows, an exchange bridge can also ingest project artifacts and emit reviewable suggestions: @@ -73,7 +73,7 @@ sources = bundle.export_transformation_sources() tests = bundle.export_transformation_quality_suggestions() ``` -The bridge may read common artifacts such as manifests, catalogs, run results, source definitions, tests, tags, metadata, and documentation fragments. In a dbt-shaped bridge, for example, those inputs may include `manifest.json`, `catalog.json`, `run_results.json`, source YAML, model YAML, tags, exposures, tests, and documentation blocks. It may emit suggested source declarations, model metadata, test definitions, tags, exposures, or documentation scaffolds. Those suggestions remain projections from InQL evidence and imported artifact evidence; they do not make the transformation framework the semantic owner of the plan. +The bridge may read common artifacts such as manifests, catalogs, run results, source definitions, tests, tags, metadata, and documentation fragments. In a dbt-shaped bridge, for example, those inputs may include `manifest.json`, `catalog.json`, `run_results.json`, source YAML, model YAML, tags, exposures, tests, and documentation blocks. It may emit suggested source declarations, model metadata, test definitions, tags, exposures, or documentation scaffolds. Those suggestions remain projections from IncQL evidence and imported artifact evidence; they do not make the transformation framework the semantic owner of the plan. ## Reference-level explanation (precise rules) @@ -81,9 +81,9 @@ An exchange bridge must declare its direction, source evidence schema versions, Outbound exchange bridges must preserve semantic target identifiers when the target format can carry them. When the target format cannot carry them directly, the bridge should preserve them in an extension, custom facet, attribute, or sidecar artifact when safe. -Inbound exchange bridges must preserve external artifact identity, source location, artifact version, and confidence. Imported records may attach metadata, origin hints, observed run facts, quality observations, verification observations, or candidate mappings to InQL semantic targets. They must not create InQL lineage, policy decisions, quality pass/fail states, verification assurance stronger than attested, or adapter coverage unless the corresponding InQL evidence contract can represent and validate that evidence. +Inbound exchange bridges must preserve external artifact identity, source location, artifact version, and confidence. Imported records may attach metadata, origin hints, observed run facts, quality observations, verification observations, or candidate mappings to IncQL semantic targets. They must not create IncQL lineage, policy decisions, quality pass/fail states, verification assurance stronger than attested, or adapter coverage unless the corresponding IncQL evidence contract can represent and validate that evidence. -Lossy mappings must be explicit. If an external lineage format cannot distinguish value, control, grouping, join, and sort lineage, the bridge must either preserve the distinction through an extension or report the loss. If an imported artifact collapses source relation, model, test, and run-result semantics into one node vocabulary, the bridge must report that limitation instead of pretending the artifact has InQL target precision. +Lossy mappings must be explicit. If an external lineage format cannot distinguish value, control, grouping, join, and sort lineage, the bridge must either preserve the distinction through an extension or report the loss. If an imported artifact collapses source relation, model, test, and run-result semantics into one node vocabulary, the bridge must report that limitation instead of pretending the artifact has IncQL target precision. Sensitive attachments must follow visibility rules. Exchange bridges must not leak sensitive payloads merely because a target format lacks redaction semantics. @@ -97,7 +97,7 @@ This RFC introduces no authoring syntax. ### Semantics -Outbound exports are projections. Inbound artifacts are evidence inputs. Neither direction may become the authoritative source of InQL plan, lineage, quality, or execution semantics. +Outbound exports are projections. Inbound artifacts are evidence inputs. Neither direction may become the authoritative source of IncQL plan, lineage, quality, or execution semantics. ### Standards alignment @@ -111,17 +111,17 @@ Exchange bridges should prefer public standards and open ecosystem specification - in-toto and SLSA provenance for signed supply-chain-style attestations, materials, subjects, builders, and reproducible provenance claims. - W3C Verifiable Credentials, JSON Web Signature, COSE, and JSON canonicalization specifications when portable signed claims or canonical signed payloads are required by a bridge profile. -Open-data governance frameworks such as the Open Data Charter, EU open-data policy guidance, public data governance training materials, and IEEE open-data governance principles should inform documentation, transparency, privacy, reuse, and stewardship guidance. They are not internal evidence schemas and must not override InQL semantic targets. +Open-data governance frameworks such as the Open Data Charter, EU open-data policy guidance, public data governance training materials, and IEEE open-data governance principles should inform documentation, transparency, privacy, reuse, and stewardship guidance. They are not internal evidence schemas and must not override IncQL semantic targets. -Standards mappings must be versioned bridge profiles. If a target standard cannot represent InQL evidence without loss, the bridge must report mapping loss and should emit an extension, custom facet, credential claim, or sidecar artifact when safe. An external standards document must not become the internal source of InQL semantics merely because an exchange bridge supports it. +Standards mappings must be versioned bridge profiles. If a target standard cannot represent IncQL evidence without loss, the bridge must report mapping loss and should emit an extension, custom facet, credential claim, or sidecar artifact when safe. An external standards document must not become the internal source of IncQL semantics merely because an exchange bridge supports it. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Exchange bridges depend on inspection artifacts, execution observations, quality observations, adapter coverage, interoperability profiles, and governed plan bundles. They should map from or into those records rather than from backend-specific plans. Transformation-framework bridges are a first-class example. A bridge may ingest manifest, catalog, run-result, source, model, test, tag, exposure, metadata, and documentation artifacts from systems such as dbt, Airflow, MWAA, Dagster, or Prefect when the bridge profile supports them. It may export suggested source definitions, model metadata, quality tests, documentation scaffolds, exposures, tags, or run validation summaries. The bridge must keep imported project semantics distinct from Prism-authored semantics and must identify any profile assumptions used to compare source and target environments. -Data-contract, data-product, and graph bridge profiles are specialized by InQL RFC 046 and InQL RFC 047. Contract and product artifacts may seed normalized evidence, product topology, and graph nodes or edges, but they remain imported evidence. Runtime lineage events may seed observed graph relationships, but they remain observed or attested evidence unless separate verification evidence supports a stronger assurance label. +Data-contract, data-product, and graph bridge profiles are specialized by IncQL RFC 046 and IncQL RFC 047. Contract and product artifacts may seed normalized evidence, product topology, and graph nodes or edges, but they remain imported evidence. Runtime lineage events may seed observed graph relationships, but they remain observed or attested evidence unless separate verification evidence supports a stronger assurance label. ### Compatibility / migration @@ -129,9 +129,9 @@ Exchange bridges must version their mappings. Adding a new internal evidence fie ## Alternatives considered -- **Adopt one external lineage model internally.** Rejected because InQL needs evidence that many external tools cannot represent directly. +- **Adopt one external lineage model internally.** Rejected because IncQL needs evidence that many external tools cannot represent directly. - **Leave all exchange to downstream systems.** Rejected because independent reconstruction causes drift. -- **Require hosted ingestion.** Rejected because local export must work in open InQL. +- **Require hosted ingestion.** Rejected because local export must work in open IncQL. - **Treat transformation project artifacts as authoritative semantics.** Rejected because those artifacts are valuable evidence, but they are not Prism's analyzed relational model. ## Drawbacks @@ -143,8 +143,8 @@ Exchange bridges must version their mappings. Adding a new internal evidence fie ## Layers affected -- **InQL specification** — exchange bridge responsibilities and loss reporting become normative. -- **InQL library package** — exchange APIs may live in core or optional modules. +- **IncQL specification** — exchange bridge responsibilities and loss reporting become normative. +- **IncQL library package** — exchange APIs may live in core or optional modules. - **Execution / interchange** — exchanges may include Substrait references, telemetry-shaped observations, lineage events, transformation artifacts, and run-result evidence. - **Documentation** — docs must identify external exchanges as evidence inputs or projections, not internal truth. @@ -153,8 +153,8 @@ Exchange bridges must version their mappings. Adding a new internal evidence fie - Which exchange bridge profiles are required by this RFC? - Which public standards bridge profiles are required before this RFC can advance beyond Draft? - Should exchange bridges live in the core package or optional integration packages? -- What sidecar format should preserve InQL-specific evidence when an external target is lossy? +- What sidecar format should preserve IncQL-specific evidence when an external target is lossy? - Which transformation-project artifact profiles are required by this RFC? -- Which data contract, product topology, and graph projection bridge profiles are required by this RFC versus owned entirely by InQL RFC 046 and InQL RFC 047? +- Which data contract, product topology, and graph projection bridge profiles are required by this RFC versus owned entirely by IncQL RFC 046 and IncQL RFC 047? diff --git a/docs/rfcs/039_pandas_familiar_exploration_api.md b/docs/rfcs/039_pandas_familiar_exploration_api.md index dc3cc8ff..c98d961e 100644 --- a/docs/rfcs/039_pandas_familiar_exploration_api.md +++ b/docs/rfcs/039_pandas_familiar_exploration_api.md @@ -1,47 +1,47 @@ -# InQL RFC 039: Pandas-familiar exploration API +# IncQL RFC 039: Pandas-familiar exploration API - **Status:** Draft - **Created:** 2026-05-30 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 000 (language specification, naming, schema shapes, and relational positions) - - InQL RFC 001 (dataset carriers and method-chain API surface) - - InQL RFC 003 (`query {}` blocks and relational authoring) - - InQL RFC 005 (pipe-forward relational syntax) - - InQL RFC 012 (unified scalar expression surface) -- **Issue:** [InQL #73](https://github.com/encero-systems/InQL/issues/73) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 000 (language specification, naming, schema shapes, and relational positions) + - IncQL RFC 001 (dataset carriers and method-chain API surface) + - IncQL RFC 003 (`query {}` blocks and relational authoring) + - IncQL RFC 005 (pipe-forward relational syntax) + - IncQL RFC 012 (unified scalar expression surface) +- **Issue:** [IncQL #73](https://github.com/encero-systems/IncQL/issues/73) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines a pandas-familiar exploration API for InQL dataset carriers. The API provides dictionary-like column access through `data["column"]`, projection through `data[["a", "b"]]`, boolean filtering through `data[predicate]`, and a small set of familiar method aliases such as `where`, `assign`, `groupby`, `sort_values`, and `head`. These forms are ergonomic aliases over InQL's existing typed relational model; they must not introduce pandas row-indexing, mutable frame, eager execution, or index-alignment semantics. +This RFC defines a pandas-familiar exploration API for IncQL dataset carriers. The API provides dictionary-like column access through `data["column"]`, projection through `data[["a", "b"]]`, boolean filtering through `data[predicate]`, and a small set of familiar method aliases such as `where`, `assign`, `groupby`, `sort_values`, and `head`. These forms are ergonomic aliases over IncQL's existing typed relational model; they must not introduce pandas row-indexing, mutable frame, eager execution, or index-alignment semantics. ## Core model -1. InQL dataset carriers may behave dictionary-like for columns. -2. InQL dataset carriers must not behave sequence-like for rows unless a future RFC defines row-position semantics explicitly. +1. IncQL dataset carriers may behave dictionary-like for columns. +2. IncQL dataset carriers must not behave sequence-like for rows unless a future RFC defines row-position semantics explicitly. 3. Bracket column access returns a bound scalar column expression, not a materialized Series-like value. 4. Bracket projection and bracket filtering lower to the same relational operators as `DataSet[T]` method chains and `query {}` blocks. -5. Pandas-familiar method names are aliases over existing InQL relational operations, not alternate execution contracts. +5. Pandas-familiar method names are aliases over existing IncQL relational operations, not alternate execution contracts. ## Motivation -InQL's cleaner relational APIs are good for production authoring, but data exploration has a different adoption problem. Authors coming from pandas need a familiar fallback surface for the workflows they reach for reflexively: selecting a column, filtering a frame, projecting a few columns, adding a derived column, grouping, sorting, and previewing rows. If InQL only exposes its cleanest API, it forces those users to learn InQL before they can inspect data, which is a poor fit for exploratory work. +IncQL's cleaner relational APIs are good for production authoring, but data exploration has a different adoption problem. Authors coming from pandas need a familiar fallback surface for the workflows they reach for reflexively: selecting a column, filtering a frame, projecting a few columns, adding a derived column, grouping, sorting, and previewing rows. If IncQL only exposes its cleanest API, it forces those users to learn IncQL before they can inspect data, which is a poor fit for exploratory work. -At the same time, copying pandas wholesale would be a design error. Pandas carries semantics that do not fit InQL's typed, planned, backend-neutral model: mutable frames, eager local execution, row-position indexing, index alignment, view-versus-copy behavior, dynamic dtype coercion, and a Series object model with local row values. Those features are familiar, but they would make InQL less coherent if imported as language semantics. +At the same time, copying pandas wholesale would be a design error. Pandas carries semantics that do not fit IncQL's typed, planned, backend-neutral model: mutable frames, eager local execution, row-position indexing, index alignment, view-versus-copy behavior, dynamic dtype coercion, and a Series object model with local row values. Those features are familiar, but they would make IncQL less coherent if imported as language semantics. -This RFC takes the narrower path: provide familiar surface forms where they map cleanly to InQL's relational model, and reject the parts of pandas that would require a different data model. +This RFC takes the narrower path: provide familiar surface forms where they map cleanly to IncQL's relational model, and reject the parts of pandas that would require a different data model. ## Goals - Define bracket column access on `DataSet[T]` carriers through `data["column"]`. - Define bracket projection through `data[["a", "b"]]`. - Define bracket filtering through `data[predicate]` where `predicate` is a boolean scalar expression bound to the same dataset relation. -- Define pandas-familiar aliases for common method-chain operations where semantics already exist in InQL. -- Preserve InQL's typed schema flow, boundedness constraints, scalar expression model, Prism planning, Substrait lowering, and execution boundaries. -- Require clear diagnostics for row-indexing spellings that pandas users might try but InQL does not support. +- Define pandas-familiar aliases for common method-chain operations where semantics already exist in IncQL. +- Preserve IncQL's typed schema flow, boundedness constraints, scalar expression model, Prism planning, Substrait lowering, and execution boundaries. +- Require clear diagnostics for row-indexing spellings that pandas users might try but IncQL does not support. ## Non-Goals @@ -52,7 +52,7 @@ This RFC takes the narrower path: provide familiar surface forms where they map - Defining pandas view-versus-copy behavior. - Making bracket access materialize local row values. - Defining all exploratory helpers such as `describe`, `sample`, rich display, plotting, or notebook integration. -- Replacing the cleaner InQL `DataSet[T]` method surface or `query {}` blocks. +- Replacing the cleaner IncQL `DataSet[T]` method surface or `query {}` blocks. ## Guide-level explanation (how authors think about it) @@ -70,7 +70,7 @@ Authors can project columns with a list of names: order_amounts = orders[["order_id", "customer_id", "amount"]] ``` -This is a projection over the same relation. The output preserves column order from the list and follows the same schema rules as the corresponding InQL projection operation. +This is a projection over the same relation. The output preserves column order from the list and follows the same schema rules as the corresponding IncQL projection operation. Authors can use familiar method aliases for common exploration flows: @@ -90,7 +90,7 @@ summary = ( ) ``` -Those names are familiar, but the semantics are still InQL. `where` is `filter`, `assign` is derived-column projection, `groupby` is `group_by`, `sort_values` is `order_by`, and `head` is `limit`. They do not execute eagerly unless the surrounding execution context materializes the result. +Those names are familiar, but the semantics are still IncQL. `where` is `filter`, `assign` is derived-column projection, `groupby` is `group_by`, `sort_values` is `order_by`, and `head` is `limit`. They do not execute eagerly unless the surrounding execution context materializes the result. Row indexing is intentionally rejected: @@ -106,15 +106,15 @@ Authors should use `head(n)` or `limit(n)` for preview-shaped relational limitin ### Applicability -The pandas-familiar exploration API applies to values whose type conforms to `DataSet[T]` as defined by InQL RFC 001. The API must preserve the concrete carrier kind where the equivalent relational operation would preserve it. For example, filtering a `LazyFrame[T]` produces a `LazyFrame[T]`; filtering a `DataStream[T]` must follow the same boundedness and capability constraints as `filter(...)` on `DataStream[T]`. +The pandas-familiar exploration API applies to values whose type conforms to `DataSet[T]` as defined by IncQL RFC 001. The API must preserve the concrete carrier kind where the equivalent relational operation would preserve it. For example, filtering a `LazyFrame[T]` produces a `LazyFrame[T]`; filtering a `DataStream[T]` must follow the same boundedness and capability constraints as `filter(...)` on `DataStream[T]`. ### Bracket column access For a dataset value `data: DataSet[T]`, `data["name"]` denotes a scalar column expression bound to `data` and the field named `name`. -The key expression in the strongly typed form must be a string literal or another compile-time-known string value whose value can be checked against the schema. If `T` is a closed local model, the named field must exist or compilation must fail. If `T` is open-ended or dynamic, field lookup must follow the schema-shape rules from InQL RFC 000, including warnings for undeclared open-ended fields where that RFC requires them. +The key expression in the strongly typed form must be a string literal or another compile-time-known string value whose value can be checked against the schema. If `T` is a closed local model, the named field must exist or compilation must fail. If `T` is open-ended or dynamic, field lookup must follow the schema-shape rules from IncQL RFC 000, including warnings for undeclared open-ended fields where that RFC requires them. -`data["name"]` must not materialize a local column, must not produce a Series-like object, and must not imply that `data` is locally available. It is a typed relational scalar expression that may be used in relational expression positions and in ordinary Incan expression positions that consume InQL scalar expressions. +`data["name"]` must not materialize a local column, must not produce a Series-like object, and must not imply that `data` is locally available. It is a typed relational scalar expression that may be used in relational expression positions and in ordinary Incan expression positions that consume IncQL scalar expressions. Bound column expressions must preserve relation provenance. A predicate built from `orders["amount"]` may filter `orders`, but must not be accepted as a filter predicate for an unrelated dataset unless an explicit relational operation such as a join establishes the appropriate relation context. @@ -124,7 +124,7 @@ For a dataset value `data: DataSet[T]`, `data[["a", "b"]]` denotes an ordered pr The projection list in the strongly typed form must be a list literal or another compile-time-known list of string column names. Each projected name must be checked using the same schema-shape rules as bracket column access. The output schema must contain the projected columns in the order supplied by the list. -Duplicate projected names must be rejected unless a future RFC defines duplicate-column schema semantics. Pandas permits duplicate column labels, but InQL's typed relation model requires deterministic field names. +Duplicate projected names must be rejected unless a future RFC defines duplicate-column schema semantics. Pandas permits duplicate column labels, but IncQL's typed relation model requires deterministic field names. Bracket projection must lower to the same relational projection semantics as the corresponding `DataSet[T]` projection surface or `query { SELECT ... }` form. It must not materialize data. @@ -143,16 +143,16 @@ The following forms must produce compile-time diagnostics: - `data[0]` - `data[1:10]` - `data[-1]` -- `data[mask]` where `mask` is a materialized local boolean list or array rather than an InQL scalar expression +- `data[mask]` where `mask` is a materialized local boolean list or array rather than an IncQL scalar expression - any bracket form whose key cannot be resolved as a string column key, compile-time-known list of string column keys, or boolean scalar expression -Diagnostics should explain that InQL supports dictionary-like column access and relational filtering, not pandas row indexing. +Diagnostics should explain that IncQL supports dictionary-like column access and relational filtering, not pandas row indexing. ### Familiar method aliases The initial pandas-familiar method alias set is: -| Alias | Canonical InQL operation | Required semantics | +| Alias | Canonical IncQL operation | Required semantics | | ----- | ------------------------ | ------------------ | | `where(predicate)` | `filter(predicate)` | Filter rows using a boolean scalar expression. | | `assign(name, expr)` | `with_column(name, expr)` | Add or replace one derived column. | @@ -164,7 +164,7 @@ These aliases must not alter the plan produced by the canonical operation except ### Method alias arguments -String column names accepted by familiar aliases must resolve using the same schema-shape rules as bracket column access. Scalar expression arguments must use the unified scalar expression model from InQL RFC 012. Aggregate arguments used after `groupby(...)` must use the aggregate-measure rules defined by the relevant aggregate RFCs. +String column names accepted by familiar aliases must resolve using the same schema-shape rules as bracket column access. Scalar expression arguments must use the unified scalar expression model from IncQL RFC 012. Aggregate arguments used after `groupby(...)` must use the aggregate-measure rules defined by the relevant aggregate RFCs. `sort_values("amount")` must be equivalent to ordering by the `amount` column in ascending order. `sort_values("amount", ascending=false)` must be equivalent to ordering by the same column in descending order. Multi-column sort arguments may be supported through a compile-time-known list of names or ordering expressions, provided they lower to the same ordering model as `order_by(...)`. @@ -172,36 +172,36 @@ String column names accepted by familiar aliases must resolve using the same sch This RFC does not add new `query {}` clause syntax. Bracket access and familiar method aliases must remain semantically equivalent to `query {}` blocks that express the same relational operations. -If pipe-forward from InQL RFC 005 is implemented, it must not define different pandas-familiar semantics. A pipe-forward stage that corresponds to `where`, projection, grouping, sorting, or limiting must lower to the same relational model as bracket and method-chain forms. +If pipe-forward from IncQL RFC 005 is implemented, it must not define different pandas-familiar semantics. A pipe-forward stage that corresponds to `where`, projection, grouping, sorting, or limiting must lower to the same relational model as bracket and method-chain forms. ## Design details ### Syntax -This RFC requires the Incan language and InQL vocabulary integration to recognize bracket access on dataset carriers for the forms specified above. The bracket syntax is intentionally overloaded only by key shape: string key for column expression, list of string keys for projection, and boolean scalar expression for filtering. +This RFC requires the Incan language and IncQL vocabulary integration to recognize bracket access on dataset carriers for the forms specified above. The bracket syntax is intentionally overloaded only by key shape: string key for column expression, list of string keys for projection, and boolean scalar expression for filtering. The RFC does not require `.loc`, `.iloc`, attribute-style column access, or assignment syntax. ### Semantics -The semantic distinction is column dictionary access versus row sequence access. `DataSet[T]` carriers may be indexed by column name because the schema is part of the relational type. They may not be indexed by row position because row order is not an inherent property of an unordered relation and because InQL execution may be lazy, distributed, streamed, or backend-planned. +The semantic distinction is column dictionary access versus row sequence access. `DataSet[T]` carriers may be indexed by column name because the schema is part of the relational type. They may not be indexed by row position because row order is not an inherent property of an unordered relation and because IncQL execution may be lazy, distributed, streamed, or backend-planned. `head(n)` is a relational limit, not proof of stable row order. Authors who need deterministic preview order should combine `sort_values(...)` or `order_by(...)` with `head(...)`. ### Interaction with Incan `model` types and schema shapes -Closed model schemas provide the strongest checks for bracket column access and projection. Open-ended and dynamic schemas follow the existing InQL schema-shape contract: declared fields remain checked, while undeclared fields may be allowed with warnings or dynamic typing where InQL RFC 000 permits that behavior. +Closed model schemas provide the strongest checks for bracket column access and projection. Open-ended and dynamic schemas follow the existing IncQL schema-shape contract: declared fields remain checked, while undeclared fields may be allowed with warnings or dynamic typing where IncQL RFC 000 permits that behavior. ### Compatibility / migration This RFC is additive. Existing `DataSet[T]` method chains and `query {}` blocks remain valid and remain the canonical semantic reference. -The main compatibility risk is expectation compatibility, not source compatibility. Authors who expect full pandas behavior must receive clear diagnostics and documentation explaining which familiar forms InQL supports and which ones are deliberately absent. +The main compatibility risk is expectation compatibility, not source compatibility. Authors who expect full pandas behavior must receive clear diagnostics and documentation explaining which familiar forms IncQL supports and which ones are deliberately absent. ## Alternatives considered -- **Expose only the clean InQL API.** Rejected because exploration workflows need familiarity, and forcing pandas users through only the cleanest InQL surface raises the cost of first use. -- **Implement broad pandas compatibility.** Rejected because pandas row indexes, eager Series values, mutable assignment, and index alignment conflict with InQL's typed relational planning model. +- **Expose only the clean IncQL API.** Rejected because exploration workflows need familiarity, and forcing pandas users through only the cleanest IncQL surface raises the cost of first use. +- **Implement broad pandas compatibility.** Rejected because pandas row indexes, eager Series values, mutable assignment, and index alignment conflict with IncQL's typed relational planning model. - **Support `.loc` and `.iloc` initially.** Rejected because both imply an index or positional row model that this RFC intentionally excludes. - **Use attribute column access such as `data.amount`.** Rejected for the initial surface because it collides with ordinary methods and carrier properties more readily than string-key access. - **Make `data["name"]` return a Series-like value.** Rejected because it would imply local materialization and row-level value access instead of a backend-neutral scalar expression. @@ -211,15 +211,15 @@ The main compatibility risk is expectation compatibility, not source compatibili - Familiar names can create false expectations about pandas parity unless diagnostics and docs are explicit. - Bracket syntax adds an overloaded authoring surface that tooling must explain carefully. - Method aliases duplicate parts of the canonical API, increasing documentation and completion surface area. -- Rejecting row indexing is correct for InQL's model but will still surprise some pandas users. +- Rejecting row indexing is correct for IncQL's model but will still surprise some pandas users. - Compile-time-known projection lists may feel less flexible than pandas during ad hoc exploration, especially for dynamic column selection. ## Layers affected -- **InQL specification** — RFCs 000, 001, 003, 005, and 012 must stay coherent with bracket access, relation provenance, scalar expression typing, and alias semantics. -- **InQL library package** — public `.incn` APIs must expose the familiar aliases and tests must cover equivalence to canonical operations where those aliases are library-level methods. +- **IncQL specification** — RFCs 000, 001, 003, 005, and 012 must stay coherent with bracket access, relation provenance, scalar expression typing, and alias semantics. +- **IncQL library package** — public `.incn` APIs must expose the familiar aliases and tests must cover equivalence to canonical operations where those aliases are library-level methods. - **Incan compiler** — parser, typechecker, lowering, diagnostics, formatter, and LSP may need dataset-aware bracket handling and source spans for useful errors. -- **Execution / interchange** — Prism, Substrait lowering, sessions, and backend adapters must receive the same logical operators as canonical InQL operations; this RFC must not introduce a separate execution path. +- **Execution / interchange** — Prism, Substrait lowering, sessions, and backend adapters must receive the same logical operators as canonical IncQL operations; this RFC must not introduce a separate execution path. - **Documentation** — reference and explanation docs should present the pandas-familiar API as an exploration facade and document the rejected pandas semantics directly. ## Unresolved questions diff --git a/docs/rfcs/040_interoperability_semantic_profiles.md b/docs/rfcs/040_interoperability_semantic_profiles.md index b804404e..965dbb1a 100644 --- a/docs/rfcs/040_interoperability_semantic_profiles.md +++ b/docs/rfcs/040_interoperability_semantic_profiles.md @@ -1,62 +1,62 @@ -# InQL RFC 040: Interoperability semantic profiles +# IncQL RFC 040: Interoperability semantic profiles - **Status:** Draft - **Created:** 2026-05-30 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 000 (core language model and layer boundaries) - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 004 (execution context) - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 008 (optimizer boundary, statistics, cost-based optimization, and adaptive execution) - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 013 (function catalog program) - - InQL RFC 024 (function extension policy) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 038 (evidence exchange bridges) - - InQL RFC 041 (Prism plan ingress and external client frontends) -- **Issue:** [InQL #74](https://github.com/encero-systems/InQL/issues/74) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 000 (core language model and layer boundaries) + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 004 (execution context) + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 008 (optimizer boundary, statistics, cost-based optimization, and adaptive execution) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 024 (function extension policy) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 038 (evidence exchange bridges) + - IncQL RFC 041 (Prism plan ingress and external client frontends) +- **Issue:** [IncQL #74](https://github.com/encero-systems/IncQL/issues/74) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines interoperability semantic profiles for InQL evidence. A profile describes the semantic environment a plan is being received from, compared with, targeted at, or observed under: an InQL baseline, client protocol, plan ingress frontend, execution engine, adapter binding, SQL dialect, catalog/schema system, transformation project, interchange consumer, or conformance baseline. Profiles give ingress coverage records, adapter requirements, coverage records, execution observations, plan diffs, bundles, and exchanges a shared context without making any external system the owner of InQL relational meaning. +This RFC defines interoperability semantic profiles for IncQL evidence. A profile describes the semantic environment a plan is being received from, compared with, targeted at, or observed under: an IncQL baseline, client protocol, plan ingress frontend, execution engine, adapter binding, SQL dialect, catalog/schema system, transformation project, interchange consumer, or conformance baseline. Profiles give ingress coverage records, adapter requirements, coverage records, execution observations, plan diffs, bundles, and exchanges a shared context without making any external system the owner of IncQL relational meaning. ## Motivation Interoperability requires more than lowering a plan and asking whether an adapter has a support flag. Different target environments can share the same relational vocabulary while differing on edge semantics: type coercion, decimal overflow, timestamp and timezone behavior, identifier resolution, null and NaN ordering, collation, case sensitivity, function definitions, aggregate edge cases, window defaults, nested data behavior, row ordering, and fallback execution. -If InQL does not name the semantic profile used for an inspection or execution, those assumptions will be scattered across adapters, Substrait metadata, docs, and runtime diagnostics. That would make coverage hard to trust. A plan could appear portable while relying on target-specific behavior that was never recorded as evidence. +If IncQL does not name the semantic profile used for an inspection or execution, those assumptions will be scattered across adapters, Substrait metadata, docs, and runtime diagnostics. That would make coverage hard to trust. A plan could appear portable while relying on target-specific behavior that was never recorded as evidence. -Profiles provide the missing layer between InQL-authored semantics, plan ingress, and adapter coverage. Prism remains the source of authored and rewritten relational meaning. Profiles describe source and target environments well enough for InQL to produce ingress diagnostics, requirements, coverage records, and observations against them. +Profiles provide the missing layer between IncQL-authored semantics, plan ingress, and adapter coverage. Prism remains the source of authored and rewritten relational meaning. Profiles describe source and target environments well enough for IncQL to produce ingress diagnostics, requirements, coverage records, and observations against them. -Profiles are intentionally ecosystem-neutral, but concrete profiles may describe systems and formats such as Oracle, PostgreSQL, SQL Server, MySQL, Athena, Presto, Trino, Spark, Snowflake, BigQuery, Redshift, Databricks, Glue Data Catalog, Hive Metastore, dbt, Airflow, MWAA, Dagster, Prefect, OpenLineage, DataHub, OpenMetadata, or Great Expectations. Listing a system as a possible profile target does not make that system normative for InQL semantics. +Profiles are intentionally ecosystem-neutral, but concrete profiles may describe systems and formats such as Oracle, PostgreSQL, SQL Server, MySQL, Athena, Presto, Trino, Spark, Snowflake, BigQuery, Redshift, Databricks, Glue Data Catalog, Hive Metastore, dbt, Airflow, MWAA, Dagster, Prefect, OpenLineage, DataHub, OpenMetadata, or Great Expectations. Listing a system as a possible profile target does not make that system normative for IncQL semantics. ## Goals - Define semantic profiles as versioned evidence records. -- Allow profiles for InQL baselines, client protocols, plan ingress frontends, execution engines, adapter bindings, SQL dialects, catalog/schema systems, transformation projects, interchange consumers, and conformance baselines. +- Allow profiles for IncQL baselines, client protocols, plan ingress frontends, execution engines, adapter bindings, SQL dialects, catalog/schema systems, transformation projects, interchange consumers, and conformance baselines. - Name the semantic dimensions that affect relational correctness and evidence interpretation. - Let adapter requirements and coverage records state which profile they were evaluated against. - Let execution observations report the profile requested before execution and the profile observed at runtime when available. - Keep profiles local and open, without requiring a hosted registry or managed control plane. -- Keep external target profiles non-authoritative for InQL semantics. +- Keep external target profiles non-authoritative for IncQL semantics. ## Non-Goals - Defining a profile for one specific external engine. -- Making any external engine, SQL dialect, or interchange format the normative InQL semantic model. +- Making any external engine, SQL dialect, or interchange format the normative IncQL semantic model. - Defining SQL transpilation, physical planning, or backend execution strategies. -- Defining transformation-project semantics as InQL semantics. +- Defining transformation-project semantics as IncQL semantics. - Defining a full conformance test suite. - Defining a global registry of every engine version or deployment configuration. - Guaranteeing semantic equivalence merely because a profile name is present. @@ -66,7 +66,7 @@ Profiles are intentionally ecosystem-neutral, but concrete profiles may describe Most authors should encounter profiles through inspection, coverage, and execution evidence: ```incan -from pub::inql.inspect import inspect_plan +from pub::incql.inspect import inspect_plan inspection = inspect_plan(summary) profile = inspection.semantic_profile("portable_relational") @@ -90,7 +90,7 @@ If the runtime adapter reports a different engine version, configuration, or sem ## Reference-level explanation (precise rules) -InQL must define an interoperability semantic profile record. A profile record must include: +IncQL must define an interoperability semantic profile record. A profile record must include: - profile identity - profile schema version @@ -106,7 +106,7 @@ InQL must define an interoperability semantic profile record. A profile record m Target class must distinguish at least: -- inql_baseline +- incql_baseline - client_protocol - plan_ingress_frontend - execution_engine @@ -138,16 +138,16 @@ Semantic dimensions must be represented as structured records rather than free-f - extension and fallback behavior - plan-stage observability -A semantic dimension record must include dimension identity, lifecycle, declared behavior when known, source, evidence references, confidence, and diagnostics. A dimension may be exact, constrained, unknown, or not_applicable. Unknown dimensions must not be treated as matching InQL semantics. +A semantic dimension record must include dimension identity, lifecycle, declared behavior when known, source, evidence references, confidence, and diagnostics. A dimension may be exact, constrained, unknown, or not_applicable. Unknown dimensions must not be treated as matching IncQL semantics. -InQL may define profile assessments that compare a plan or bundle with a profile. A profile assessment must include the plan target, profile identity, affected semantic targets, assessed dimensions, result state, evidence references, confidence, and diagnostics. +IncQL may define profile assessments that compare a plan or bundle with a profile. A profile assessment must include the plan target, profile identity, affected semantic targets, assessed dimensions, result state, evidence references, confidence, and diagnostics. Profile assessment result state must distinguish at least: -- matched: InQL can determine that the plan's required semantics match the profile for the assessed dimension +- matched: IncQL can determine that the plan's required semantics match the profile for the assessed dimension - constrained: the profile can satisfy the dimension only under recorded restrictions - mismatched: the profile does not satisfy the plan's required semantics for the dimension -- unknown: InQL cannot determine whether the profile satisfies the dimension +- unknown: IncQL cannot determine whether the profile satisfies the dimension - not_applicable: the dimension does not apply to the plan or target profile Adapter requirements and coverage records may cite profile records and profile assessments. If coverage depends on a profile, the coverage record must identify the profile. Coverage evaluated under one profile must not be reused under a different profile unless the evidence proves that the relevant semantic dimensions are equivalent. @@ -166,11 +166,11 @@ This RFC introduces no authoring syntax. ### Semantics -Semantic profiles are evidence contexts. They describe the target environment against which InQL evidence is checked, exported, or observed. They do not define InQL relational meaning. +Semantic profiles are evidence contexts. They describe the target environment against which IncQL evidence is checked, exported, or observed. They do not define IncQL relational meaning. -Profiles may be authored, built into InQL, imported from artifacts, produced by adapters, or observed during execution. The source and lifecycle must be recorded so tools can distinguish a trusted built-in profile from an adapter-reported runtime observation or an imported profile. +Profiles may be authored, built into IncQL, imported from artifacts, produced by adapters, or observed during execution. The source and lifecycle must be recorded so tools can distinguish a trusted built-in profile from an adapter-reported runtime observation or an imported profile. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Prism remains the source of authored and rewritten relational meaning. Profile assessments consume Prism targets, lineage, schema flow, function registry facts, ingress coverage records, and adapter requirements. They must not infer semantic structure from backend plan strings or external client protocol node identifiers. @@ -195,29 +195,29 @@ Profile schemas must be versioned from the start. Profile names that appear in s ## Alternatives considered - **Use adapter support flags only.** Rejected because support depends on target semantics, engine version, configuration, and execution mode. -- **Use Substrait as the profile model.** Rejected because Substrait is an interchange boundary and does not capture every InQL evidence dimension. -- **Make one external engine profile normative.** Rejected because InQL needs to interoperate with multiple targets without importing one target's semantics as the language definition. +- **Use Substrait as the profile model.** Rejected because Substrait is an interchange boundary and does not capture every IncQL evidence dimension. +- **Make one external engine profile normative.** Rejected because IncQL needs to interoperate with multiple targets without importing one target's semantics as the language definition. - **Rely only on conformance tests.** Rejected because tests are valuable evidence but do not replace structured profile records, coverage states, or diagnostics. - **Leave profiles to downstream integrations.** Rejected because independent profile reconstruction would cause drift across adapters, CI, notebooks, agents, transformation projects, and governance exchanges. ## Drawbacks - Profiles add another evidence concept that must stay distinct from requirements and coverage. -- Profile dimension vocabulary will require maintenance as InQL and target environments grow. +- Profile dimension vocabulary will require maintenance as IncQL and target environments grow. - Early profiles may contain many unknown dimensions, which can make reports feel conservative. - Runtime-observed profiles can differ from requested profiles, requiring clear diagnostics. ## Layers affected -- **InQL specification** — semantic profile records, dimensions, and assessment states become part of the relational evidence vocabulary. -- **InQL library package** — inspection, coverage, bundle, and export APIs must be able to expose profile records when available. -- **Execution / interchange** — sessions and adapters may report requested and observed profile evidence without owning InQL semantics. +- **IncQL specification** — semantic profile records, dimensions, and assessment states become part of the relational evidence vocabulary. +- **IncQL library package** — inspection, coverage, bundle, and export APIs must be able to expose profile records when available. +- **Execution / interchange** — sessions and adapters may report requested and observed profile evidence without owning IncQL semantics. - **Documentation** — docs must explain profiles as evidence contexts, not as alternative semantic authorities. ## Unresolved questions - Which semantic dimensions are mandatory in the first implementation? -- Should built-in InQL profiles live in core or in optional integration packages? +- Should built-in IncQL profiles live in core or in optional integration packages? - How should profile records compare target configurations without leaking sensitive deployment details? - Should conformance test results become profile evidence in this RFC or a later RFC? - Which transformation-project profile dimensions are needed before exchange bridges can safely emit test and metadata suggestions? diff --git a/docs/rfcs/041_prism_plan_ingress_frontends.md b/docs/rfcs/041_prism_plan_ingress_frontends.md index 29dbed59..d17b175f 100644 --- a/docs/rfcs/041_prism_plan_ingress_frontends.md +++ b/docs/rfcs/041_prism_plan_ingress_frontends.md @@ -1,41 +1,41 @@ -# InQL RFC 041: Prism plan ingress and external client frontends +# IncQL RFC 041: Prism plan ingress and external client frontends - **Status:** Draft - **Created:** 2026-05-30 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 000 (core language model and layer boundaries) - - InQL RFC 004 (execution context) - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 012 (unified scalar expression surface) - - InQL RFC 013 (function catalog program) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 040 (interoperability semantic profiles) -- **Issue:** [InQL #75](https://github.com/encero-systems/InQL/issues/75) -- **RFC PR:** [InQL #60](https://github.com/encero-systems/InQL/pull/60) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 000 (core language model and layer boundaries) + - IncQL RFC 004 (execution context) + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 012 (unified scalar expression surface) + - IncQL RFC 013 (function catalog program) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 040 (interoperability semantic profiles) +- **Issue:** [IncQL #75](https://github.com/encero-systems/IncQL/issues/75) +- **RFC PR:** [IncQL #60](https://github.com/encero-systems/IncQL/pull/60) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines Prism plan ingress and external client frontends for InQL. A frontend receives an external authoring or client protocol such as Spark Connect, SQL, or another unresolved relational plan surface, decodes it into a Prism-owned unresolved ingress plan, and asks Prism to analyze that plan into ordinary InQL relational semantics. The frontend may preserve client-origin evidence, client-session evidence, protocol diagnostics, and ingress coverage records, but it must not make the external protocol, Spark, Substrait, DataFusion, or any backend adapter the semantic owner of the plan. +This RFC defines Prism plan ingress and external client frontends for IncQL. A frontend receives an external authoring or client protocol such as Spark Connect, SQL, or another unresolved relational plan surface, decodes it into a Prism-owned unresolved ingress plan, and asks Prism to analyze that plan into ordinary IncQL relational semantics. The frontend may preserve client-origin evidence, client-session evidence, protocol diagnostics, and ingress coverage records, but it must not make the external protocol, Spark, Substrait, DataFusion, or any backend adapter the semantic owner of the plan. ## Motivation -InQL should be able to interoperate with established client APIs without pretending that those APIs own InQL's semantics. Spark Connect is the clearest pressure: a PySpark client can submit plan-shaped calls over a protocol boundary, and those calls may depend on client session state such as configuration, current catalog, temporary views, or function aliases. InQL should not route those calls through Spark just to recover meaning later. Prism should receive an unresolved representation, resolve names and functions, apply InQL semantic rules under an explicit profile and session context, and then continue through the normal planning, evidence, Substrait, and execution paths. +IncQL should be able to interoperate with established client APIs without pretending that those APIs own IncQL's semantics. Spark Connect is the clearest pressure: a PySpark client can submit plan-shaped calls over a protocol boundary, and those calls may depend on client session state such as configuration, current catalog, temporary views, or function aliases. IncQL should not route those calls through Spark just to recover meaning later. Prism should receive an unresolved representation, resolve names and functions, apply IncQL semantic rules under an explicit profile and session context, and then continue through the normal planning, evidence, Substrait, and execution paths. -Without a first-class ingress contract, external API support will be squeezed into session adapters, backend adapters, Substrait metadata, or compatibility flags. That would hide the real boundary. Execution adapters run plans. Plan ingress frontends receive external plan requests and let Prism create InQL plans. +Without a first-class ingress contract, external API support will be squeezed into session adapters, backend adapters, Substrait metadata, or compatibility flags. That would hide the real boundary. Execution adapters run plans. Plan ingress frontends receive external plan requests and let Prism create IncQL plans. ## Goals - Define plan ingress frontends as distinct from execution session adapters. - Define a Prism-owned unresolved ingress plan model for external client plans. -- Preserve client-origin evidence without treating external node identifiers as InQL semantic identities. +- Preserve client-origin evidence without treating external node identifiers as IncQL semantic identities. - Represent client session state that can affect Prism analysis. - Allow Spark Connect-style clients to submit supported relation, expression, and command calls that Prism analyzes directly. - Require structured unsupported-feature diagnostics and ingress coverage records. @@ -46,31 +46,31 @@ Without a first-class ingress contract, external API support will be squeezed in - Defining full Spark API parity. - Defining every Spark Connect protobuf message, gRPC service method, or streaming transport detail. -- Making Spark, PySpark, Spark Connect, DataFusion, or Substrait the source of InQL relational meaning. +- Making Spark, PySpark, Spark Connect, DataFusion, or Substrait the source of IncQL relational meaning. - Defining SQL transpilation as the internal planning model. - Reimplementing every external client session lifecycle rule. - Defining a hosted compatibility service or global conformance registry. -- Requiring every InQL deployment to expose an external client protocol. +- Requiring every IncQL deployment to expose an external client protocol. ## Guide-level explanation (how authors think about it) -An external client frontend lets an existing tool submit relational intent while InQL keeps Prism as the planner: +An external client frontend lets an existing tool submit relational intent while IncQL keeps Prism as the planner: ```incan -from pub::inql.frontends.spark import spark_connect_frontend -from pub::inql.session import datafusion_session +from pub::incql.frontends.spark import spark_connect_frontend +from pub::incql.session import datafusion_session frontend = spark_connect_frontend(session_factory=datafusion_session) frontend.serve("127.0.0.1:15002") ``` -A PySpark client may send a supported Spark Connect plan to that endpoint. The frontend decodes the request into a Prism ingress plan, Prism analyzes it, and the selected InQL session executes the resulting plan through the normal adapter path. +A PySpark client may send a supported Spark Connect plan to that endpoint. The frontend decodes the request into a Prism ingress plan, Prism analyzes it, and the selected IncQL session executes the resulting plan through the normal adapter path. -The important user model is not that InQL becomes Spark internally. The model is that InQL can speak a client protocol at the edge while keeping Prism responsible for names, functions, types, lineage, diagnostics, evidence, and execution requirements. +The important user model is not that IncQL becomes Spark internally. The model is that IncQL can speak a client protocol at the edge while keeping Prism responsible for names, functions, types, lineage, diagnostics, evidence, and execution requirements. ## Reference-level explanation (precise rules) -InQL must define a plan ingress frontend boundary. A frontend may parse, decode, authenticate, route, frame client requests, and maintain client session state. It must not resolve relational semantics by delegating to an external engine when Prism can represent the requested plan. +IncQL must define a plan ingress frontend boundary. A frontend may parse, decode, authenticate, route, frame client requests, and maintain client session state. It must not resolve relational semantics by delegating to an external engine when Prism can represent the requested plan. An ingress frontend must produce a Prism-owned unresolved ingress plan or a structured unsupported diagnostic. An unresolved ingress plan must represent at least: @@ -97,7 +97,7 @@ Ingress frontends must distinguish at least the following coverage states for pr - supported: Prism can represent and analyze the feature under the selected profile - partially_supported: Prism can represent the feature only under recorded restrictions - unsupported: Prism cannot represent or analyze the feature -- unknown: InQL cannot determine support for the feature +- unknown: IncQL cannot determine support for the feature Unsupported and unknown ingress features must be reported before execution when they affect plan semantics. They must not be silently lowered to backend-specific behavior. @@ -105,7 +105,7 @@ Spark Connect compatibility must be modeled as a frontend protocol profile plus Semantic profiles may affect ingress analysis. Profile dimensions may control identifier resolution, case sensitivity, function aliases, type coercion, null and NaN behavior, timestamp semantics, decimal behavior, ANSI mode, and other compatibility-sensitive rules. Profile evidence must be explicit when the frontend uses those rules. -Commands that do not describe relational computation, such as session configuration, catalog inspection, temporary view registration, cache control, or client lifecycle operations, must be represented as command ingress nodes, mapped to explicit InQL client session or execution session behavior, or rejected with structured diagnostics. Accepted commands that mutate client session state must produce client session evidence. They must not be disguised as ordinary relational plan nodes. +Commands that do not describe relational computation, such as session configuration, catalog inspection, temporary view registration, cache control, or client lifecycle operations, must be represented as command ingress nodes, mapped to explicit IncQL client session or execution session behavior, or rejected with structured diagnostics. Accepted commands that mutate client session state must produce client session evidence. They must not be disguised as ordinary relational plan nodes. Plan ingress evidence must be inspectable. Tools must be able to see which frontend produced a plan, which client session context affected analysis, which client protocol features were used, which features were unsupported or partially supported, which profile governed analysis, and how client-origin references map to Prism targets. @@ -113,7 +113,7 @@ Plan ingress evidence must be inspectable. Tools must be able to see which front ### Syntax -This RFC introduces no InQL authoring syntax. Frontends are package or service APIs around Prism and Session. +This RFC introduces no IncQL authoring syntax. Frontends are package or service APIs around Prism and Session. ### Semantics @@ -121,7 +121,7 @@ Plan ingress is a semantic boundary before Prism analysis. It is not an executio The unresolved ingress model should be general enough for Spark Connect, SQL parsers, notebook clients, and future external plan protocols, but each frontend must declare its own protocol coverage, session-state model, and profile assumptions. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Method chains, `query {}` blocks, SQL frontends, and Spark Connect frontends may all produce Prism plans. Equivalent relational intent should converge on comparable Prism targets after analysis, subject to explicit profile differences. @@ -133,14 +133,14 @@ Adapter requirements and coverage remain execution-facing evidence. Ingress cove ### Compatibility / migration -Existing InQL plans and sessions remain valid without ingress evidence. Frontends are additive. Tools that require client-origin evidence must report missing ingress evidence as unsupported or unknown rather than inferring it from display names or backend plans. +Existing IncQL plans and sessions remain valid without ingress evidence. Frontends are additive. Tools that require client-origin evidence must report missing ingress evidence as unsupported or unknown rather than inferring it from display names or backend plans. ## Alternatives considered - **Use Spark as the planner.** Rejected because that would make Spark the semantic owner and reduce Prism to an execution bridge. -- **Translate Spark Connect directly to Substrait.** Rejected because Substrait is an interchange boundary, not the full InQL semantic analysis and evidence model. +- **Translate Spark Connect directly to Substrait.** Rejected because Substrait is an interchange boundary, not the full IncQL semantic analysis and evidence model. - **Treat Spark Connect support as a backend adapter.** Rejected because receiving client plan calls and executing an analyzed plan are different boundaries. -- **Only support InQL-native authoring.** Rejected because external client protocols are valuable when they feed Prism honestly instead of bypassing it. +- **Only support IncQL-native authoring.** Rejected because external client protocols are valuable when they feed Prism honestly instead of bypassing it. - **Accept unsupported calls and hope the backend handles them.** Rejected because unknown frontend semantics must be visible before execution. ## Drawbacks @@ -152,8 +152,8 @@ Existing InQL plans and sessions remain valid without ingress evidence. Frontend ## Layers affected -- **InQL specification** — plan ingress, frontend coverage, origin mapping, and Prism analysis boundaries become normative vocabulary. -- **InQL library package** — frontend APIs, unresolved ingress plan records, diagnostics, and inspection records must be exposed through public modules where implemented. +- **IncQL specification** — plan ingress, frontend coverage, origin mapping, and Prism analysis boundaries become normative vocabulary. +- **IncQL library package** — frontend APIs, unresolved ingress plan records, diagnostics, and inspection records must be exposed through public modules where implemented. - **Execution / interchange** — Session and backend adapters execute Prism-owned plans and may report adapter coverage, but they do not own ingress semantics. - **Documentation** — docs must distinguish external client protocol support from Spark engine compatibility, Substrait interchange, and DataFusion execution. diff --git a/docs/rfcs/042_async_verification_evidence.md b/docs/rfcs/042_async_verification_evidence.md index e860ee06..444e64b4 100644 --- a/docs/rfcs/042_async_verification_evidence.md +++ b/docs/rfcs/042_async_verification_evidence.md @@ -1,30 +1,30 @@ -# InQL RFC 042: Async verification evidence +# IncQL RFC 042: Async verification evidence - **Status:** Draft - **Created:** 2026-06-20 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 034 (quality assertions and observations) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 038 (evidence exchange bridges) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 043 (canonical equality and digest profiles) - - InQL RFC 044 (verifier statements and proof artifacts) - - InQL RFC 045 (constraint evidence and verification-aware planning) -- **Issue:** [InQL #77](https://github.com/encero-systems/InQL/issues/77) -- **RFC PR:** [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 034 (quality assertions and observations) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 038 (evidence exchange bridges) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 043 (canonical equality and digest profiles) + - IncQL RFC 044 (verifier statements and proof artifacts) + - IncQL RFC 045 (constraint evidence and verification-aware planning) +- **Issue:** [IncQL #77](https://github.com/encero-systems/IncQL/issues/77) +- **RFC PR:** [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines async verification evidence for InQL. Verification assertions are stable semantic targets, verification runs emit append-only observations over time, and current verification state is a projection over those observations rather than a mutable field. The model separates lifecycle, outcome, assurance, scope, and commitment context so tools can distinguish deterministic verification, external attestations, sampled checks, accepted waivers, unknown evidence, and proof-backed verification without pretending that all checks carry the same trust. +This RFC defines async verification evidence for IncQL. Verification assertions are stable semantic targets, verification runs emit append-only observations over time, and current verification state is a projection over those observations rather than a mutable field. The model separates lifecycle, outcome, assurance, scope, and commitment context so tools can distinguish deterministic verification, external attestations, sampled checks, accepted waivers, unknown evidence, and proof-backed verification without pretending that all checks carry the same trust. ## Core model @@ -34,13 +34,13 @@ This RFC defines async verification evidence for InQL. Verification assertions a 4. A current verification state is a projection over observations for a specific assertion, scope, source snapshot, target snapshot, semantic profile context, and stream watermark. 5. Lifecycle, outcome, and assurance are separate axes. A check can be complete but failed, passed but only attested, or failed but waived. 6. Assurance is scoped. A relation may be sampled overall while individual partitions are verified, waived, unknown, or later proven. -7. Cryptographic proof verification plugs into the same event model through verifier statements and proof artifacts defined by InQL RFC 044. This RFC defines observation state, not proof-system mechanics. +7. Cryptographic proof verification plugs into the same event model through verifier statements and proof artifacts defined by IncQL RFC 044. This RFC defines observation state, not proof-system mechanics. 8. Verification is always relative to recorded snapshots, stream positions, commitments, or attestations. It does not prove that those references faithfully represent reality unless their own authority and basis are also verified. 9. Privacy is a separate evidence concern. Verification assurance must not imply zero-knowledge, confidentiality, or payload redaction. ## Motivation -Migration, modernization, replicated analytics, and cross-system validation work rarely complete as one atomic yes-or-no check. Source counts may be reported by a connector before target data finishes loading. Partition digests may stream in over minutes or hours. A sampled comparison may later be replaced by a deterministic check. A mismatch may be waived for a specific partition while the rest of the relation remains verified. A proof verifier may also prove a bounded query result against committed inputs through the statement and artifact model defined by InQL RFC 044. InQL needs a vocabulary that can represent those movements without overwriting older evidence or collapsing weak and strong evidence into the same status. +Migration, modernization, replicated analytics, and cross-system validation work rarely complete as one atomic yes-or-no check. Source counts may be reported by a connector before target data finishes loading. Partition digests may stream in over minutes or hours. A sampled comparison may later be replaced by a deterministic check. A mismatch may be waived for a specific partition while the rest of the relation remains verified. A proof verifier may also prove a bounded query result against committed inputs through the statement and artifact model defined by IncQL RFC 044. IncQL needs a vocabulary that can represent those movements without overwriting older evidence or collapsing weak and strong evidence into the same status. The existing evidence RFCs already define execution observations, quality observations, adapter coverage, governed bundles, exchange bridges, and semantic profiles. They do not yet define a cross-cutting assurance axis or an async verification event stream. Reusing only quality status would conflate "the predicate passed" with "the evidence is independently verified." Reusing only execution status would conflate "the job succeeded" with "the migration is correct." Reusing only adapter coverage would conflate "the adapter can perform a check" with "the check has actually been performed." @@ -52,7 +52,7 @@ The existing evidence RFCs already define execution observations, quality observ - Let verification evidence attach to semantic targets, snapshots, stream positions, watermarks, profiles, and evidence references. - Require snapshot and commitment context to record authority, time, and basis when those facts are available. - Define assurance labels for proven, verified, attested, sampled, waived, and unknown evidence. -- Define how proof-backed observations use the same lifecycle, outcome, assurance, scope, and commitment model while deferring statement and artifact details to InQL RFC 044. +- Define how proof-backed observations use the same lifecycle, outcome, assurance, scope, and commitment model while deferring statement and artifact details to IncQL RFC 044. - Require unsupported and omitted verification coverage to stay explicit in observations and projections. - Clarify how verification evidence composes with quality observations, execution observations, adapter coverage, governed plan bundles, evidence exchange bridges, and semantic profiles. @@ -169,7 +169,7 @@ Assurance label must distinguish at least: - proven: a cryptographic proof was verified against recorded commitments for the observation scope - verified: a deterministic check ran against recorded source and target snapshots or stream positions for the observation scope -- attested: a connector, external tool, catalog, or runtime reported a fact that InQL did not independently verify +- attested: a connector, external tool, catalog, or runtime reported a fact that IncQL did not independently verify - sampled: only part of the population was checked, or the check was deterministic only for a sampled subset - waived: a mismatch, missing check, or weaker evidence was explicitly accepted with a recorded reason - unknown: no usable evidence is available for the observation scope @@ -180,7 +180,7 @@ The `waived` assurance label must not be rendered as equivalent to `verified` or The `verified` assurance label requires recorded inputs. For relation, row, or partition digest checks, the observation must identify the digest algorithm and canonicalization profile used, or must report unknown or unsupported evidence rather than claiming deterministic verification. -The `proven` assurance label requires recorded public inputs sufficient for an independent verifier to understand the checked statement. At minimum, a proven observation must identify the proof system or verifier profile, source commitments, result commitments or result references, the verified statement identity, commitment authority, commitment basis, and verifier diagnostics. Statement and artifact details belong to InQL RFC 044; proof-system mathematics remain outside this RFC. +The `proven` assurance label requires recorded public inputs sufficient for an independent verifier to understand the checked statement. At minimum, a proven observation must identify the proof system or verifier profile, source commitments, result commitments or result references, the verified statement identity, commitment authority, commitment basis, and verifier diagnostics. Statement and artifact details belong to IncQL RFC 044; proof-system mathematics remain outside this RFC. Verification observations are append-only. An implementation must not update an old observation in place to change its outcome, assurance, scope, or evidence. Corrections, revocations, upgrades, downgrades, waivers, and finalizations must be represented as later observations with explicit supersedes or diagnostic references when they affect prior evidence. @@ -188,7 +188,7 @@ A current verification projection is derived evidence. It must identify the asse Projection rules must preserve weaker and stronger evidence separately. If a table has verified partitions, sampled partitions, waived partitions, and unknown partitions, the table-level projection should report mixed outcome or partial coverage rather than flattening the table to verified. Unsupported operators, omitted columns, omitted predicates, profile gaps, skipped partitions, and unknown digest semantics must remain visible as coverage diagnostics. -Verification evidence may reference quality observations when the checked condition is expressed as an InQL quality assertion. In that case, the quality observation status remains the predicate outcome, while the verification observation assurance records how the predicate result was established. +Verification evidence may reference quality observations when the checked condition is expressed as an IncQL quality assertion. In that case, the quality observation status remains the predicate outcome, while the verification observation assurance records how the predicate result was established. Verification evidence may reference execution observations when a verification run executes through a session or adapter. Execution success must not imply verification success, and verification success must not imply that every execution detail was covered. @@ -198,7 +198,7 @@ Verification evidence must carry semantic profile context when source or target Verification evidence must not imply privacy. If a verification process also provides payload redaction, confidentiality, encryption, or zero-knowledge properties, those properties must be represented as separate evidence or profile facts. -Inbound exchange bridges may import external verification facts, but imported facts are attested unless InQL can represent and validate the underlying evidence under this RFC. Outbound exchange bridges must preserve assertion identity, scope, outcome, assurance, coverage, and diagnostics when the target format can carry them, or must report mapping loss. +Inbound exchange bridges may import external verification facts, but imported facts are attested unless IncQL can represent and validate the underlying evidence under this RFC. Outbound exchange bridges must preserve assertion identity, scope, outcome, assurance, coverage, and diagnostics when the target format can carry them, or must report mapping loss. Governed plan bundles may include verification assertions, runs, observations, current projections, snapshot references, proof references, and waiver records. Bundles must distinguish unsupported verification evidence from empty verification evidence. @@ -214,7 +214,7 @@ Verification is evidence-producing relational work. It may compare source and ta Current state is not authoritative storage. It is a projection over append-only observations. Consumers that need auditability must retain the observation stream or a bundle that can identify the observations used to compute the projection. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces Verification assertions reuse semantic targets from RFC 028 and may use metadata attachments from RFC 029 for provenance, source, visibility, and evidence references. @@ -226,7 +226,7 @@ Verification checks may require RFC 033 adapter coverage. Unknown or uncovered v Verification evidence may be included in RFC 036 governed plan bundles as an evidence family with required, optional, unavailable, and unsupported section states. -RFC 038 evidence exchange bridges may import and export verification evidence. Bridge mappings must not silently upgrade attested external facts to verified or proven InQL evidence. +RFC 038 evidence exchange bridges may import and export verification evidence. Bridge mappings must not silently upgrade attested external facts to verified or proven IncQL evidence. RFC 040 semantic profiles provide source and target context for verification. Profile mismatches or unknown dimensions must be diagnostic evidence and may prevent verified or proven assurance. @@ -265,8 +265,8 @@ This section is non-normative. A practical implementation can store verification ## Layers affected -- **InQL specification** — verification assertion, run, observation, assurance, and projection vocabulary become part of the relational evidence program. -- **InQL library package** — inspection, quality, session, and bundle APIs should be able to expose verification assertions, observations, and projections. +- **IncQL specification** — verification assertion, run, observation, assurance, and projection vocabulary become part of the relational evidence program. +- **IncQL library package** — inspection, quality, session, and bundle APIs should be able to expose verification assertions, observations, and projections. - **Execution / interchange** — adapters may need capability records for snapshot capture, canonical digests, event streams, reconciliation checks, and proof verification. - **Documentation** — docs must explain the difference between lifecycle, outcome, assurance, coverage, and waived evidence. @@ -275,7 +275,7 @@ This section is non-normative. A practical implementation can store verification - Which verification helper APIs are normative rather than illustrative? - Which canonical row, partition, and relation digest profiles are required before deterministic verification can claim `verified` assurance? - What minimum waiver fields are required to keep `waived` useful without defining organization-wide approval policy? -- Which InQL RFC 044 proof artifact fields are required before an implementation may emit `proven` assurance? +- Which IncQL RFC 044 proof artifact fields are required before an implementation may emit `proven` assurance? - Which projection rule versions should this RFC standardize for relation-level rollups over partition, sample, and watermark observations? - Should a separate RFC define authoring syntax for verification blocks, or should verification remain an API and artifact surface? diff --git a/docs/rfcs/043_canonical_equality_digest_profiles.md b/docs/rfcs/043_canonical_equality_digest_profiles.md index a7035a48..c521aacf 100644 --- a/docs/rfcs/043_canonical_equality_digest_profiles.md +++ b/docs/rfcs/043_canonical_equality_digest_profiles.md @@ -1,25 +1,25 @@ -# InQL RFC 043: Canonical equality and digest profiles +# IncQL RFC 043: Canonical equality and digest profiles - **Status:** Draft - **Created:** 2026-06-20 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 034 (quality assertions and observations) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 042 (async verification evidence) -- **Issue:** [InQL #78](https://github.com/encero-systems/InQL/issues/78) -- **RFC PR:** [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 034 (quality assertions and observations) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 042 (async verification evidence) +- **Issue:** [IncQL #78](https://github.com/encero-systems/IncQL/issues/78) +- **RFC PR:** [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines canonical equality and digest profiles for InQL verification evidence. A profile records how rows, fields, relations, partitions, and result tables are normalized, ordered, compared, and hashed so deterministic verification can claim `verified` assurance without relying on hidden engine-specific equality rules. +This RFC defines canonical equality and digest profiles for IncQL verification evidence. A profile records how rows, fields, relations, partitions, and result tables are normalized, ordered, compared, and hashed so deterministic verification can claim `verified` assurance without relying on hidden engine-specific equality rules. ## Core model @@ -33,7 +33,7 @@ This RFC defines canonical equality and digest profiles for InQL verification ev Async verification can say that a partition, relation, or result was checked, but it cannot responsibly say that the check was deterministic unless both sides agree on what equality means. Real systems differ on decimal scale, timestamp precision, time zones, Unicode normalization, string collation, null ordering, NaN handling, duplicate rows, binary encoding, and nested value serialization. Without a canonical profile, a row digest can match or fail for reasons unrelated to the relational claim being verified. -The practical verification path needs digestible evidence even when no proof verifier is involved. Row counts, aggregate checks, per-partition digests, keyed row digests, and full relation digests are useful only if they carry enough information for another tool or an InQL RFC 044 verifier to understand the statement that was checked. +The practical verification path needs digestible evidence even when no proof verifier is involved. Row counts, aggregate checks, per-partition digests, keyed row digests, and full relation digests are useful only if they carry enough information for another tool or an IncQL RFC 044 verifier to understand the statement that was checked. ## Goals @@ -48,7 +48,7 @@ The practical verification path needs digestible evidence even when no proof ver - Defining every concrete canonicalization profile in this RFC. - Requiring one global canonical encoding for all engines and formats. -- Replacing InQL scalar, aggregate, ordering, or profile semantics. +- Replacing IncQL scalar, aggregate, ordering, or profile semantics. - Defining cryptographic proof systems or proof artifact formats. - Guaranteeing privacy, encryption, or zero-knowledge properties for digest evidence. @@ -147,11 +147,11 @@ This RFC introduces no authoring syntax. Helper APIs and artifact records are th ### Semantics -Canonical equality profiles define evidence comparison semantics. They do not change authored InQL relational semantics, query result semantics, or backend execution behavior. A profile explains how verification normalizes and compares observed values. +Canonical equality profiles define evidence comparison semantics. They do not change authored IncQL relational semantics, query result semantics, or backend execution behavior. A profile explains how verification normalizes and compares observed values. Digest profiles are downstream of equality profiles. A digest can only support a claim to the extent that the canonicalization and comparison rules support that claim. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces RFC 040 semantic profiles provide source and target context. Canonical equality profiles may use that context but must not replace it. @@ -165,7 +165,7 @@ RFC 034 quality assertions may be compared through canonical profiles when asser Canonical equality and digest profile artifacts should be bridgeable to dataset and quality metadata standards where the mapping is meaningful. W3C DCAT, DCAT-AP, schema.org Dataset, and Dublin Core can describe datasets, distributions, access metadata, licensing, and descriptive metadata around the relations being compared. W3C Data Quality Vocabulary can describe quality metrics and measurements that use canonical equality or digest results as evidence. -When a digest profile uses JSON or signed payloads, bridge profiles should use public canonicalization and signature specifications rather than ad hoc encodings. The digest algorithm, canonicalization profile, payload framing, and rollup rules must still be InQL evidence fields; an external serialization format alone is not a canonical equality profile. +When a digest profile uses JSON or signed payloads, bridge profiles should use public canonicalization and signature specifications rather than ad hoc encodings. The digest algorithm, canonicalization profile, payload framing, and rollup rules must still be IncQL evidence fields; an external serialization format alone is not a canonical equality profile. ### Compatibility / migration @@ -191,8 +191,8 @@ This section is non-normative. A practical implementation can support exact prof ## Layers affected -- **InQL specification** — canonical equality and digest profile vocabulary becomes part of verification evidence. -- **InQL library package** — verification helpers should be able to select and report canonical profiles. +- **IncQL specification** — canonical equality and digest profile vocabulary becomes part of verification evidence. +- **IncQL library package** — verification helpers should be able to select and report canonical profiles. - **Execution / interchange** — adapters may need capability records for canonical encoding and digest generation. - **Documentation** — docs must explain that digest equality depends on canonicalization and profile compatibility. diff --git a/docs/rfcs/044_verifier_statements_proof_artifacts.md b/docs/rfcs/044_verifier_statements_proof_artifacts.md index 8d17bd0b..6f62c629 100644 --- a/docs/rfcs/044_verifier_statements_proof_artifacts.md +++ b/docs/rfcs/044_verifier_statements_proof_artifacts.md @@ -1,30 +1,30 @@ -# InQL RFC 044: Verifier statements and proof artifacts +# IncQL RFC 044: Verifier statements and proof artifacts - **Status:** Draft - **Created:** 2026-06-20 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 042 (async verification evidence) - - InQL RFC 043 (canonical equality and digest profiles) -- **Issue:** [InQL #79](https://github.com/encero-systems/InQL/issues/79) -- **RFC PR:** [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 043 (canonical equality and digest profiles) +- **Issue:** [IncQL #79](https://github.com/encero-systems/IncQL/issues/79) +- **RFC PR:** [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines verifier statements and proof artifact records for InQL evidence. A verifier statement binds a semantic plan target, profile context, source commitments, result references, canonical equality rules, and verifier profile into a stable statement that deterministic checks and cryptographic proof systems can verify without relying on SQL text or backend-specific plan fragments as the source of meaning. +This RFC defines verifier statements and proof artifact records for IncQL evidence. A verifier statement binds a semantic plan target, profile context, source commitments, result references, canonical equality rules, and verifier profile into a stable statement that deterministic checks and cryptographic proof systems can verify without relying on SQL text or backend-specific plan fragments as the source of meaning. ## Core model 1. Verification statements are explicit public evidence records. -2. A statement binds to InQL semantic targets and profile context, not merely to a query string. +2. A statement binds to IncQL semantic targets and profile context, not merely to a query string. 3. Source commitments and result commitments are separate from the proof or evidence that verifies a statement. 4. A proof artifact records what verifier profile accepted, what public inputs were checked, and which scope the result covers. 5. `proven` assurance requires a successful proof verification observation, but this RFC does not define a proof system. @@ -33,7 +33,7 @@ This RFC defines verifier statements and proof artifact records for InQL evidenc Async verification needs a durable way to describe the thing being checked. A result can be checked by deterministic row digests or by a cryptographic proof, but both checks should point at the same kind of statement: the plan identity, semantic assumptions, commitments, result reference, and equality rules. Without a statement record, proof artifacts would be tied to whichever SQL string, backend plan, or connector payload happened to produce them. -Cryptographic query proof systems usually expose a compact verifier interface, but the surrounding application still needs to define which database commitment is trusted, which result is being opened, which query semantics apply, and which unsupported operators are outside the statement. InQL should own that statement layer while leaving concrete proof systems pluggable. +Cryptographic query proof systems usually expose a compact verifier interface, but the surrounding application still needs to define which database commitment is trusted, which result is being opened, which query semantics apply, and which unsupported operators are outside the statement. IncQL should own that statement layer while leaving concrete proof systems pluggable. ## Goals @@ -94,7 +94,7 @@ A verifier statement must include statement identity, statement schema version, Statement identity must be derived from normalized statement fields or otherwise be reproducible from the statement artifact. If the statement identity is assigned externally, the artifact must also carry a reproducible fingerprint. -The plan target must refer to InQL semantic evidence. SQL text, backend physical plans, serialized Substrait, or external client protocol nodes may be included as supporting artifacts, but they must not be the sole source of statement meaning. +The plan target must refer to IncQL semantic evidence. SQL text, backend physical plans, serialized Substrait, or external client protocol nodes may be included as supporting artifacts, but they must not be the sole source of statement meaning. A source commitment reference must include commitment identity, commitment authority when known, commitment time or observed time when known, capture basis, source scope, commitment algorithm or artifact type, evidence references, and diagnostics. Unknown authority or unknown capture basis must remain explicit. @@ -124,7 +124,7 @@ This RFC introduces no authoring syntax. Statement creation should be available Verifier statements describe checkable claims. They do not execute the plan, choose a backend, or define proof mathematics. A statement is valid only to the extent that its semantic targets, profiles, commitments, and result references are valid and in scope. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces RFC 028 provides semantic targets for plan and result references. @@ -138,9 +138,9 @@ RFC 033 adapter coverage may report whether a backend or verifier supports a req ### Standards alignment -Verifier statements and proof artifacts should align with public signed-attestation and provenance specifications when used outside InQL. W3C PROV can describe statement derivation, proof generation, proof verification, source commitments, result commitments, and responsible agents. in-toto and SLSA provenance can represent signed provenance over materials, subjects, builders, and reproducible verification steps. W3C Verifiable Credentials, JSON Web Signature, COSE, and JSON canonicalization specifications may be used by bridge profiles for portable signed claims and canonical signed payloads. +Verifier statements and proof artifacts should align with public signed-attestation and provenance specifications when used outside IncQL. W3C PROV can describe statement derivation, proof generation, proof verification, source commitments, result commitments, and responsible agents. in-toto and SLSA provenance can represent signed provenance over materials, subjects, builders, and reproducible verification steps. W3C Verifiable Credentials, JSON Web Signature, COSE, and JSON canonicalization specifications may be used by bridge profiles for portable signed claims and canonical signed payloads. -These standards provide envelopes, provenance vocabulary, and signature mechanics. They do not define InQL statement semantics, Prism plan identity, canonical equality profiles, proof coverage, or `proven` assurance by themselves. A bridge must preserve statement identity, verifier profile, commitment authority, commitment basis, public inputs, proof artifact identity, and coverage diagnostics. +These standards provide envelopes, provenance vocabulary, and signature mechanics. They do not define IncQL statement semantics, Prism plan identity, canonical equality profiles, proof coverage, or `proven` assurance by themselves. A bridge must preserve statement identity, verifier profile, commitment authority, commitment basis, public inputs, proof artifact identity, and coverage diagnostics. ### Compatibility / migration @@ -148,8 +148,8 @@ This RFC is additive. Existing verification observations can remain statement-fr ## Alternatives considered -- **Use SQL text as the statement.** Rejected because SQL text does not capture InQL semantic target identity, profile context, canonical equality rules, or source commitments. -- **Use backend plans as the statement.** Rejected because backend plans are execution artifacts, not the authoritative InQL semantic contract. +- **Use SQL text as the statement.** Rejected because SQL text does not capture IncQL semantic target identity, profile context, canonical equality rules, or source commitments. +- **Use backend plans as the statement.** Rejected because backend plans are execution artifacts, not the authoritative IncQL semantic contract. - **Define proof artifacts only after choosing a proof system.** Rejected because deterministic verification and proof systems both need the same statement boundary. - **Treat a successful proof as source truth.** Rejected because proof verification is relative to commitments and public inputs. @@ -166,8 +166,8 @@ This section is non-normative. A deterministic-digest implementation can generat ## Layers affected -- **InQL specification** — verifier statements, commitment references, verifier profiles, proof artifacts, and proof verification observations become evidence vocabulary. -- **InQL library package** — evidence APIs should be able to emit statement artifacts and attach them to verification observations. +- **IncQL specification** — verifier statements, commitment references, verifier profiles, proof artifacts, and proof verification observations become evidence vocabulary. +- **IncQL library package** — evidence APIs should be able to emit statement artifacts and attach them to verification observations. - **Execution / interchange** — adapters and verifier integrations may report coverage for statement families and proof artifact families. - **Documentation** — docs must explain statement identity, commitment trust, and the difference between verified and proven assurance. diff --git a/docs/rfcs/045_constraint_evidence_verification_planning.md b/docs/rfcs/045_constraint_evidence_verification_planning.md index 4f4bd507..7386a2e7 100644 --- a/docs/rfcs/045_constraint_evidence_verification_planning.md +++ b/docs/rfcs/045_constraint_evidence_verification_planning.md @@ -1,27 +1,27 @@ -# InQL RFC 045: Constraint evidence and verification-aware planning +# IncQL RFC 045: Constraint evidence and verification-aware planning - **Status:** Draft - **Created:** 2026-06-20 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 008 (optimizer boundary, statistics, cost-based optimization, and adaptive execution) - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 034 (quality assertions and observations) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 042 (async verification evidence) - - InQL RFC 043 (canonical equality and digest profiles) - - InQL RFC 044 (verifier statements and proof artifacts) -- **Issue:** [InQL #80](https://github.com/encero-systems/InQL/issues/80) -- **RFC PR:** [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 008 (optimizer boundary, statistics, cost-based optimization, and adaptive execution) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 034 (quality assertions and observations) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 043 (canonical equality and digest profiles) + - IncQL RFC 044 (verifier statements and proof artifacts) +- **Issue:** [IncQL #80](https://github.com/encero-systems/IncQL/issues/80) +- **RFC PR:** [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines constraint evidence and verification-aware planning for InQL. Constraints such as uniqueness, primary-key shape, foreign-key relationships, non-nullness, sortedness, partition coverage, and row-count bounds must be represented as evidence with assurance, and verification planners may use those constraints only when their preconditions are recorded and strong enough for the requested assurance. +This RFC defines constraint evidence and verification-aware planning for IncQL. Constraints such as uniqueness, primary-key shape, foreign-key relationships, non-nullness, sortedness, partition coverage, and row-count bounds must be represented as evidence with assurance, and verification planners may use those constraints only when their preconditions are recorded and strong enough for the requested assurance. ## Core model @@ -35,7 +35,7 @@ This RFC defines constraint evidence and verification-aware planning for InQL. C Real verification work often depends on facts such as "this key is unique," "this table covers every partition in this range," "this relation is sorted by this field," or "this join key is referentially complete." Those facts may come from model declarations, catalogs, source metadata, target metadata, connector attestations, prior checks, or deterministic verification runs. Treating them as ordinary assumptions would make verification overconfident. Ignoring them entirely would make verification too expensive or too weak. -Query-proof systems demonstrate the value of planning with proof cost in mind. InQL should generalize that idea for operational verification: choose a digest strategy, sample strategy, join check, aggregate check, or proof backend based on available evidence and requested assurance, while keeping every required precondition visible. +Query-proof systems demonstrate the value of planning with proof cost in mind. IncQL should generalize that idea for operational verification: choose a digest strategy, sample strategy, join check, aggregate check, or proof backend based on available evidence and requested assurance, while keeping every required precondition visible. ## Goals @@ -138,7 +138,7 @@ Verification-aware planning chooses how to verify; it does not alter the authore Constraint evidence describes facts about data, schemas, or relationships. It is only as strong as its evidence source and assurance label. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces RFC 008 defines optimizer boundaries for execution planning. Verification-aware planning is separate: it plans evidence work and may use cost information, but it must not become ordinary query optimization. @@ -154,7 +154,7 @@ RFC 044 verifier statements may include constraints as public evidence or precon Constraint evidence and verification plans should be projectable to public provenance, quality, lineage, and telemetry standards where useful. W3C PROV can represent constraint declarations, checks, imports, and verification planning steps as provenance activities and entities. W3C Data Quality Vocabulary can represent constraint checks as quality measurements or metrics. OpenLineage can expose verification-plan and constraint-evidence summaries as run, job, dataset, or custom facets. OpenTelemetry can expose planning and verification work as traces, spans, events, and metrics. -External catalog, lineage, and quality standards may provide constraint inputs, but imported constraints remain evidence with explicit assurance. A bridge must not upgrade catalog-declared or externally attested constraints to verified constraint evidence unless InQL can represent and validate the underlying check. +External catalog, lineage, and quality standards may provide constraint inputs, but imported constraints remain evidence with explicit assurance. A bridge must not upgrade catalog-declared or externally attested constraints to verified constraint evidence unless IncQL can represent and validate the underlying check. ### Compatibility / migration @@ -180,8 +180,8 @@ This section is non-normative. A practical implementation can plan strategies su ## Layers affected -- **InQL specification** — constraint evidence and verification plan vocabulary become part of the evidence model. -- **InQL library package** — inspection and verification APIs should be able to emit constraint evidence and verification plans. +- **IncQL specification** — constraint evidence and verification plan vocabulary become part of the evidence model. +- **IncQL library package** — inspection and verification APIs should be able to emit constraint evidence and verification plans. - **Execution / interchange** — adapters may report coverage for uniqueness checks, referential checks, partition coverage, and staged verification strategies. - **Documentation** — docs must explain that constraints are evidence with assurance, not hidden assumptions. diff --git a/docs/rfcs/046_data_contract_ingress.md b/docs/rfcs/046_data_contract_ingress.md index 536f081c..7f255e8c 100644 --- a/docs/rfcs/046_data_contract_ingress.md +++ b/docs/rfcs/046_data_contract_ingress.md @@ -1,42 +1,42 @@ -# InQL RFC 046: Data contract ingress and product topology +# IncQL RFC 046: Data contract ingress and product topology - **Status:** Draft - **Created:** 2026-06-20 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 034 (quality assertions and observations) - - InQL RFC 035 (governed attributes and policy checkpoints) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 038 (evidence exchange bridges) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 042 (async verification evidence) - - InQL RFC 045 (constraint evidence and verification-aware planning) - - InQL RFC 047 (semantic evidence graph and agent query surface) -- **Issue:** [InQL #81](https://github.com/encero-systems/InQL/issues/81) -- **RFC PR:** [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 034 (quality assertions and observations) + - IncQL RFC 035 (governed attributes and policy checkpoints) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 038 (evidence exchange bridges) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 045 (constraint evidence and verification-aware planning) + - IncQL RFC 047 (semantic evidence graph and agent query surface) +- **Issue:** [IncQL #81](https://github.com/encero-systems/IncQL/issues/81) +- **RFC PR:** [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines data contract ingress and product topology evidence for InQL. Existing contract and product formats such as Open Data Contract Standard, Open Data Product Standard, and legacy Data Contract Specification artifacts are imported as evidence, normalized onto InQL semantic targets, and lowered into metadata attachments, quality assertions, constraint evidence, source bindings, product-port dependencies, and governed bundle records without making any external contract format the internal source of InQL semantics. +This RFC defines data contract ingress and product topology evidence for IncQL. Existing contract and product formats such as Open Data Contract Standard, Open Data Product Standard, and legacy Data Contract Specification artifacts are imported as evidence, normalized onto IncQL semantic targets, and lowered into metadata attachments, quality assertions, constraint evidence, source bindings, product-port dependencies, and governed bundle records without making any external contract format the internal source of IncQL semantics. ## Core model -1. InQL does not define a new data contract standard. +1. IncQL does not define a new data contract standard. 2. Contract artifacts are imported evidence with source, format, version, status, scope, fingerprint, and diagnostics. -3. Contract clauses lower into existing InQL evidence families where possible. +3. Contract clauses lower into existing IncQL evidence families where possible. 4. Product topology describes ports, contracts, dependencies, management endpoints, support, ownership, and SBOM references around contract evidence. 5. Imported declarations are not verified facts unless later verification evidence proves them. 6. Contract ingress is an exchange bridge profile, but normalized contract evidence must be useful inside governed bundles and evidence graphs. ## Motivation -Data contracts already have active open formats. Open Data Contract Standard covers schema objects, properties, data quality rules, server bindings, service-level agreements, roles, teams, authoritative definitions, custom properties, and related metadata. Open Data Product Standard covers data product envelopes, input ports, output ports, contract identifiers, product dependencies, management ports, support, ownership, and SBOM references. The older Data Contract Specification remains relevant because tools and existing repositories still use it, even though its maintainers recommend migration to Open Data Contract Standard. InQL should absorb these surfaces as evidence rather than inventing a competing vocabulary. +Data contracts already have active open formats. Open Data Contract Standard covers schema objects, properties, data quality rules, server bindings, service-level agreements, roles, teams, authoritative definitions, custom properties, and related metadata. Open Data Product Standard covers data product envelopes, input ports, output ports, contract identifiers, product dependencies, management ports, support, ownership, and SBOM references. The older Data Contract Specification remains relevant because tools and existing repositories still use it, even though its maintainers recommend migration to Open Data Contract Standard. IncQL should absorb these surfaces as evidence rather than inventing a competing vocabulary. The existing evidence RFCs already have better homes for most contract facts. Required fields, uniqueness, primary keys, foreign keys, partitioning, and referential relationships belong to constraint evidence. Quality rules belong to quality assertions. Server and platform locations belong to source binding and semantic profile evidence. Classifications and critical data elements belong to governed attributes and policy checkpoints. Contract identifiers and product ports belong to bundle and graph topology. What is missing is a normative ingress layer that records where those facts came from, how they map, and what assurance they carry. @@ -46,24 +46,24 @@ The existing evidence RFCs already have better homes for most contract facts. Re - Treat Open Data Contract Standard as the primary data-contract ingress profile. - Treat Open Data Product Standard as the primary data-product topology ingress profile. - Support legacy Data Contract Specification artifacts as an import and migration profile. -- Normalize contract clauses onto InQL semantic targets and existing evidence families. +- Normalize contract clauses onto IncQL semantic targets and existing evidence families. - Keep imported declarations distinct from deterministic verification results. - Preserve source artifact identity, version, fingerprint, status, mapping coverage, unsupported fields, and diagnostics. - Allow governed plan bundles and evidence graphs to include contract and product topology evidence. ## Non-Goals -- Defining a new data contract syntax or custom InQL vocabulary for contracts. +- Defining a new data contract syntax or custom IncQL vocabulary for contracts. - Replacing Open Data Contract Standard, Open Data Product Standard, or existing data-contract tooling. - Defining legal agreement semantics, data usage agreement lifecycle, signatures, approvals, or contract negotiation workflows. - Defining data-product hosting, catalog storage, access-request processing, or product management UI. -- Defining syntax sugar for authoring contracts inside InQL. +- Defining syntax sugar for authoring contracts inside IncQL. - Treating imported contract facts as verified evidence by default. - Guaranteeing complete lossless import from every third-party contract format. ## Guide-level explanation (how authors think about it) -An author or CI workflow can import a contract artifact and bind it to InQL semantic targets: +An author or CI workflow can import a contract artifact and bind it to IncQL semantic targets: ```incan contract = import_data_contract("contracts/orders.odcs.yaml") @@ -74,7 +74,7 @@ bundle = governed_plan_bundle( ) ``` -The names are illustrative. The contract importer records the artifact as evidence, validates the known external schema where possible, maps recognized clauses to InQL evidence records, and reports unsupported clauses instead of silently dropping them. +The names are illustrative. The contract importer records the artifact as evidence, validates the known external schema where possible, maps recognized clauses to IncQL evidence records, and reports unsupported clauses instead of silently dropping them. ```text artifact=contracts/orders.odcs.yaml @@ -107,7 +107,7 @@ outcome=passed assurance=verified ``` -Product topology works the same way. A data product may declare output ports tied to contract identifiers and input dependencies on other contract identifiers. InQL imports those as topology evidence so an evidence graph can answer dependency and impact questions without pretending the product format owns relational semantics. +Product topology works the same way. A data product may declare output ports tied to contract identifiers and input dependencies on other contract identifiers. IncQL imports those as topology evidence so an evidence graph can answer dependency and impact questions without pretending the product format owns relational semantics. ```incan product = import_data_product("products/customer.odps.yaml") @@ -126,7 +126,7 @@ Format family must distinguish at least: - custom: an explicitly named custom contract format - unknown: an unclassified contract-like artifact -Imported contract clauses must lower into InQL evidence families where the mapping is known. The importer must not create a parallel internal contract vocabulary when an existing evidence family can represent the clause. +Imported contract clauses must lower into IncQL evidence families where the mapping is known. The importer must not create a parallel internal contract vocabulary when an existing evidence family can represent the clause. Open Data Contract Standard ingress must map recognized clauses as follows: @@ -153,7 +153,7 @@ Open Data Product Standard ingress must map recognized clauses as follows: Legacy Data Contract Specification ingress must map recognized clauses as follows: -- `info`, `servers`, `terms`, `models`, `fields`, `definitions`, `quality`, `servicelevels`, `lineage`, and `links` must lower to the same InQL evidence families used for Open Data Contract Standard where semantics match. +- `info`, `servers`, `terms`, `models`, `fields`, `definitions`, `quality`, `servicelevels`, `lineage`, and `links` must lower to the same IncQL evidence families used for Open Data Contract Standard where semantics match. - Legacy `lineage` clauses may seed lineage hints or external lineage evidence, but they must not become Prism-authored lineage. - Legacy artifacts should carry a migration diagnostic indicating that Open Data Contract Standard is the preferred primary contract format when a bridge profile can report that safely. @@ -161,9 +161,9 @@ An imported declaration must use assurance `attested` unless another evidence re Imported schema validation, contract linting, and external CLI test results may be attached as execution, quality, or verification evidence. The assurance of those results depends on their captured inputs, engine identity, source snapshots, and diagnostics; it must not be inferred solely from the contract format. -Contract mappings must preserve unsupported fields. If a recognized format contains clauses that have no InQL mapping, the contract artifact record must report those clauses as unsupported, custom, or lossy according to the mapping profile. Unknown fields must not be silently ignored when the bridge is configured for strict evidence import. +Contract mappings must preserve unsupported fields. If a recognized format contains clauses that have no IncQL mapping, the contract artifact record must report those clauses as unsupported, custom, or lossy according to the mapping profile. Unknown fields must not be silently ignored when the bridge is configured for strict evidence import. -Contract mappings must preserve target ambiguity. If an artifact describes an object or field that cannot be bound to an InQL semantic target, the importer may create an external target candidate, but it must not pretend that candidate is a resolved InQL target. +Contract mappings must preserve target ambiguity. If an artifact describes an object or field that cannot be bound to an IncQL semantic target, the importer may create an external target candidate, but it must not pretend that candidate is a resolved IncQL target. Contract artifact fingerprints should use stable canonicalization for the imported artifact when a canonicalization profile is available. If the importer cannot canonicalize the format safely, it should record a raw artifact fingerprint and a diagnostic rather than inventing semantic identity from unstable formatting. @@ -173,7 +173,7 @@ Governed plan bundles may include imported contract artifacts, normalized contra ### Syntax -This RFC introduces no InQL contract authoring syntax. Import APIs, bridge profiles, governed bundle sections, and evidence graph projections are the normative surface. Syntax sugar for authoring or referencing contracts inside InQL belongs to a separate RFC if it becomes necessary. +This RFC introduces no IncQL contract authoring syntax. Import APIs, bridge profiles, governed bundle sections, and evidence graph projections are the normative surface. Syntax sugar for authoring or referencing contracts inside IncQL belongs to a separate RFC if it becomes necessary. ### Semantics @@ -181,7 +181,7 @@ Contract ingress is evidence import. It records declarations, expectations, owne Product topology is graph-shaped evidence around contracts and ports. It explains which contracts are promised by outputs, expected by inputs, managed by endpoints, supported by teams, and referenced by artifacts. It does not redefine relational transformations, Prism lineage, or execution observations. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces RFC 028 provides semantic targets that imported contract clauses may bind to. @@ -203,7 +203,7 @@ RFC 047 may project imported contract and product topology evidence into a graph ### Standards alignment -Open Data Contract Standard is the primary source for data contract ingress. Open Data Product Standard is the primary source for data product topology ingress. Data Contract Specification should be supported as a legacy import profile when an implementation chooses to support existing artifacts, but new InQL examples should prefer Open Data Contract Standard and Open Data Product Standard. +Open Data Contract Standard is the primary source for data contract ingress. Open Data Product Standard is the primary source for data product topology ingress. Data Contract Specification should be supported as a legacy import profile when an implementation chooses to support existing artifacts, but new IncQL examples should prefer Open Data Contract Standard and Open Data Product Standard. Contract and product ingress should also remain bridgeable to W3C PROV, DCAT, DCAT-AP, schema.org Dataset, Dublin Core, W3C Data Quality Vocabulary, OpenLineage, OpenTelemetry, in-toto, SLSA provenance, W3C Verifiable Credentials, JSON Web Signature, COSE, and related bridge profiles through RFC 038 when the target standard can represent the relevant evidence. @@ -211,7 +211,7 @@ Open-data governance principles can guide documentation around provenance, reuse ### Compatibility / migration -This RFC is additive. Existing InQL evidence artifacts remain valid without contract ingress sections. +This RFC is additive. Existing IncQL evidence artifacts remain valid without contract ingress sections. Legacy Data Contract Specification support should be treated as compatibility import. When both a legacy artifact and an Open Data Contract Standard artifact describe the same contract, a bridge must record the source of each fact and must not merge conflicting declarations without diagnostics. @@ -219,8 +219,8 @@ Adding support for a new external contract format must add a mapping profile rat ## Alternatives considered -- **Define an InQL-native contract YAML.** Rejected because existing open standards already cover the contract and product surfaces InQL needs to consume. -- **Adopt Open Data Contract Standard as the internal model.** Rejected because InQL evidence needs semantic targets, assurance, verification state, bundles, and graph projections that are broader than a contract document. +- **Define an IncQL-native contract YAML.** Rejected because existing open standards already cover the contract and product surfaces IncQL needs to consume. +- **Adopt Open Data Contract Standard as the internal model.** Rejected because IncQL evidence needs semantic targets, assurance, verification state, bundles, and graph projections that are broader than a contract document. - **Treat imported contracts as truth.** Rejected because contracts are declarations until checked against specific data, snapshots, or runtime observations. - **Support only one contract format.** Rejected because product topology and legacy import needs are distinct, and existing users may arrive with older artifacts. - **Hide unsupported fields during import.** Rejected because governance and verification workflows need to know what evidence was lost or not understood. @@ -239,14 +239,14 @@ This section is non-normative. A practical implementation can start with schema ## Layers affected -- **InQL specification** — data contract artifact, normalized contract evidence, product topology, and mapping coverage vocabulary become part of the evidence model. -- **InQL library package** — import and inspection APIs should be able to ingest supported contract artifacts and expose normalized evidence records. +- **IncQL specification** — data contract artifact, normalized contract evidence, product topology, and mapping coverage vocabulary become part of the evidence model. +- **IncQL library package** — import and inspection APIs should be able to ingest supported contract artifacts and expose normalized evidence records. - **Execution / interchange** — exchange bridges may import contract tests, external CLI results, source bindings, product ports, and contract dependency evidence. - **Documentation** — docs must explain that contract declarations are imported evidence and not verified data facts. ## Unresolved questions -- Which Open Data Contract Standard clauses are required for a conforming InQL importer? +- Which Open Data Contract Standard clauses are required for a conforming IncQL importer? - Which Open Data Product Standard clauses are required before product topology is considered supported? - Should legacy Data Contract Specification import be required or optional? - What canonical artifact fingerprint profile should be recommended for YAML contract files? diff --git a/docs/rfcs/047_semantic_evidence_graph_agent_surface.md b/docs/rfcs/047_semantic_evidence_graph_agent_surface.md index aa8c2f73..435ccd54 100644 --- a/docs/rfcs/047_semantic_evidence_graph_agent_surface.md +++ b/docs/rfcs/047_semantic_evidence_graph_agent_surface.md @@ -1,34 +1,34 @@ -# InQL RFC 047: Semantic evidence graph and agent query surface +# IncQL RFC 047: Semantic evidence graph and agent query surface - **Status:** Draft - **Created:** 2026-06-20 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 027 (relational evidence program) - - InQL RFC 028 (semantic identity and target model) - - InQL RFC 029 (typed metadata attachments) - - InQL RFC 030 (Prism lineage graph) - - InQL RFC 031 (local inspection APIs and artifacts) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 034 (quality assertions and observations) - - InQL RFC 035 (governed attributes and policy checkpoints) - - InQL RFC 036 (governed plan bundle) - - InQL RFC 037 (plan diff and blast-radius inputs) - - InQL RFC 038 (evidence exchange bridges) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 042 (async verification evidence) - - InQL RFC 044 (verifier statements and proof artifacts) - - InQL RFC 045 (constraint evidence and verification-aware planning) - - InQL RFC 046 (data contract ingress and product topology) -- **Issue:** [InQL #82](https://github.com/encero-systems/InQL/issues/82) -- **RFC PR:** [InQL #83](https://github.com/encero-systems/InQL/pull/83) -- **Written against:** Incan v0.3-era InQL + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 028 (semantic identity and target model) + - IncQL RFC 029 (typed metadata attachments) + - IncQL RFC 030 (Prism lineage graph) + - IncQL RFC 031 (local inspection APIs and artifacts) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 034 (quality assertions and observations) + - IncQL RFC 035 (governed attributes and policy checkpoints) + - IncQL RFC 036 (governed plan bundle) + - IncQL RFC 037 (plan diff and blast-radius inputs) + - IncQL RFC 038 (evidence exchange bridges) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 042 (async verification evidence) + - IncQL RFC 044 (verifier statements and proof artifacts) + - IncQL RFC 045 (constraint evidence and verification-aware planning) + - IncQL RFC 046 (data contract ingress and product topology) +- **Issue:** [IncQL #82](https://github.com/encero-systems/IncQL/issues/82) +- **RFC PR:** [IncQL #83](https://github.com/encero-systems/IncQL/pull/83) +- **Written against:** Incan v0.3-era IncQL - **Shipped in:** — ## Summary -This RFC defines a semantic evidence graph and deterministic agent query surface for InQL. The graph is a projection over InQL evidence records, not a new source of truth: declared contracts, Prism semantic lineage, imported catalog evidence, runtime observations, adapter coverage, verification observations, proof artifacts, waivers, product topology, and governed bundle records can be queried together through stable node and edge families with provenance, assurance, coverage, time, snapshot, and diagnostic context preserved. +This RFC defines a semantic evidence graph and deterministic agent query surface for IncQL. The graph is a projection over IncQL evidence records, not a new source of truth: declared contracts, Prism semantic lineage, imported catalog evidence, runtime observations, adapter coverage, verification observations, proof artifacts, waivers, product topology, and governed bundle records can be queried together through stable node and edge families with provenance, assurance, coverage, time, snapshot, and diagnostic context preserved. ## Core model @@ -43,17 +43,17 @@ This RFC defines a semantic evidence graph and deterministic agent query surface Lineage, contract, observability, quality, and verification questions are naturally graph questions. A user may need to ask where a business term or output field comes from, which products depend on a contract, which runtime jobs wrote a dataset, which outputs depend on waived evidence, which source fields contribute to a metric, or what changes if a model, field, source, contract, or profile changes. The existing RFCs define the evidence families needed to answer those questions, but they do not yet define one graph-shaped query surface that composes them. -Recent open lineage and graph-observability work reinforces the split InQL should preserve. Runtime OpenLineage events describe what jobs actually ran and which datasets they read or wrote. Semantic lineage and contract metadata describe what a field, term, model, or product is supposed to mean. Verification observations describe which claims have actually been checked and with what assurance. A graph query surface is useful precisely because it can traverse all of those layers while preserving their different evidence basis. +Recent open lineage and graph-observability work reinforces the split IncQL should preserve. Runtime OpenLineage events describe what jobs actually ran and which datasets they read or wrote. Semantic lineage and contract metadata describe what a field, term, model, or product is supposed to mean. Verification observations describe which claims have actually been checked and with what assurance. A graph query surface is useful precisely because it can traverse all of those layers while preserving their different evidence basis. ## Goals -- Define an InQL semantic evidence graph projection. +- Define an IncQL semantic evidence graph projection. - Define stable node and edge families for evidence-backed graph queries. - Preserve provenance, assurance, lifecycle, outcome, coverage, profile context, snapshot context, and diagnostics on graph elements. - Support deterministic impact, provenance, dependency, verification, and contract-topology queries. - Define an agent query surface that returns bounded evidence-backed answers and refuses to infer absent mappings. - Allow projection to property graph tables, openCypher-compatible stores, OpenLineage-shaped views, W3C PROV, local artifacts, and governed bundles. -- Keep graph storage, graph database choice, hosted query services, and conversational UI outside InQL core. +- Keep graph storage, graph database choice, hosted query services, and conversational UI outside IncQL core. ## Non-Goals @@ -182,9 +182,9 @@ Edge kind should include at least: Relationship layer must distinguish at least declared, semantic, observed, verified, proven, waived, imported, and projected relationships. A declared relationship from a contract, an observed relationship from runtime lineage, a semantic relationship from Prism, and a verified relationship from a verification observation must not collapse into the same edge without preserving their layer and basis. -Graph projection rules must preserve InQL lineage kinds. Value lineage, control lineage, grouping lineage, ordering lineage, join lineage, and other Prism lineage distinctions must not be flattened to a generic dependency edge unless the projection explicitly reports mapping loss. +Graph projection rules must preserve IncQL lineage kinds. Value lineage, control lineage, grouping lineage, ordering lineage, join lineage, and other Prism lineage distinctions must not be flattened to a generic dependency edge unless the projection explicitly reports mapping loss. -OpenLineage runtime events may project to run, job, dataset, read, write, input, output, and facet-backed nodes or edges. Runtime events must remain observed or attested evidence unless InQL evidence verifies the corresponding claim. +OpenLineage runtime events may project to run, job, dataset, read, write, input, output, and facet-backed nodes or edges. Runtime events must remain observed or attested evidence unless IncQL evidence verifies the corresponding claim. Contract and product artifacts may project to contract, clause, product, port, and dependency nodes or edges. Imported contract facts must retain their source artifact and must not become verified graph relationships by default. @@ -206,7 +206,7 @@ Governed plan bundles may include graph projection manifests, graph node and edg ### Syntax -This RFC introduces no InQL query syntax or graph query language. Graph construction, export, and agent query functions are inspection and artifact surfaces. Future syntax may reference graph queries, but it must lower to the same graph projection and query-result records. +This RFC introduces no IncQL query syntax or graph query language. Graph construction, export, and agent query functions are inspection and artifact surfaces. Future syntax may reference graph queries, but it must lower to the same graph projection and query-result records. ### Semantics @@ -214,7 +214,7 @@ The graph is a navigable evidence index. It does not redefine relational semanti The graph may materialize edges from multiple sources that disagree. Conflict is represented as graph evidence with different basis, outcome, assurance, source, or diagnostics, not by deleting one side silently. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces RFC 028 provides semantic target identities for graph nodes. @@ -240,7 +240,7 @@ OpenLineage is the primary runtime lineage exchange surface for run, job, datase W3C PROV can represent graph evidence as entities, activities, agents, derivations, generated-by relationships, used relationships, and responsibility links. OpenTelemetry can carry trace and span correlation for graph-linked execution and verification work. W3C Data Quality Vocabulary can project quality metrics and measurements. Open Data Contract Standard and Open Data Product Standard can seed contract and product topology nodes. in-toto, SLSA provenance, W3C Verifiable Credentials, JSON Web Signature, COSE, and canonical signed payload specifications can provide signed attestation envelopes when bridge profiles support them. -Property graph tables, openCypher-compatible stores, RDF-shaped stores, and graph visualization tools may be projection targets. None of those storage or query formats is the internal source of InQL evidence semantics. +Property graph tables, openCypher-compatible stores, RDF-shaped stores, and graph visualization tools may be projection targets. None of those storage or query formats is the internal source of IncQL evidence semantics. ### Compatibility / migration @@ -250,9 +250,9 @@ If a graph projection profile changes node identities, edge identities, or mappi ## Alternatives considered -- **Use OpenLineage as the whole graph model.** Rejected because runtime job and dataset events are necessary but do not carry all InQL semantic targets, declared contracts, verification assurance, proof artifacts, waivers, or Prism lineage kinds. +- **Use OpenLineage as the whole graph model.** Rejected because runtime job and dataset events are necessary but do not carry all IncQL semantic targets, declared contracts, verification assurance, proof artifacts, waivers, or Prism lineage kinds. - **Use Prism lineage as the whole graph model.** Rejected because Prism semantic lineage does not by itself cover runtime observations, product topology, imported contracts, quality observations, or verification state. -- **Require a specific graph database.** Rejected because InQL should define evidence semantics and projections, not storage infrastructure. +- **Require a specific graph database.** Rejected because IncQL should define evidence semantics and projections, not storage infrastructure. - **Let agents read raw artifacts directly.** Rejected because agents should call deterministic tools over bounded evidence, not infer lineage or assurance from unstructured logs. - **Flatten all relationships to `depends_on`.** Rejected because declared, semantic, observed, verified, proven, and waived relationships carry different meaning and risk. @@ -270,8 +270,8 @@ This section is non-normative. A practical implementation can emit graph node an ## Layers affected -- **InQL specification** — graph node, edge, projection, and agent query-result vocabulary become part of the evidence model. -- **InQL library package** — inspection APIs should be able to construct graph projections and answer bounded graph queries. +- **IncQL specification** — graph node, edge, projection, and agent query-result vocabulary become part of the evidence model. +- **IncQL library package** — inspection APIs should be able to construct graph projections and answer bounded graph queries. - **Execution / interchange** — exchange bridges may import OpenLineage events and export property graph, OpenLineage, PROV, or telemetry-shaped projections. - **Documentation** — docs must explain declared, semantic, observed, verified, proven, waived, imported, and projected relationship layers. diff --git a/docs/rfcs/048_cluster_execution_backend_mode.md b/docs/rfcs/048_cluster_execution_backend_mode.md index bb9610e3..1f23f7a2 100644 --- a/docs/rfcs/048_cluster_execution_backend_mode.md +++ b/docs/rfcs/048_cluster_execution_backend_mode.md @@ -1,41 +1,41 @@ -# InQL RFC 048: Cluster execution backend mode +# IncQL RFC 048: Cluster execution backend mode - **Status:** Draft - **Created:** 2026-07-05 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 001 (dataset types and execution backend boundary) - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 004 (execution context) - - InQL RFC 007 (Prism planning engine) - - InQL RFC 008 (optimizer boundary, statistics, CBO, and adaptive execution) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 041 (Prism plan ingress and external client frontends) + - IncQL RFC 001 (dataset types and execution backend boundary) + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 004 (execution context) + - IncQL RFC 007 (Prism planning engine) + - IncQL RFC 008 (optimizer boundary, statistics, CBO, and adaptive execution) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 041 (Prism plan ingress and external client frontends) - **Issue:** — - **RFC PR:** — -- **Written against:** Incan 0.4.0-rc3 and InQL's v0.3-era package migration context +- **Written against:** Incan 0.4.0-rc3 and IncQL's v0.3-era package migration context - **Shipped in:** — ## Summary -This RFC defines cluster execution as a backend mode of InQL's existing `Session` execution boundary. Cluster mode must not create a second InQL semantic model: authors still produce typed InQL logical plans, Prism and Substrait remain the logical boundary, and backend adapters remain responsible for planning, scheduling, execution, runtime observations, and capability diagnostics. DataFusion remains the default local backend; a DataFusion-compatible cluster backend such as Ballista is the first concrete proof target because it can accept Substrait logical plans while adding scheduler, worker, shuffle, and distributed-observability concerns. Streaming uses the same backend-mode framing, but adds long-running lifecycle, checkpoint, watermark, offset, and sink-commit requirements for `DataStream[T]`. +This RFC defines cluster execution as a backend mode of IncQL's existing `Session` execution boundary. Cluster mode must not create a second IncQL semantic model: authors still produce typed IncQL logical plans, Prism and Substrait remain the logical boundary, and backend adapters remain responsible for planning, scheduling, execution, runtime observations, and capability diagnostics. DataFusion remains the default local backend; a DataFusion-compatible cluster backend such as Ballista is the first concrete proof target because it can accept Substrait logical plans while adding scheduler, worker, shuffle, and distributed-observability concerns. Streaming uses the same backend-mode framing, but adds long-running lifecycle, checkpoint, watermark, offset, and sink-commit requirements for `DataStream[T]`. ## Motivation -InQL started with a local DataFusion reference backend. That is enough for v0.1 read, transform, collect, and write workflows, but it is not enough for larger analytical work where data lives in object stores, catalogs, warehouses, or lakehouse tables and execution must happen near the data. Cluster execution is therefore not optional product polish; it is part of the credible execution story. +IncQL started with a local DataFusion reference backend. That is enough for v0.1 read, transform, collect, and write workflows, but it is not enough for larger analytical work where data lives in object stores, catalogs, warehouses, or lakehouse tables and execution must happen near the data. Cluster execution is therefore not optional product polish; it is part of the credible execution story. -The risk is allowing cluster concerns to leak into InQL's semantic layer. Distributed schedulers need job identifiers, worker registration, shuffle behavior, retry policy, adaptive execution, remote source access, and large-result handling. Those facts matter, but they are not query semantics. InQL needs an explicit cluster lane so the implementation can support distributed execution without making physical partitioning, scheduler behavior, or backend-specific knobs part of the portable author contract. +The risk is allowing cluster concerns to leak into IncQL's semantic layer. Distributed schedulers need job identifiers, worker registration, shuffle behavior, retry policy, adaptive execution, remote source access, and large-result handling. Those facts matter, but they are not query semantics. IncQL needs an explicit cluster lane so the implementation can support distributed execution without making physical partitioning, scheduler behavior, or backend-specific knobs part of the portable author contract. -The Apache DataFusion / Ballista 53.0.0 release is relevant because Ballista adds a Substrait scheduler client and presents a concrete DataFusion-family cluster proof target ([release notes](https://datafusion.apache.org/blog/output/2026/05/24/datafusion-ballista-53.0.0/)). InQL should use that as implementation evidence, not as the normative definition of cluster mode. +The Apache DataFusion / Ballista 53.0.0 release is relevant because Ballista adds a Substrait scheduler client and presents a concrete DataFusion-family cluster proof target ([release notes](https://datafusion.apache.org/blog/output/2026/05/24/datafusion-ballista-53.0.0/)). IncQL should use that as implementation evidence, not as the normative definition of cluster mode. ## Goals - Define cluster execution as a `Session` backend mode. -- Preserve one InQL semantic model across local and cluster execution. +- Preserve one IncQL semantic model across local and cluster execution. - Require explicit backend capability checks for cluster execution. - Define the minimum source, sink, credential, UDF, result, error, and observation boundaries cluster mode must respect. -- Distinguish bounded cluster jobs from streaming cluster jobs without splitting the InQL semantic model. +- Distinguish bounded cluster jobs from streaming cluster jobs without splitting the IncQL semantic model. - Treat Ballista or another DataFusion-compatible cluster runner as a proof target behind the backend abstraction. - Keep cluster scheduler behavior visible through execution observations rather than hidden in adapter internals. @@ -44,19 +44,19 @@ The Apache DataFusion / Ballista 53.0.0 release is relevant because Ballista add - Defining a cluster scheduler implementation. - Standardizing Ballista as the only cluster backend. - Defining every DataFusion, Ballista, Spark, or warehouse configuration knob. -- Introducing cluster-specific InQL query syntax. +- Introducing cluster-specific IncQL query syntax. - Defining storage catalog APIs, credential providers, or object-store configuration in detail. - Defining complete streaming semantics, watermark policies, trigger semantics, or state-store implementations. -- Guaranteeing that every InQL feature can execute on every cluster backend. -- Making physical partition IDs, shuffle layout, worker placement, or retry behavior part of portable InQL semantics. +- Guaranteeing that every IncQL feature can execute on every cluster backend. +- Making physical partition IDs, shuffle layout, worker placement, or retry behavior part of portable IncQL semantics. ## Guide-level explanation (how authors think about it) Authors should think of cluster execution as changing where a plan runs, not what the plan means. ```incan -from pub::inql import Session -from pub::inql import backends +from pub::incql import Session +from pub::incql import backends session = ( Session.builder() @@ -97,7 +97,7 @@ cluster = ( ) ``` -Cluster mode may reject a plan before execution when the selected backend cannot satisfy required capabilities. For example, an InQL-owned UDF, generator relation, typed variant value, sketch value, or extension metadata shape may be unsupported by a cluster backend. That is a backend coverage result, not an invalid InQL program. +Cluster mode may reject a plan before execution when the selected backend cannot satisfy required capabilities. For example, an IncQL-owned UDF, generator relation, typed variant value, sketch value, or extension metadata shape may be unsupported by a cluster backend. That is a backend coverage result, not an invalid IncQL program. For large outputs, authors and tools should prefer `execute(...)` and `write(...)` over `collect(...)`. `collect(...)` remains a materialization boundary, but cluster adapters may require limits, previews, or explicit approval for large result collection. @@ -118,27 +118,27 @@ The important difference is lifecycle rather than semantics. A bounded `LazyFram Cluster execution is a backend selection under the existing `Session` contract. A session owns one execution backend for a given execution boundary. The backend may be local or cluster-backed, but the author-facing session type remains `Session`. -Backend-specific configuration must live in `pub::inql.backends` or an equivalent backend namespace. InQL must not introduce backend-named root session types as the portable API. +Backend-specific configuration must live in `pub::incql.backends` or an equivalent backend namespace. IncQL must not introduce backend-named root session types as the portable API. ### Semantic equivalence -For a plan that both local and cluster backends can execute, cluster execution must preserve the same InQL logical result as local execution: +For a plan that both local and cluster backends can execute, cluster execution must preserve the same IncQL logical result as local execution: - schema and output aliases must match the logical plan -- row values must match the InQL semantic contract +- row values must match the IncQL semantic contract - ordering is guaranteed only when the logical plan declares ordering - backend runtime metrics and physical plans may differ - adaptive execution may change physical strategy but not logical results -If a backend cannot preserve the InQL contract, it must report uncovered, partially covered, or unknown coverage through RFC 033-style adapter coverage records or fail with an execution error. It must not silently reinterpret the plan. +If a backend cannot preserve the IncQL contract, it must report uncovered, partially covered, or unknown coverage through RFC 033-style adapter coverage records or fail with an execution error. It must not silently reinterpret the plan. ### Plan transport -Cluster adapters should accept the same logical boundary as local adapters: Prism-owned logical intent lowered to Substrait plus InQL-owned registry metadata and read-root bindings. A DataFusion-family cluster backend may use Substrait submission when available. +Cluster adapters should accept the same logical boundary as local adapters: Prism-owned logical intent lowered to Substrait plus IncQL-owned registry metadata and read-root bindings. A DataFusion-family cluster backend may use Substrait submission when available. Substrait is not enough by itself. The adapter must also account for: -- InQL extension metadata and registry function anchors +- IncQL extension metadata and registry function anchors - logical read-root bindings - source and sink descriptors - backend capability coverage @@ -152,7 +152,7 @@ Cluster workers must be able to resolve every source and sink used by a plan. A Cluster backend configuration must distinguish: -- logical source names used by InQL plans +- logical source names used by IncQL plans - backend-resolved table providers or catalogs - object-store and filesystem profiles - credential references @@ -162,7 +162,7 @@ Credentials must not be embedded in Prism records, Substrait plans, evidence bun ### UDF and helper deployment -If an InQL helper executes through a backend UDF, callback, extension function, or adapter-provided implementation, cluster mode must ensure that the implementation is available on every worker that may execute it. +If an IncQL helper executes through a backend UDF, callback, extension function, or adapter-provided implementation, cluster mode must ensure that the implementation is available on every worker that may execute it. Missing UDF deployment is a backend coverage or execution error. It must not be treated as a different function result, a null result, or a fallback to untyped SQL text. @@ -240,13 +240,13 @@ Illustrative names such as `DataFusionCluster` are non-normative until implement ### Semantics -Cluster mode changes execution placement and physical execution strategy. It does not change InQL expression semantics, query schema rules, function registry meaning, or Substrait lowering contracts. +Cluster mode changes execution placement and physical execution strategy. It does not change IncQL expression semantics, query schema rules, function registry meaning, or Substrait lowering contracts. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces - RFC 001 remains the carrier and dataset boundary. Cluster mode does not introduce new carrier types. - RFC 001 remains the source of `BoundedDataSet[T]` / `UnboundedDataSet[T]` capability gating. Streaming cluster execution must not loosen static `DataStream[T]` restrictions. -- RFC 002 remains the Substrait interchange boundary. Cluster mode may use Substrait submission but must preserve InQL metadata and binding contracts. +- RFC 002 remains the Substrait interchange boundary. Cluster mode may use Substrait submission but must preserve IncQL metadata and binding contracts. - RFC 004 remains the session and backend-selection boundary. This RFC narrows how cluster mode fits under that boundary. - RFC 008 owns optimizer and adaptive-execution boundaries. Cluster mode may expose runtime facts used by the backend, but Prism remains the semantic logical planning owner. - RFC 032 owns execution observations. Cluster-specific runtime facts should appear there. @@ -257,19 +257,19 @@ Cluster mode changes execution placement and physical execution strategy. It doe Existing local DataFusion sessions remain valid. Cluster mode is additive. -Implementations may initially expose cluster mode as experimental or capability-gated. A cluster backend may report unknown or uncovered coverage for existing InQL features until worker-side deployment, Substrait support, and runtime behavior are proven. +Implementations may initially expose cluster mode as experimental or capability-gated. A cluster backend may report unknown or uncovered coverage for existing IncQL features until worker-side deployment, Substrait support, and runtime behavior are proven. Streaming cluster execution may remain experimental after bounded cluster execution is available. A backend can support bounded `LazyFrame[T]` cluster jobs while reporting uncovered or unknown coverage for `DataStream[T]` jobs. -Docs that say distributed execution, cluster scheduling, or shuffle are fully out of scope for InQL should be revised once this RFC is accepted. The better boundary is: cluster scheduling mechanics are out of InQL semantics, but cluster execution backend mode is part of the `Session` execution architecture. +Docs that say distributed execution, cluster scheduling, or shuffle are fully out of scope for IncQL should be revised once this RFC is accepted. The better boundary is: cluster scheduling mechanics are out of IncQL semantics, but cluster execution backend mode is part of the `Session` execution architecture. ## Alternatives considered -- **Leave cluster execution entirely to operational layers.** Rejected because InQL's `Session` already owns backend selection, execution, collection, and writes. Operational layers may scope and inject sessions, but the cluster backend boundary belongs under `Session`. +- **Leave cluster execution entirely to operational layers.** Rejected because IncQL's `Session` already owns backend selection, execution, collection, and writes. Operational layers may scope and inject sessions, but the cluster backend boundary belongs under `Session`. - **Create a separate `ClusterSession` type.** Rejected because it splits the author-facing execution model and makes local versus cluster execution look semantically different. -- **Make Ballista the normative backend.** Rejected because Ballista is a strong proof target, not the portable contract. InQL should support other cluster-capable backends when they can satisfy the same boundary. -- **Use raw SQL submission for cluster execution.** Rejected because raw SQL is not an InQL `Session` escape hatch and would bypass typed logical intent, registry metadata, and Substrait contracts. -- **Expose every scheduler knob as portable InQL API.** Rejected because backend-specific tuning belongs in backend options and evidence, not in the portable query surface. +- **Make Ballista the normative backend.** Rejected because Ballista is a strong proof target, not the portable contract. IncQL should support other cluster-capable backends when they can satisfy the same boundary. +- **Use raw SQL submission for cluster execution.** Rejected because raw SQL is not an IncQL `Session` escape hatch and would bypass typed logical intent, registry metadata, and Substrait contracts. +- **Expose every scheduler knob as portable IncQL API.** Rejected because backend-specific tuning belongs in backend options and evidence, not in the portable query surface. - **Treat streams as just long bounded jobs.** Rejected because unbounded inputs need lifecycle, checkpoint, watermark, offset, and sink semantics that finite jobs do not. ## Drawbacks @@ -283,17 +283,17 @@ Docs that say distributed execution, cluster scheduling, or shuffle are fully ou ## Layers affected -- **InQL specification** — must define cluster execution as a backend mode without changing query semantics. -- **InQL library package** — should expose cluster backend selection and typed configuration once implementation begins. +- **IncQL specification** — must define cluster execution as a backend mode without changing query semantics. +- **IncQL library package** — should expose cluster backend selection and typed configuration once implementation begins. - **Execution / interchange** — adapters must preserve Substrait, registry metadata, read-root binding, coverage, and observation contracts across cluster submission. Streaming adapters must additionally report lifecycle, checkpoint, offset, watermark, and sink-commit behavior where applicable. -- **Documentation** — RFC 004 and execution-context docs should distinguish "cluster scheduling mechanics are not InQL semantics" from "cluster backend mode is supported by the session architecture." +- **Documentation** — RFC 004 and execution-context docs should distinguish "cluster scheduling mechanics are not IncQL semantics" from "cluster backend mode is supported by the session architecture." ## Unresolved questions - What public backend type name should the first cluster proof target use: `DataFusionCluster`, `Ballista`, or a more general scheduler-backed DataFusion selection? - Which cluster backend options are portable enough for `backends` and which must remain opaque encoded backend options? - What default guardrail should `collect(...)` use in cluster mode for large results? -- How should worker-side deployment of InQL-owned UDFs and adapter callbacks be represented in coverage records? +- How should worker-side deployment of IncQL-owned UDFs and adapter callbacks be represented in coverage records? - Which execution observations are required before cluster mode can move from experimental to planned? - Should cluster execution require a stable storage profile abstraction before implementation, or can the first proof target use backend-specific storage options? - Which streaming lifecycle states and capability names are required before `DataStream[T]` cluster execution can move beyond experimental? diff --git a/docs/rfcs/050_addon_component_registry.md b/docs/rfcs/050_addon_component_registry.md index a7c4903f..7ec3eef1 100644 --- a/docs/rfcs/050_addon_component_registry.md +++ b/docs/rfcs/050_addon_component_registry.md @@ -1,42 +1,42 @@ -# InQL RFC 050: Addon component registry and package contract +# IncQL RFC 050: Addon component registry and package contract - **Status:** Draft - **Created:** 2026-07-11 - **Author(s):** Danny Meijer (@dannymeijer) - **Related:** - - InQL RFC 002 (Apache Substrait integration) - - InQL RFC 004 (execution context) - - InQL RFC 007 (Prism logical planning and optimization engine) - - InQL RFC 009 (session format handler registry) - - InQL RFC 014 (function registry and catalog governance) - - InQL RFC 027 (relational evidence program) - - InQL RFC 032 (execution observations) - - InQL RFC 033 (adapter requirements and coverage) - - InQL RFC 040 (interoperability semantic profiles) - - InQL RFC 041 (Prism plan ingress and external client frontends) - - InQL RFC 048 (cluster execution backend mode) -- **Issue:** [InQL #101](https://github.com/encero-systems/InQL/issues/101) -- **RFC PR:** [InQL #102](https://github.com/encero-systems/InQL/pull/102) -- **Written against:** Incan 0.4.0 and InQL 0.1.0 + - IncQL RFC 002 (Apache Substrait integration) + - IncQL RFC 004 (execution context) + - IncQL RFC 007 (Prism logical planning and optimization engine) + - IncQL RFC 009 (session format handler registry) + - IncQL RFC 014 (function registry and catalog governance) + - IncQL RFC 027 (relational evidence program) + - IncQL RFC 032 (execution observations) + - IncQL RFC 033 (adapter requirements and coverage) + - IncQL RFC 040 (interoperability semantic profiles) + - IncQL RFC 041 (Prism plan ingress and external client frontends) + - IncQL RFC 048 (cluster execution backend mode) +- **Issue:** [IncQL #101](https://github.com/encero-systems/IncQL/issues/101) +- **RFC PR:** [IncQL #102](https://github.com/encero-systems/IncQL/pull/102) +- **Written against:** Incan 0.4.0 and IncQL 0.1.0 - **Shipped in:** — ## Summary -This RFC defines the package and registry contract through which ordinary Incan packages can extend InQL with data connectors, compute runtimes, plan-ingress frontends, and evidence providers. An addon package may provide several components, but each component must be registered, identified, selected, inspected, and versioned independently. InQL must keep pure component descriptors separate from executable hooks, use open namespaced component identifiers instead of backend or source enums, freeze a registry snapshot at an execution or frontend boundary, and keep read/write components separate from the runtime that computes a plan. DataFusion remains the built-in default runtime and first registry-shaped implementation; DuckDB is the first external proof target rather than a privileged core backend. +This RFC defines the package and registry contract through which ordinary Incan packages can extend IncQL with data connectors, compute runtimes, plan-ingress frontends, and evidence providers. An addon package may provide several components, but each component must be registered, identified, selected, inspected, and versioned independently. IncQL must keep pure component descriptors separate from executable hooks, use open namespaced component identifiers instead of backend or source enums, freeze a registry snapshot at an execution or frontend boundary, and keep read/write components separate from the runtime that computes a plan. DataFusion remains the built-in default runtime and first registry-shaped implementation; DuckDB is the first external proof target rather than a privileged core backend. ## Motivation -InQL's current execution path is appropriate for proving DataFusion integration, but it does not scale to an ecosystem. Adding another backend currently tends to require edits to core backend enums, source enums, session dispatch, public exports, the core package manifest, and source registration logic. That makes every integration a core feature and encourages one backend object to absorb source reading, sink writing, computation, capability reporting, and configuration. +IncQL's current execution path is appropriate for proving DataFusion integration, but it does not scale to an ecosystem. Adding another backend currently tends to require edits to core backend enums, source enums, session dispatch, public exports, the core package manifest, and source registration logic. That makes every integration a core feature and encourages one backend object to absorb source reading, sink writing, computation, capability reporting, and configuration. The desired platform is broader. PostgreSQL may be a bounded reader and writer while DuckDB performs local computation, or PostgreSQL may also provide a SQL-pushdown runtime. Kafka is primarily a streaming connector. Spark may provide a compute runtime and a Spark Connect ingress frontend without owning the underlying storage. Snowflake may provide table connectors, warehouse compute, Snowpipe writers, streaming ingestion, catalog access, and observations as separate components in one package. Open table formats may provide connector and catalog components without providing compute at all. Treating each platform as one monolithic adapter hides those differences and makes composition accidental. Treating Substrait as the plugin contract is also insufficient: Substrait carries logical plans, but it does not install code, bind credentials, inspect catalogs, negotiate connector/runtime compatibility, manage streams, or report component metadata. -InQL therefore needs a stable addon boundary before more vendor integrations are implemented. The boundary must fit Incan's package, manifest, feature, and lockfile model; preserve Prism as the owner of relational meaning; and integrate with adapter coverage, semantic profiles, execution observations, and plan ingress without merging those layers. +IncQL therefore needs a stable addon boundary before more vendor integrations are implemented. The boundary must fit Incan's package, manifest, feature, and lockfile model; preserve Prism as the owner of relational meaning; and integrate with adapter coverage, semantic profiles, execution observations, and plan ingress without merging those layers. ## Goals -- Define ordinary Incan packages as the installation and sideloading unit for InQL addons. +- Define ordinary Incan packages as the installation and sideloading unit for IncQL addons. - Define independently registered data connector, compute runtime, plan ingress, and evidence provider component kinds. - Replace closed backend and source identities with stable, namespaced component identifiers and open source, sink, runtime-mode, and capability keys. - Separate pure, inspectable component descriptors from executable component hooks and live runtime state. @@ -70,18 +70,18 @@ An addon is an ordinary Incan package dependency. Local development and private ```toml [dependencies] -inql = { path = "../InQL" } -inql-duckdb = { path = "../inql-duckdb" } -inql-postgres = { path = "../inql-postgres" } +incql = { path = "../IncQL" } +incql-duckdb = { path = "../incql-duckdb" } +incql-postgres = { path = "../incql-postgres" } ``` The eventual registry-resolved form uses the same package contract: ```toml [dependencies] -inql = "0.1" -inql-duckdb = { version = "0.1", features = ["httpfs", "iceberg"] } -inql-postgres = { version = "0.1", features = ["copy", "logical-replication"] } +incql = "0.1" +incql-duckdb = { version = "0.1", features = ["httpfs", "iceberg"] } +incql-postgres = { version = "0.1", features = ["copy", "logical-replication"] } ``` Depending on or importing a package makes its checked code and descriptor metadata available. It must not silently mutate a process-global registry or change which runtime a Session uses. @@ -91,9 +91,9 @@ Depending on or importing a package makes its checked code and descriptor metada Addon packages expose typed component constructors. Authors install the components they intend to use and select the compute runtime explicitly: ```incan -from pub::inql import AddonRegistry, Session -from pub::inql_duckdb import duckdb -from pub::inql_postgres import postgres +from pub::incql import AddonRegistry, Session +from pub::incql_duckdb import duckdb +from pub::incql_postgres import postgres registry = AddonRegistry.builtins() registry.install(duckdb.files())? @@ -117,11 +117,11 @@ orders = postgres.table("analytics.orders") session.register("orders", orders)? ``` -This does not imply that DuckDB can consume every PostgreSQL binding. Before execution, InQL must ask the connector and selected runtime whether they have a compatible binding path and must evaluate the plan's adapter requirements. If no compatible route exists, the Session reports structured uncovered or unknown coverage rather than silently changing runtime or materializing through an undocumented fallback. +This does not imply that DuckDB can consume every PostgreSQL binding. Before execution, IncQL must ask the connector and selected runtime whether they have a compatible binding path and must evaluate the plan's adapter requirements. If no compatible route exists, the Session reports structured uncovered or unknown coverage rather than silently changing runtime or materializing through an undocumented fallback. ### One package, several components -An addon package is a distribution unit, not a semantic component. For example, an `inql-snowflake` package may expose separate table connector, warehouse runtime, Snowpipe writer, Snowpipe Streaming writer, catalog inspector, and observation provider components. An `inql-spark` package may expose a Spark runtime and Spark Connect ingress frontend. Installing one component must not imply that the others are installed or selected. +An addon package is a distribution unit, not a semantic component. For example, an `incql-snowflake` package may expose separate table connector, warehouse runtime, Snowpipe writer, Snowpipe Streaming writer, catalog inspector, and observation provider components. An `incql-spark` package may expose a Spark runtime and Spark Connect ingress frontend. Installing one component must not imply that the others are installed or selected. This separation also allows a package to expose convenient bundles without collapsing component identity: @@ -148,15 +148,15 @@ A descriptor's capability declaration is discoverability metadata. It is not pro ### Terminology and ownership -An **addon package** is an Incan package that depends on InQL and exports one or more addon components. +An **addon package** is an Incan package that depends on IncQL and exports one or more addon components. -An **addon component** is one independently identified implementation of a known InQL extension boundary. Components are the unit of registration, lookup, compatibility checks, inspection, and runtime selection. +An **addon component** is one independently identified implementation of a known IncQL extension boundary. Components are the unit of registration, lookup, compatibility checks, inspection, and runtime selection. A **component descriptor** is serializable metadata describing one component. A descriptor must be inspectable without opening network connections, resolving credentials, loading data, or creating backend-native runtime state. An **executable component** is checked code that implements the hook contract for exactly one component kind. Executable hooks and live resources are not part of the descriptor. -An **addon registry** contains descriptors and executable component factories or implementations under the same component identities. A registry is scoped to the Session, frontend service, inspection operation, or other explicit owner that receives it. InQL must not rely on an implicitly mutable process-global registry. +An **addon registry** contains descriptors and executable component factories or implementations under the same component identities. A registry is scoped to the Session, frontend service, inspection operation, or other explicit owner that receives it. IncQL must not rely on an implicitly mutable process-global registry. An **addon bundle** is a convenience value containing multiple independently registered components. A bundle is not a component kind and does not receive one combined identity. @@ -164,7 +164,7 @@ Prism remains the owner of analyzed relational meaning. Components may supply bi ### Component kinds -InQL must define these top-level component kinds: +IncQL must define these top-level component kinds: | Component kind | Responsibility | Representative surfaces | | --- | --- | --- | @@ -181,19 +181,19 @@ Plan ingress is not a compute runtime. Evidence provision is not plan analysis. ### Component identity and versions -Each component descriptor must contain a stable component id. Component ids must be lowercase, dot-separated, namespaced ASCII identifiers such as `inql.datafusion.local`, `inql.duckdb.files`, or `vendor.postgres.tables`. Package names and component ids need not be identical because one package may publish several components. +Each component descriptor must contain a stable component id. Component ids must be lowercase, dot-separated, namespaced ASCII identifiers such as `incql.datafusion.local`, `incql.duckdb.files`, or `vendor.postgres.tables`. Package names and component ids need not be identical because one package may publish several components. A component id identifies a contract across compatible package releases. A package must not reuse an id for a component of another kind or for behavior that violates the previous component contract. Versioning must distinguish: - addon package version -- InQL addon API version required by the component +- IncQL addon API version required by the component - descriptor schema version - component or provider version when it differs from the package version - external engine or protocol version, which belongs in semantic profiles or runtime observations rather than package identity -The registry must reject a component whose required addon API version is incompatible with the InQL version constructing the registry. +The registry must reject a component whose required addon API version is incompatible with the IncQL version constructing the registry. ### Component descriptor @@ -205,7 +205,7 @@ pub model AddonPackageDescriptor: pub version: str pub source_identity: str pub enabled_features: list[str] - pub requires_inql: str + pub requires_incql: str pub enum AddonComponentKind(str): DataConnector = "data_connector" @@ -230,7 +230,7 @@ pub model AddonComponentDescriptor: pub configuration_schema: str ``` -Field names may be adjusted to fit established InQL naming conventions, but implementations must preserve these facts and distinctions. +Field names may be adjusted to fit established IncQL naming conventions, but implementations must preserve these facts and distinctions. Descriptor surfaces say which hook families a component exposes. Capability declarations say which RFC 033 capability families the component knows how to assess or may cover. They must not use `supported` as a substitute for contextual coverage. Stability labels such as experimental, preview, and stable may qualify a surface, but they must remain distinct from covered, partially covered, uncovered, and unknown. @@ -242,7 +242,7 @@ Descriptor facts must be deterministic for a resolved package and feature set. T ### Executable component contracts -InQL must expose a distinct executable hook contract for each component kind rather than one optional-method object that can do everything. +IncQL must expose a distinct executable hook contract for each component kind rather than one optional-method object that can do everything. A data connector contract must be able to expose the surfaces it declares, including as applicable: @@ -273,7 +273,7 @@ The exact Incan mechanism for executable hooks may be nominal interfaces, checke ### Registry lifecycle and registration -`AddonRegistry.builtins()` must return a registry containing InQL's built-in component descriptors and executable implementations. `AddonRegistry.empty()` may be provided for tooling or controlled environments. +`AddonRegistry.builtins()` must return a registry containing IncQL's built-in component descriptors and executable implementations. `AddonRegistry.empty()` may be provided for tooling or controlled environments. Installing a component must: @@ -295,7 +295,7 @@ Frontend services and inspection tools may own registry snapshots without constr ### Open source, sink, and runtime descriptors -Core InQL must replace closed source and backend enums with open descriptors. The normative conceptual shapes are: +Core IncQL must replace closed source and backend enums with open descriptors. The normative conceptual shapes are: ```incan pub model AddonOption: @@ -323,7 +323,7 @@ pub model RuntimeSelection: `value_schema` must make option interpretation explicit. A future typed configuration-value carrier may replace the textual storage field without changing the requirement that values have declared schemas. -`source_kind`, `sink_kind`, and `runtime_mode` are component-owned open keys. Core InQL must not add an enum variant for every external format, table type, topic type, warehouse mode, or runtime. +`source_kind`, `sink_kind`, and `runtime_mode` are component-owned open keys. Core IncQL must not add an enum variant for every external format, table type, topic type, warehouse mode, or runtime. A source or sink descriptor must identify the connector that owns its interpretation. A descriptor must not select the compute runtime. A runtime selection must identify an installed `compute_runtime` component and must not imply installation of any connector from the same package. @@ -339,7 +339,7 @@ A compute runtime must declare which binding protocols it can consume or produce When a package provides both a connector and compute runtime, the runtime may consume a package-native binding protocol. That does not merge the two components. For example, a PostgreSQL table connector and PostgreSQL SQL-pushdown runtime may cooperate directly while remaining separately selectable and inspectable. -If no compatible binding path is available, execution must fail before data movement when possible with a structured binding or coverage diagnostic. InQL must not assume an Arrow materialization fallback, silently choose another runtime, or ask the connector to select a runtime. +If no compatible binding path is available, execution must fail before data movement when possible with a structured binding or coverage diagnostic. IncQL must not assume an Arrow materialization fallback, silently choose another runtime, or ask the connector to select a runtime. Sink compatibility must include commit and failure semantics. A runtime that can compute a result is not automatically able to commit it to an installed sink. Connector-owned commit, abort, retry, replay, and streaming guarantees must remain visible in coverage and observations. @@ -369,18 +369,18 @@ Evidence providers are non-authoritative sources under RFC 027. Registration gra ### Package, feature, and lockfile contract -Addon distribution must use Incan package resolution. Local path dependencies are the first required sideloading mechanism. Git and registry-resolved dependencies may be added by the Incan package system without changing the InQL component contract. +Addon distribution must use Incan package resolution. Local path dependencies are the first required sideloading mechanism. Git and registry-resolved dependencies may be added by the Incan package system without changing the IncQL component contract. -Addon-specific Rust dependencies, native build requirements, and generated bindings belong to the addon package rather than core InQL unless they are required by a built-in component. +Addon-specific Rust dependencies, native build requirements, and generated bindings belong to the addon package rather than core IncQL unless they are required by a built-in component. -Package features may enable or disable component surfaces, optional components, or capability implementations. A resolved descriptor must reflect the actual enabled feature set. Core InQL should not become the primary feature bundle for provider-specific integrations such as `spark`, `snowflake`, or `postgres`. +Package features may enable or disable component surfaces, optional components, or capability implementations. A resolved descriptor must reflect the actual enabled feature set. Core IncQL should not become the primary feature bundle for provider-specific integrations such as `spark`, `snowflake`, or `postgres`. The resolved dependency and lock model must eventually preserve enough information to reproduce addon availability: - package name, version, and source identity - enabled package features - transitive package and Rust dependency resolution -- required InQL addon API version +- required IncQL addon API version - component ids and descriptor fingerprints - native or system requirements when the package manager can represent them @@ -433,7 +433,7 @@ Inspection APIs must distinguish: ### Syntax -This RFC introduces no InQL query syntax. It defines InQL package and library contracts around Session, Prism frontends, and evidence tooling. +This RFC introduces no IncQL query syntax. It defines IncQL package and library contracts around Session, Prism frontends, and evidence tooling. API names in examples are illustrative where the reference-level rules do not prescribe an exact name. The component kinds, identity distinctions, descriptor/runtime separation, explicit registration, frozen registry lifecycle, open binding descriptors, and connector/runtime split are normative. @@ -445,12 +445,12 @@ Addon metadata is evidence and configuration. It is not a truth store for fields Component composition must be explicit. A package relationship, shared vendor name, or shared connection configuration must not be used as an implicit signal that connector, runtime, ingress, or evidence components are installed or compatible. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces - RFC 002 remains the logical interchange boundary. Component registration and physical bindings must not be encoded as ad hoc Substrait semantics. - RFC 004 remains the Session and execution-context contract. This RFC refines its backend reference into a registry-owned runtime selection and its source/sink identifiers into connector-owned descriptors. - RFC 007 keeps Prism as semantic owner. Compute and ingress components feed or execute Prism plans without redefining them. -- RFC 009's format-handler contract, if retained, becomes an implementation strategy or subordinate surface of a data connector. InQL must not maintain a second independent plugin root that can bypass component identity, package provenance, registry freezing, or connector/runtime compatibility checks. +- RFC 009's format-handler contract, if retained, becomes an implementation strategy or subordinate surface of a data connector. IncQL must not maintain a second independent plugin root that can bypass component identity, package provenance, registry freezing, or connector/runtime compatibility checks. - RFC 014's declaration-side function registry is prior art for stable ids and inspectable metadata, but addon registration is explicitly owner-scoped because components carry resources and external behavior. - RFC 032 observations must identify the components involved in execution and I/O. - RFC 033 owns requirement and coverage semantics; this RFC supplies stable component identity and declarations used by those records. @@ -460,7 +460,7 @@ Component composition must be explicit. A package relationship, shared vendor na ### Compatibility / migration -`Session.default()` remains source-compatible and continues to use DataFusion. Internally it must build or receive the built-in registry, install `inql.datafusion.files` and `inql.datafusion.local`, and select the local runtime. +`Session.default()` remains source-compatible and continues to use DataFusion. Internally it must build or receive the built-in registry, install `incql.datafusion.files` and `incql.datafusion.local`, and select the local runtime. Existing `csv_source`, `parquet_source`, `arrow_source`, `read_csv`, `read_parquet`, `read_arrow`, `write_csv`, and `write_parquet` helpers may remain as compatibility conveniences. They must construct connector-owned source or sink descriptors and route through the registry path. @@ -478,7 +478,7 @@ DataFusion must be migrated first as a built-in file connector and local compute - **Use Substrait as the addon API.** Rejected because plan interchange does not install code or define binding, lifecycle, credentials, coverage, and package contracts. - **Auto-register components when a package is imported.** Rejected because import order and hidden global mutation make sessions difficult to reproduce, inspect, test, and isolate. - **Load arbitrary shared libraries or addon directories at runtime.** Rejected for the initial contract because it bypasses Incan package resolution, checked types, lockfiles, and ordinary dependency review. -- **Put every integration behind features on core `inql`.** Rejected because it centralizes provider dependencies and release cadence in core and prevents independent addon packages. +- **Put every integration behind features on core `incql`.** Rejected because it centralizes provider dependencies and release cadence in core and prevents independent addon packages. - **Assume Arrow makes all connectors and runtimes interchangeable.** Rejected because streaming, pushdown, catalogs, native types, transactionality, sink commits, and operational constraints require contextual compatibility checks. ## Drawbacks @@ -492,8 +492,8 @@ DataFusion must be migrated first as a built-in file connector and local compute ## Layers affected -- **InQL specification** — addon component kinds, identities, descriptors, registration, selection, and composition become normative contracts. -- **InQL library package** — Session, source/sink descriptors, runtime selection, inspection, and built-in DataFusion integration must use the registry-shaped path. +- **IncQL specification** — addon component kinds, identities, descriptors, registration, selection, and composition become normative contracts. +- **IncQL library package** — Session, source/sink descriptors, runtime selection, inspection, and built-in DataFusion integration must use the registry-shaped path. - **Incan compiler and package tooling** — package dependencies, feature resolution, checked component metadata, hook dispatch, and lockfile facts must support addon packages as the implementation matures. - **Execution / interchange** — connectors and runtimes must negotiate bindings and report coverage without changing Prism or Substrait semantics. - **Documentation** — execution, addon authoring, sideloading, capability inspection, and migration docs must explain package availability, component installation, and runtime selection separately. diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index 5193fa28..ce62ec55 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -1,8 +1,8 @@ -# InQL RFCs +# IncQL RFCs -InQL uses its **own** RFC series (starting at 000), independent of the [Incan language RFCs][incan-rfcs]. +IncQL uses its **own** RFC series (starting at 000), independent of the [Incan language RFCs][incan-rfcs]. -**New RFC:** copy [TEMPLATE.md], name the file `NNN_short_slug.md`, pick the next number from the table (or from open issues), and open a PR. Section order and header fields follow that template. For workflow and conventions, see [Writing InQL RFCs]. +**New RFC:** copy [TEMPLATE.md], name the file `NNN_short_slug.md`, pick the next number from the table (or from open issues), and open a PR. Section order and header fields follow that template. For workflow and conventions, see [Writing IncQL RFCs]. | RFC | Status | Title | | | -------------- | ----------- | ------------------------------------------------------------------------------------------------- | --- | @@ -59,20 +59,20 @@ InQL uses its **own** RFC series (starting at 000), independent of the [Incan la -**v0.1 tracking:** RFCs 000–004 plus RFC 007 remain the foundation that defines when InQL v0.1 is complete: authors can read data, write typed queries, lower through Prism to Substrait, execute through DataFusion, and write results. The table also marks additional v0.1-shipped slices that landed before the whole foundation is closed, including the function catalog and evidence/observation work. +**v0.1 tracking:** RFCs 000–004 plus RFC 007 remain the foundation that defines when IncQL v0.1 is complete: authors can read data, write typed queries, lower through Prism to Substrait, execute through DataFusion, and write results. The table also marks additional v0.1-shipped slices that landed before the whole foundation is closed, including the function catalog and evidence/observation work. -New RFCs should follow [TEMPLATE.md] (aligned with Incan’s RFC structure, adapted for InQL). +New RFCs should follow [TEMPLATE.md] (aligned with Incan’s RFC structure, adapted for IncQL). [TEMPLATE.md]: TEMPLATE.md -[Writing InQL RFCs]: ../contributing/writing_rfcs.md -[rfc-000]: 000_inql_syntax.md -[rfc-001]: 001_inql_dataset.md +[Writing IncQL RFCs]: ../contributing/writing_rfcs.md +[rfc-000]: 000_incql_syntax.md +[rfc-001]: 001_incql_dataset.md [rfc-002]: 002_apache_substrait_integration.md -[rfc-003]: 003_inql_query_blocks.md -[rfc-004]: 004_inql_execution_context.md -[rfc-005]: 005_inql_pipe_forward.md +[rfc-003]: 003_incql_query_blocks.md +[rfc-004]: 004_incql_execution_context.md +[rfc-005]: 005_incql_pipe_forward.md [rfc-006]: 006_unnest_core_substrait.md [rfc-007]: 007_prism_planning_engine.md [rfc-008]: 008_optimizer_boundary_stats_cbo_aqe.md diff --git a/docs/rfcs/TEMPLATE.md b/docs/rfcs/TEMPLATE.md index 90fd4cb6..f229b069 100644 --- a/docs/rfcs/TEMPLATE.md +++ b/docs/rfcs/TEMPLATE.md @@ -1,10 +1,10 @@ -# InQL RFC template +# IncQL RFC template -> Use this template for RFCs in [`docs/rfcs/`](README.md). Keep each RFC focused: one coherent proposal, with clear motivation, precise semantics, and where work lands (**this package**, **Incan compiler**, **execution layer**). For workflow and tips, see [Writing InQL RFCs](../contributing/writing_rfcs.md). +> Use this template for RFCs in [`docs/rfcs/`](README.md). Keep each RFC focused: one coherent proposal, with clear motivation, precise semantics, and where work lands (**this package**, **Incan compiler**, **execution layer**). For workflow and tips, see [Writing IncQL RFCs](../contributing/writing_rfcs.md). ## Title -InQL RFC NNN: \ +IncQL RFC NNN: \ - **Status:** Draft - **Created:** \ - **Author(s):** \ -- **Related:** — +- **Related:** — - **Issue:** \ - **RFC PR:** \ - **Written against:** \ -- **Shipped in:** — +- **Shipped in:** — ## Summary @@ -49,7 +49,7 @@ Explain the problem and why it matters: ## Guide-level explanation (how authors think about it) -Explain the feature the way an InQL author would reason about it. Prefer concrete examples. +Explain the feature the way an IncQL author would reason about it. Prefer concrete examples. ```incan # Example @@ -74,7 +74,7 @@ New or changed syntax, if any. Precise behavior. -### Interaction with other InQL surfaces +### Interaction with other IncQL surfaces How this composes with: @@ -99,9 +99,9 @@ Complexity, performance, mental model, or maintenance cost. Describe which parts of the system this RFC impacts, in **normative** language (`must`, `must not`, `should`). Do not turn this into a task list or internal file manifest — that belongs in implementation issues and PRs. -- **InQL specification** — coherence with other `docs/rfcs/` documents. -- **InQL library package** — public `.incn` API, tests, manifests. -- **Incan compiler** — parsing, checking, lowering, or diagnostics for InQL constructs (work may live in the Incan repository). +- **IncQL specification** — coherence with other `docs/rfcs/` documents. +- **IncQL library package** — public `.incn` API, tests, manifests. +- **Incan compiler** — parsing, checking, lowering, or diagnostics for IncQL constructs (work may live in the Incan repository). - **Execution / interchange** — Substrait consumers, session, adapters, runtime (often specified relative to an execution context RFC). - **Documentation** — README, architecture notes, contributor docs. diff --git a/docs/whitepapers/README.md b/docs/whitepapers/README.md index 3a5a6d44..5f1013b5 100644 --- a/docs/whitepapers/README.md +++ b/docs/whitepapers/README.md @@ -1,9 +1,9 @@ -# InQL whitepapers +# IncQL whitepapers -This directory holds non-normative design papers for larger InQL product and architecture directions. +This directory holds non-normative design papers for larger IncQL product and architecture directions. Whitepapers are intentionally broader than RFCs. They frame a problem, name a north star, and identify design pressure before a specific normative proposal is ready. When a whitepaper direction becomes implementation work, split the actionable contracts into focused RFCs under `docs/rfcs/`. ## Papers -- [InQL-DB: local ACID analytical memory for InQL applications](inql_db.md) +- [IncQL-DB: local ACID analytical memory for IncQL applications](incql_db.md) diff --git a/docs/whitepapers/inql_db.md b/docs/whitepapers/incql_db.md similarity index 74% rename from docs/whitepapers/inql_db.md rename to docs/whitepapers/incql_db.md index 70f92a1b..30165273 100644 --- a/docs/whitepapers/inql_db.md +++ b/docs/whitepapers/incql_db.md @@ -1,36 +1,36 @@ -# InQL-DB: local ACID analytical memory for InQL applications +# IncQL-DB: local ACID analytical memory for IncQL applications **Status:** Exploratory whitepaper **Date:** 2026-05-06 -**Audience:** InQL and Incan contributors, application authors evaluating local analytical state, and implementers of future InQL execution backends. +**Audience:** IncQL and Incan contributors, application authors evaluating local analytical state, and implementers of future IncQL execution backends. -**Scope:** This document is non-normative. It defines a product and architecture north star for InQL-DB. Normative behavior should later be split into focused InQL RFCs for backend selection, storage layout, transaction semantics, vector search, and CLI contracts. +**Scope:** This document is non-normative. It defines a product and architecture north star for IncQL-DB. Normative behavior should later be split into focused IncQL RFCs for backend selection, storage layout, transaction semantics, vector search, and CLI contracts. ## Thesis -InQL-DB is an embedded, directory-backed, ACID, columnar store for typed InQL data logic. +IncQL-DB is an embedded, directory-backed, ACID, columnar store for typed IncQL data logic. It is designed for app-local memory, retrieval-augmented generation stores, edge analytics, durable local caches, and small analytical datasets that should live inside an application without a database server, SQL dependency, or heavyweight runtime stack. The product shape is closer to SQLite-style local ownership plus DuckDB-style analytical execution plus Delta-inspired transaction logs than to a general DuckDB clone. -InQL-DB should be opened directly by an application: +IncQL-DB should be opened directly by an application: ```incan -let db = InQLDB.open("./agent_memory.inqldb") +let db = IncQLDB.open("./agent_memory.incqldb") ``` -and inspected with the InQL CLI: +and inspected with the IncQL CLI: ```bash -inql db inspect ./agent_memory.inqldb +incql db inspect ./agent_memory.incqldb ``` -## Why this belongs in InQL +## Why this belongs in IncQL -InQL already owns the typed relational authoring layer: +IncQL already owns the typed relational authoring layer: - Incan `model` types provide row shapes. - `DataSet[T]`, `LazyFrame[T]`, `DataFrame[T]`, and `DataStream[T]` define carrier semantics. @@ -38,7 +38,7 @@ InQL already owns the typed relational authoring layer: - Substrait is the portable logical boundary. - `Session` owns execution and binding. -InQL-DB should therefore be an InQL execution and storage backend, not a separate query language. Authors should use InQL's DSL and carrier APIs. InQL-DB should consume the resulting logical plans and provide a local physical data plane. +IncQL-DB should therefore be an IncQL execution and storage backend, not a separate query language. Authors should use IncQL's DSL and carrier APIs. IncQL-DB should consume the resulting logical plans and provide a local physical data plane. No SQL support is required for the core product. @@ -46,19 +46,19 @@ No SQL support is required for the core product. ### Embedded and zero-admin -InQL-DB should run in-process with the host Incan application. It should not require a daemon, local service, external coordinator, warehouse, object store, or administrator-owned catalog. +IncQL-DB should run in-process with the host Incan application. It should not require a daemon, local service, external coordinator, warehouse, object store, or administrator-owned catalog. DuckDB demonstrates the value of this shape for analytical workloads. DuckDB describes itself as having "zero external dependencies" and running "in-process in its host application or as a single binary." SQLite demonstrates the same deployment principle for transactional embedded storage: "There is no intermediary server process." -InQL-DB should inherit that operational model while remaining native to InQL. +IncQL-DB should inherit that operational model while remaining native to IncQL. ### Directory-backed by default -An InQL-DB database should always be a directory ending in `.inqldb/`. +An IncQL-DB database should always be a directory ending in `.incqldb/`. ```text -agent_memory.inqldb/ - _inql_log/ +agent_memory.incqldb/ + _incql_log/ catalog/ tables/ indexes/ @@ -67,21 +67,21 @@ agent_memory.inqldb/ LOCK ``` -A directory gives InQL-DB room for immutable column segments, transaction logs, checkpoints, vector indexes, blobs, temporary files, and future compaction artifacts without overloading a single monolithic file format too early. +A directory gives IncQL-DB room for immutable column segments, transaction logs, checkpoints, vector indexes, blobs, temporary files, and future compaction artifacts without overloading a single monolithic file format too early. -### InQL and Substrait only +### IncQL and Substrait only The core query surfaces are: -- InQL DSL and `DataSet[T]` APIs for authors. +- IncQL DSL and `DataSet[T]` APIs for authors. - Prism for internal logical plan state. - Substrait for portable logical interchange where needed. -SQL is not part of the core. A future compatibility package may translate a SQL dialect into InQL or Substrait, but that is outside the InQL-DB identity. +SQL is not part of the core. A future compatibility package may translate a SQL dialect into IncQL or Substrait, but that is outside the IncQL-DB identity. ### ACID from the start -InQL-DB should be a real database, not a collection of best-effort files. +IncQL-DB should be a real database, not a collection of best-effort files. The north star is full ACID semantics for local embedded use: @@ -95,7 +95,7 @@ The initial concurrency model should remain deliberately small: single writer, m ### Analytical and retrieval-native -InQL-DB should be optimized for local analytical scans and retrieval workflows: +IncQL-DB should be optimized for local analytical scans and retrieval workflows: - Columnar scans over typed vectors. - Predicate and projection pushdown. @@ -114,7 +114,7 @@ The target use cases include: ## Non-goals -InQL-DB should not initially attempt to be: +IncQL-DB should not initially attempt to be: - a DuckDB-compatible engine, - a SQL dialect, @@ -134,16 +134,16 @@ Those systems are useful references, not compatibility targets. Incan models │ ▼ -InQL carriers / query DSL +IncQL carriers / query DSL │ ▼ Prism logical planning │ ▼ -Substrait / InQL logical plan boundary +Substrait / IncQL logical plan boundary │ ▼ -InQL-DB backend +IncQL-DB backend │ ├── transaction manager ├── catalog @@ -153,22 +153,22 @@ InQL-DB backend └── recovery / compaction ``` -The important boundary is that InQL-DB owns physical execution and local storage. It does not own authoring semantics. +The important boundary is that IncQL-DB owns physical execution and local storage. It does not own authoring semantics. ## Storage model -InQL-DB should take direct inspiration from Delta-style storage: immutable columnar data files plus an append-only transaction log that defines the current table state. +IncQL-DB should take direct inspiration from Delta-style storage: immutable columnar data files plus an append-only transaction log that defines the current table state. Delta Lake is relevant because it combines versioned columnar files with a transaction log. Delta documentation describes a table as data files plus "a transaction log that stores metadata about the transactions." Delta's FAQ describes ACID support through "versioned Parquet files" and "a transaction log to keep track of all the commits." -InQL-DB should adapt that pattern for local embedded use. +IncQL-DB should adapt that pattern for local embedded use. ### Directory layout ```text -agent_memory.inqldb/ +agent_memory.incqldb/ LOCK - _inql_log/ + _incql_log/ 00000000000000000000.commit 00000000000000000001.commit 00000000000000000064.checkpoint @@ -189,7 +189,7 @@ agent_memory.inqldb/ tmp/ ``` -This layout is intentionally database-directory-first rather than table-directory-first. Delta's table model is useful, but InQL-DB needs a single local database artifact that can hold many tables, indexes, and catalog entries. +This layout is intentionally database-directory-first rather than table-directory-first. Delta's table model is useful, but IncQL-DB needs a single local database artifact that can hold many tables, indexes, and catalog entries. ### Commit log @@ -226,17 +226,17 @@ Segments should be Parquet-inspired: - checksums, - footer metadata. -Apache Parquet is the right reference point because it is a column-oriented format designed for efficient storage and retrieval, with compression and encoding schemes. InQL-DB should borrow those physical ideas while keeping its own format contract. +Apache Parquet is the right reference point because it is a column-oriented format designed for efficient storage and retrieval, with compression and encoding schemes. IncQL-DB should borrow those physical ideas while keeping its own format contract. The internal format should be called something like `iqseg`, not Parquet, unless the project deliberately chooses full Parquet compatibility later. -Reasons to keep an InQL-owned segment format: +Reasons to keep an IncQL-owned segment format: -- InQL type metadata can be represented directly. +- IncQL type metadata can be represented directly. - Fixed-dimension vectors can be encoded as a native physical type. - Vector index references can be tied to segment versions. - Delete/tombstone handling can be optimized for local MVCC. -- Format evolution can follow InQL-DB's own protocol. +- Format evolution can follow IncQL-DB's own protocol. - The implementation can start smaller than full Parquet. ### Catalog @@ -257,7 +257,7 @@ Catalog changes are committed through the same log as data changes. There should ## Transaction model -InQL-DB should target local ACID with single-writer, multi-reader MVCC. +IncQL-DB should target local ACID with single-writer, multi-reader MVCC. ### Write path @@ -266,7 +266,7 @@ InQL-DB should target local ACID with single-writer, multi-reader MVCC. 3. Write new segment/index files under `tmp/`. 4. Flush and checksum new files. 5. Move files into their final content-addressed or version-addressed locations. -6. Write a commit file under `_inql_log/` with all add/remove/schema actions. +6. Write a commit file under `_incql_log/` with all add/remove/schema actions. 7. Flush the commit file and containing directory. 8. Release the writer lock. @@ -297,7 +297,7 @@ Recovery must not require application-specific code. ## Execution model -InQL-DB execution should be vectorized. +IncQL-DB execution should be vectorized. Core runtime types: @@ -327,7 +327,7 @@ Execution should work in batches so operators can stream chunks without material Vector support is part of the product center, not an extension afterthought. -InQL-DB should support: +IncQL-DB should support: - fixed-dimension vector columns, - distance functions such as cosine, dot product, and L2, @@ -348,7 +348,7 @@ model Memory: source: str created_at: Instant -let db = InQLDB.open("./agent_memory.inqldb") +let db = IncQLDB.open("./agent_memory.incqldb") let hits = db.table[Memory]("memories") .filter(.source == "docs") @@ -357,13 +357,13 @@ let hits = db.table[Memory]("memories") .collect() ``` -This should lower through InQL planning into an InQL-DB physical plan that can combine metadata filtering and vector search. +This should lower through IncQL planning into an IncQL-DB physical plan that can combine metadata filtering and vector search. ## Governed RAG store Vector search is not enough for agentic retrieval. -InQL-DB should support governed RAG stores as a first-class data pattern: retrieval tables where every returned item carries provenance, approval state, corpus version, retrieval evidence, and policy compatibility metadata. +IncQL-DB should support governed RAG stores as a first-class data pattern: retrieval tables where every returned item carries provenance, approval state, corpus version, retrieval evidence, and policy compatibility metadata. This matters for advisory systems such as Hees.ai, where the retrieval layer is part of the safety model. A retrieved entry is not merely text. It is an approved evidence unit with constraints. @@ -453,7 +453,7 @@ This makes RAG auditable rather than merely semantic. ## HyperQuant evidence-provider ledger -HyperQuant should be treated as an evidence-provider implementation behind InQL-DB and Hees.ai storage contracts, not as the semantic owner of retrieval behavior. +HyperQuant should be treated as an evidence-provider implementation behind IncQL-DB and Hees.ai storage contracts, not as the semantic owner of retrieval behavior. The storage problem is not only vector search. HyperQuant needs a durable audit ledger for evidence-provider runs: @@ -468,7 +468,7 @@ query + package/policy/corpus/index context This distinction matters because governed systems must explain more than the nearest neighbors. They must explain what was considered, what was eligible, what was rejected, and which package, policy, corpus, index, and provider versions controlled the run. -InQL-DB should support these logical records: +IncQL-DB should support these logical records: ```incan model EvidenceProviderRun: @@ -518,7 +518,7 @@ model EvidenceProviderFingerprint: Eligible evidence and rejected evidence may be represented as filtered views over `EvidenceCandidate`, or as separate physical tables if the storage engine needs different retention or indexing behavior. The important contract is that rejected evidence is first-class data, not a log message. -For federated domain runtimes, InQL-DB should also support a run-level grouping record: +For federated domain runtimes, IncQL-DB should also support a run-level grouping record: ```incan model FederatedEvidenceRun: @@ -538,30 +538,30 @@ The governing rule is: Vectors nominate evidence. They do not authorize evidence. ``` -Vector similarity, quantized indexes, and approximate-nearest-neighbor search can propose candidates. Package, policy, corpus, authority, and admissibility rules decide whether those candidates may become evidence. InQL-DB must preserve that boundary in storage so later inspection can distinguish retrieval mechanics from governance decisions. +Vector similarity, quantized indexes, and approximate-nearest-neighbor search can propose candidates. Package, policy, corpus, authority, and admissibility rules decide whether those candidates may become evidence. IncQL-DB must preserve that boundary in storage so later inspection can distinguish retrieval mechanics from governance decisions. ## CLI -The command surface should live under `inql db`. +The command surface should live under `incql db`. ```bash -inql db init ./agent_memory.inqldb -inql db inspect ./agent_memory.inqldb -inql db tables ./agent_memory.inqldb -inql db schema ./agent_memory.inqldb memories -inql db verify ./agent_memory.inqldb -inql db compact ./agent_memory.inqldb -inql db recover ./agent_memory.inqldb -inql db export ./agent_memory.inqldb memories --to memories.parquet +incql db init ./agent_memory.incqldb +incql db inspect ./agent_memory.incqldb +incql db tables ./agent_memory.incqldb +incql db schema ./agent_memory.incqldb memories +incql db verify ./agent_memory.incqldb +incql db compact ./agent_memory.incqldb +incql db recover ./agent_memory.incqldb +incql db export ./agent_memory.incqldb memories --to memories.parquet ``` The CLI is not the primary runtime. It exists for inspection, verification, maintenance, local workflows, and debugging. ## Relationship to interop -CSV, Parquet, Arrow, and other external formats are important, but they should not define InQL-DB's core identity. +CSV, Parquet, Arrow, and other external formats are important, but they should not define IncQL-DB's core identity. -InQL already owns source and sink concepts at the session layer. InQL-DB should provide a native store and expose import/export or scan/write adapters through InQL's source/sink system. +IncQL already owns source and sink concepts at the session layer. IncQL-DB should provide a native store and expose import/export or scan/write adapters through IncQL's source/sink system. This keeps the database small and avoids making every external format a storage dependency. @@ -569,11 +569,11 @@ This keeps the database small and avoids making every external format a storage ### SurrealDB -SurrealDB is a useful product reference for InQL-DB's app-local AI memory direction, but not the storage or query architecture to copy. +SurrealDB is a useful product reference for IncQL-DB's app-local AI memory direction, but not the storage or query architecture to copy. -SurrealDB describes itself as a multi-model database that combines document, graph, time-series, relational, geospatial, and key-value data models with "vector, full-text, hybrid" retrieval. It also explicitly targets the same deployment zone InQL-DB cares about: a single Rust binary that can run "embedded (in-app), in the browser (via WebAssembly), in the edge." +SurrealDB describes itself as a multi-model database that combines document, graph, time-series, relational, geospatial, and key-value data models with "vector, full-text, hybrid" retrieval. It also explicitly targets the same deployment zone IncQL-DB cares about: a single Rust binary that can run "embedded (in-app), in the browser (via WebAssembly), in the edge." -That validates several InQL-DB product bets: +That validates several IncQL-DB product bets: - embedded local deployment matters for AI and edge applications, - vector retrieval should sit next to structured filtering rather than in a separate service, @@ -582,10 +582,10 @@ That validates several InQL-DB product bets: SurrealDB's features also show that native vector indexing belongs in the core. Its feature documentation describes HNSW vector indexing for approximate nearest-neighbour search with euclidean, cosine, and manhattan distance metrics, and its vector docs distinguish brute-force exact search from HNSW approximate search. -InQL-DB should not adopt SurrealDB's center of gravity. SurrealDB says it is "at its core, a document database" where each record is stored on an underlying key-value engine. InQL-DB should remain typed, InQL-native, analytical, and columnar: +IncQL-DB should not adopt SurrealDB's center of gravity. SurrealDB says it is "at its core, a document database" where each record is stored on an underlying key-value engine. IncQL-DB should remain typed, IncQL-native, analytical, and columnar: -- InQL models define schemas. -- InQL and Substrait define the query boundary. +- IncQL models define schemas. +- IncQL and Substrait define the query boundary. - Columnar segments are the primary physical store. - Vectorized execution is the primary execution model. - SQL-like or SurrealQL-like syntax is not part of the core. @@ -597,7 +597,7 @@ The useful comparison is therefore: | SurrealDB | multi-model document/KV database with SurrealQL | | DuckDB | embedded analytical SQL engine | | Delta | immutable columnar files plus transaction log | -| InQL-DB | embedded typed analytical memory store for InQL plans | +| IncQL-DB | embedded typed analytical memory store for IncQL plans | ## Implementation path @@ -605,11 +605,11 @@ The whitepaper north star should be split into RFCs before implementation. Recommended RFC sequence: -1. **InQL-DB backend RFC** - Define `BackendKind.InQLDBEngine`, session selection, plan execution boundary, error classes, and compatibility with existing DataFusion-backed sessions. +1. **IncQL-DB backend RFC** + Define `BackendKind.IncQLDBEngine`, session selection, plan execution boundary, error classes, and compatibility with existing DataFusion-backed sessions. 2. **Directory and transaction-log RFC** - Define `.inqldb/` layout, commit file format, checkpointing, locking, recovery, and snapshot semantics. + Define `.incqldb/` layout, commit file format, checkpointing, locking, recovery, and snapshot semantics. 3. **Column segment RFC** Define `iqseg` physical layout, primitive types, nullability, statistics, compression hooks, and checksums. @@ -624,13 +624,13 @@ Recommended RFC sequence: Define evidence-provider runs, candidate evidence, rejected evidence, provider fingerprints, index provenance, federated evidence-run grouping, and replay/debug contracts. 7. **CLI RFC** - Define `inql db` commands and diagnostics. + Define `incql db` commands and diagnostics. ## First useful vertical slice The first implementation should prove the whole architecture with a narrow plan subset: -- open/create `.inqldb/`, +- open/create `.incqldb/`, - create one table from an Incan model-derived schema, - append batches, - commit with WAL/log semantics, @@ -638,23 +638,23 @@ The first implementation should prove the whole architecture with a narrow plan - filter/project/limit, - collect into `DataFrame[T]`, - persist one evidence-provider run with eligible and rejected candidate rows, -- inspect and verify with `inql db`. +- inspect and verify with `incql db`. Vector search can follow immediately after this slice, starting with brute-force search over fixed-dimension vectors before adding ANN index files. ## Open questions -- Should the segment format target later Parquet compatibility, or remain permanently InQL-native? +- Should the segment format target later Parquet compatibility, or remain permanently IncQL-native? - Should commit files be binary from the start, or use a temporary text format until the transaction protocol stabilizes? - What is the smallest ACID guarantee acceptable for the first implementation while still preserving the north star? -- How should InQL model field IDs be assigned and preserved across schema evolution? +- How should IncQL model field IDs be assigned and preserved across schema evolution? - Should vector indexes be table-level, segment-level, or both? - Should eligible and rejected evidence be stored as separate physical tables or as status-filtered candidate rows? - What fingerprint inputs are required to replay or compare a HyperQuant provider run? - How should federated evidence runs group package-specific provider runs without merging specialist evidence into one anonymous context? - How should compaction coordinate data segments, delete files, and index rebuilds? -- What is the compatibility story for opening old `.inqldb/` directories after protocol changes? +- What is the compatibility story for opening old `.incqldb/` directories after protocol changes? ## References @@ -668,6 +668,6 @@ Vector search can follow immediately after this slice, starting with brute-force - [SurrealDB concepts](https://surrealdb.com/docs/surrealdb/introduction/concepts) - [SurrealDB features](https://surrealdb.com/features) - [SurrealDB vector database docs](https://surrealdb.com/docs/surrealdb/models/vector) -- [InQL architecture](../architecture.md) -- [InQL RFC 004: Execution context](../rfcs/004_inql_execution_context.md) -- [InQL RFC 007: Prism logical planning and optimization engine](../rfcs/007_prism_planning_engine.md) +- [IncQL architecture](../architecture.md) +- [IncQL RFC 004: Execution context](../rfcs/004_incql_execution_context.md) +- [IncQL RFC 007: Prism logical planning and optimization engine](../rfcs/007_prism_planning_engine.md) diff --git a/examples/README.md b/examples/README.md index 97ff5408..06a7e0f5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ -# InQL examples +# IncQL examples -Examples demonstrating InQL model-shaped dataset types, scalar-expression builders, and Session execution patterns. +Examples demonstrating IncQL model-shaped dataset types, scalar-expression builders, and Session execution patterns. ## Overview @@ -45,7 +45,7 @@ incan run examples/advanced_retail_analytics.incn - a grouped paid-order rollup using `sum`, `avg`, `min`, `max`, `count`, and `count_distinct` - a generated tag view that composes window ranking with `explode(...)` -`advanced_retail_query_blocks/` is the same fixture exercised from a standalone dependency consumer. It imports `pub::inql` and runs query blocks for the high-value projection, grouped rollup, and generated-tag window view: +`advanced_retail_query_blocks/` is the same fixture exercised from a standalone dependency consumer. It imports `pub::incql` and runs query blocks for the high-value projection, grouped rollup, and generated-tag window view: ```incan high_value = query { @@ -78,14 +78,14 @@ rollup = query { ## What these examples show -These examples document the API patterns for the InQL dataset and Session surface: +These examples document the API patterns for the IncQL dataset and Session surface: 1. Model contracts are visible at source boundaries as `LazyFrame[OrderId]`, `LazyFrame[OrderLine]`, and `LazyFrame[AggregateOrder]` 2. Carrier transformations remain typed Incan functions rather than stringly runtime scripts 3. Builder-based aggregation runs through `col(...)`, `sum(...)`, and `count()` 4. Builder-based scalar expressions run through `col(...)`, `lit(...)`, `eq(...)`, `gt(...)`, `add(...)`, and `mul(...)` -5. Query blocks activate through `pub::inql` in dependency consumers, desugar into carrier calls, and meet the rest of - InQL at the Substrait boundary +5. Query blocks activate through `pub::incql` in dependency consumers, desugar into carrier calls, and meet the rest of + IncQL at the Substrait boundary 6. Session execution provides `collect`, `display`, and write sinks over DataFusion They serve three purposes: @@ -99,7 +99,7 @@ They serve three purposes: - **RFC 041** (First-Class Rust Interop Authoring): Implemented in Incan v0.2 - **RFC 042** (Traits Are Always Abstract): Implemented in Incan v0.2 -These RFCs provide the trait and interop foundation InQL builds on. +These RFCs provide the trait and interop foundation IncQL builds on. ## Scope boundaries diff --git a/examples/advanced_retail_analytics.incn b/examples/advanced_retail_analytics.incn index a0ef6aad..fcd825c7 100644 --- a/examples/advanced_retail_analytics.incn +++ b/examples/advanced_retail_analytics.incn @@ -1,7 +1,7 @@ """ Example: advanced retail analytics over a 100-row CSV fixture. -This is a compact spike of the larger InQL surface: +This is a compact spike of the larger IncQL surface: - typed Session CSV ingestion - scalar cleanup and derivation diff --git a/examples/advanced_retail_query_blocks/incan.lock b/examples/advanced_retail_query_blocks/incan.lock index 5953fe38..8bbf77d1 100644 --- a/examples/advanced_retail_query_blocks/incan.lock +++ b/examples/advanced_retail_query_blocks/incan.lock @@ -27,7 +27,7 @@ version = "0.3.0-rc49" dependencies = [ "incan_derive", "incan_stdlib", - "inql", + "incql", ] [[package]] @@ -1872,7 +1872,7 @@ dependencies = [ ] [[package]] -name = "inql" +name = "incql" version = "0.3.0-rc49" dependencies = [ "blake2", diff --git a/examples/advanced_retail_query_blocks/incan.toml b/examples/advanced_retail_query_blocks/incan.toml index 7215eb76..92f075c7 100644 --- a/examples/advanced_retail_query_blocks/incan.toml +++ b/examples/advanced_retail_query_blocks/incan.toml @@ -3,7 +3,7 @@ name = "advanced_retail_query_blocks" version = "0.1.0" [dependencies] -inql = { path = "../.." } +incql = { path = "../.." } [project.scripts] main = "src/main.incn" diff --git a/examples/advanced_retail_query_blocks/src/main.incn b/examples/advanced_retail_query_blocks/src/main.incn index 74f5f64d..fa66a044 100644 --- a/examples/advanced_retail_query_blocks/src/main.incn +++ b/examples/advanced_retail_query_blocks/src/main.incn @@ -1,13 +1,13 @@ """ -Advanced retail analytics over InQL's dependency-activated query-block surface. +Advanced retail analytics over IncQL's dependency-activated query-block surface. Run from this directory with: incan run src/main.incn """ -import pub::inql -from pub::inql import ( +import pub::incql +from pub::incql import ( DataFrame, LazyFrame, Session, diff --git a/examples/models.incn b/examples/models.incn index 64390db6..99fabe77 100644 --- a/examples/models.incn +++ b/examples/models.incn @@ -1,8 +1,8 @@ """ -Compact schemas used by the InQL examples. +Compact schemas used by the IncQL examples. Each executable example binds source reads to `LazyFrame[Model]` so the row contract is visible at the boundary where -data enters InQL. +data enters IncQL. ## Models diff --git a/incan.lock b/incan.lock index 804ae803..f1f57986 100644 --- a/incan.lock +++ b/incan.lock @@ -1863,7 +1863,7 @@ dependencies = [ ] [[package]] -name = "inql" +name = "incql" version = "0.3.0" dependencies = [ "blake2", diff --git a/incan.toml b/incan.toml index 6c3091e3..ee3c02d4 100644 --- a/incan.toml +++ b/incan.toml @@ -1,5 +1,5 @@ [project] -name = "inql" +name = "incql" version = "0.1.0" [rust-dependencies] @@ -8,7 +8,7 @@ prost = "0.14" prost-types = "0.14" # Build needs `protoc` on PATH (`brew install protobuf`), or `features = ["protoc"]` + CMake. substrait = { version = "0.63", features = ["protoc"] } -# Datafusion is InQL default query engine, and provides the Substrait to Datafusion translation. +# Datafusion is IncQL default query engine, and provides the Substrait to Datafusion translation. datafusion = "53" datafusion_common = { package = "datafusion-common", version = "53" } datafusion_expr = { package = "datafusion-expr", version = "53" } diff --git a/scripts/smoke_pub_consumer.sh b/scripts/smoke_pub_consumer.sh index 2e82200e..1eee8bac 100644 --- a/scripts/smoke_pub_consumer.sh +++ b/scripts/smoke_pub_consumer.sh @@ -10,22 +10,22 @@ PROJECT_DIR="$SMOKE_ROOT/project" rm -rf "$PROJECT_DIR" mkdir -p "$PROJECT_DIR" -"$INCAN_BIN" init "$PROJECT_DIR" --name inql_pub_consumer_smoke >/dev/null +"$INCAN_BIN" init "$PROJECT_DIR" --name incql_pub_consumer_smoke >/dev/null cat > "$PROJECT_DIR/incan.toml" < "$PROJECT_DIR/src/main.incn" <<'EOF' -from pub::inql import Session, SessionError, always_true +from pub::incql import Session, SessionError, always_true @derive(Clone) model Order: @@ -41,11 +41,11 @@ def main() -> Result[None, SessionError]: EOF cat > "$PROJECT_DIR/src/query_blocks_smoke.incn" < str: suffix = suffix + "_" + sketch_type.value_domain.value() if sketch_type.precision != 14: suffix = suffix + f"_p{sketch_type.precision}" - if sketch_type.format.value() != "inql_hll_v1": + if sketch_type.format.value() != "incql_hll_v1": suffix = suffix + "_" + sketch_type.format.value() return suffix None => return "" diff --git a/src/backends.incn b/src/backends.incn index c0bab02c..f209bddd 100644 --- a/src/backends.incn +++ b/src/backends.incn @@ -1,7 +1,7 @@ """ Backend configuration objects for session execution. -The root `Session` API stays portable, while backend-specific configuration lives under `pub::inql.backends`. +The root `Session` API stays portable, while backend-specific configuration lives under `pub::incql.backends`. Backend selection is an adapter-neutral kind plus options envelope so adding another backend does not require adding one field per implementation to the core Session state. """ diff --git a/src/dataset/mod.incn b/src/dataset/mod.incn index 6d7badaf..387495d4 100644 --- a/src/dataset/mod.incn +++ b/src/dataset/mod.incn @@ -1,5 +1,5 @@ """ -Dataset carriers for InQL. +Dataset carriers for IncQL. This module defines the *author-facing* type hierarchy used to carry schema-parameterized tabular data through relational pipelines: @@ -29,8 +29,8 @@ The current method-chain surface in this module is the explicit builder-based AP Illustrative current-shape examples: ```incan -from pub::inql import LazyFrame -from pub::inql.functions import add, avg, col, count, gt, max, min, sum +from pub::incql import LazyFrame +from pub::incql.functions import add, avg, col, count, gt, max, min, sum from models import Order, OrderSummary def high_value_orders(orders: LazyFrame[Order]) -> LazyFrame[Order]: @@ -45,7 +45,7 @@ def summarize_orders(orders: LazyFrame[Order]) -> LazyFrame[OrderSummary]: See also: -- `docs/rfcs/001_inql_dataset.md` +- `docs/rfcs/001_incql_dataset.md` - `docs/language/explanation/dataset_carriers.md` - `docs/language/reference/dataset_methods.md` - `docs/language/reference/builders/filters.md` @@ -107,7 +107,7 @@ pub trait DataSet[T with Clone]: def to_substrait_plan(self) -> Plan def try_to_substrait_plan(self) -> Result[Plan, SubstraitLoweringError] def filter(self, predicate: ColumnExpr) -> Self - # TODO(InQL RFC 004): Replace the Self-only join surface with heterogeneous join typing and an explicit output-schema contract. + # TODO(IncQL RFC 004): Replace the Self-only join surface with heterogeneous join typing and an explicit output-schema contract. def join(self, other: Self, on: ColumnExpr, relation_name: str = "") -> Self def left_join(self, other: Self, on: ColumnExpr, relation_name: str = "") -> Self def with_column(self, name: str, expr: ColumnExpr) -> Self diff --git a/src/evidence.incn b/src/evidence.incn index a52df070..99ca4b58 100644 --- a/src/evidence.incn +++ b/src/evidence.incn @@ -51,7 +51,7 @@ pub enum MetadataLifecycle(str): pub enum MetadataSource(str): """Source family for one metadata attachment.""" - InQL = "inql" + IncQL = "incql" User = "user" Prism = "prism" Session = "session" diff --git a/src/filter_builders.incn b/src/filter_builders.incn index 40edbce7..1b567e57 100644 --- a/src/filter_builders.incn +++ b/src/filter_builders.incn @@ -1,5 +1,5 @@ """ -Filter-oriented builder surface for the current InQL scalar-expression slice. +Filter-oriented builder surface for the current IncQL scalar-expression slice. These names provide predicate-friendly helper spellings over the same scalar-expression model used by filters, projections, grouping keys, and aggregate inputs. diff --git a/src/function_registry.incn b/src/function_registry.incn index d87716f1..9adf5b11 100644 --- a/src/function_registry.incn +++ b/src/function_registry.incn @@ -1,5 +1,5 @@ """ -Package-owned function registry metadata for the current public InQL function surface. +Package-owned function registry metadata for the current public IncQL function surface. Public helpers register themselves with declaration-side registry decorators. The decorator owns non-derivable machine metadata and records one runtime entry while returning the helper unchanged. Function signatures and canonical names @@ -9,12 +9,12 @@ source helper name is constrained by the host language, such as the `modulo(...) from substrait.function_extensions import FUNCTION_EXTENSION_URI -pub const CORE_FUNCTION_NAMESPACE: str = "inql.functions" +pub const CORE_FUNCTION_NAMESPACE: str = "incql.functions" @derive(Clone) pub enum FunctionPolicyCategory(str): - """InQL RFC 024 portability and extension-policy category for one function-like metadata item.""" + """IncQL RFC 024 portability and extension-policy category for one function-like metadata item.""" PortableCore = "portable_core" ExtensionOnly = "extension_only" @@ -86,7 +86,7 @@ pub enum FunctionInputType(str): @derive(Clone, Eq) pub enum SubstraitMappingKind(str): - """Substrait interchange category for one portable InQL function.""" + """Substrait interchange category for one portable IncQL function.""" CoreFunction = "core_function" ExtensionFunction = "extension_function" @@ -97,7 +97,7 @@ pub enum SubstraitMappingKind(str): @derive(Clone, Eq) pub model FunctionVersion: - """One InQL package version used by function lifecycle metadata.""" + """One IncQL package version used by function lifecycle metadata.""" pub major: int pub minor: int @@ -240,7 +240,7 @@ pub const HLL_SKETCH_CONSTRUCT_POLICY: FunctionSketchPolicy = FunctionSketchPoli role="construct", default_domain="string_identifier", default_precision=14, - format="inql_hll_v1", + format="incql_hll_v1", returns_sketch=true, accepts_sketch=false, result_interpretation="typed HyperLogLog sketch state", @@ -251,7 +251,7 @@ pub const HLL_SKETCH_MERGE_POLICY: FunctionSketchPolicy = FunctionSketchPolicy( role="merge", default_domain="string_identifier", default_precision=14, - format="inql_hll_v1", + format="incql_hll_v1", returns_sketch=true, accepts_sketch=true, result_interpretation="merged typed HyperLogLog sketch state", @@ -262,7 +262,7 @@ pub const HLL_SKETCH_ESTIMATE_POLICY: FunctionSketchPolicy = FunctionSketchPolic role="estimate", default_domain="string_identifier", default_precision=14, - format="inql_hll_v1", + format="incql_hll_v1", returns_sketch=false, accepts_sketch=true, result_interpretation="approximate distinct-count estimate from typed HyperLogLog sketch state", @@ -273,7 +273,7 @@ pub const HLL_SKETCH_SERIALIZE_POLICY: FunctionSketchPolicy = FunctionSketchPoli role="serialize", default_domain="string_identifier", default_precision=14, - format="inql_hll_v1", + format="incql_hll_v1", returns_sketch=false, accepts_sketch=true, result_interpretation="explicit serialized HyperLogLog sketch payload", @@ -284,7 +284,7 @@ pub const HLL_SKETCH_DESERIALIZE_POLICY: FunctionSketchPolicy = FunctionSketchPo role="deserialize", default_domain="string_identifier", default_precision=14, - format="inql_hll_v1", + format="incql_hll_v1", returns_sketch=true, accepts_sketch=false, result_interpretation="typed HyperLogLog sketch state decoded from explicit serialized payload", @@ -472,7 +472,7 @@ pub model FunctionSpec: @derive(Clone) pub model FunctionRegistryEntry: - """Runtime projection for one registered InQL function.""" + """Runtime projection for one registered IncQL function.""" pub function_ref: str pub namespace: str @@ -674,7 +674,7 @@ def _entry_matches_spec(entry: FunctionRegistryEntry, canonical_name: str, spec: pub def function_ref_for(canonical_name: str) -> str: - """Return the durable registry reference for one canonical InQL function name.""" + """Return the durable registry reference for one canonical IncQL function name.""" return namespaced_function_ref(CORE_FUNCTION_NAMESPACE, canonical_name) @@ -789,7 +789,7 @@ pub variant_scalar_spec = partial FunctionSpec(canonical_name=None, namespace=CO pub extension_only_spec = partial FunctionSpec(canonical_name=None, policy_category=FunctionPolicyCategory.ExtensionOnly, aliases=[], alias_policy=FunctionAliasPolicy.OptInCompatibility, aggregate_modifiers=NO_AGGREGATE_MODIFIERS, approximation=NO_APPROXIMATION, sketch=NO_SKETCH_POLICY, variant=NO_VARIANT_POLICY) -"""Build one opt-in compatibility alias spec for semantics that remain typeable by InQL.""" +"""Build one opt-in compatibility alias spec for semantics that remain typeable by IncQL.""" pub compatibility_alias_spec = partial FunctionSpec(canonical_name=None, policy_category=FunctionPolicyCategory.CompatibilityAlias, alias_policy=FunctionAliasPolicy.OptInCompatibility, aggregate_modifiers=NO_AGGREGATE_MODIFIERS, approximation=NO_APPROXIMATION, sketch=NO_SKETCH_POLICY, variant=NO_VARIANT_POLICY) diff --git a/src/functions/casts/cast.incn b/src/functions/casts/cast.incn index e5b6acaa..6c920d97 100644 --- a/src/functions/casts/cast.incn +++ b/src/functions/casts/cast.incn @@ -89,7 +89,7 @@ pub def cast(expr: ColumnExpr, target_type: str) -> ColumnExpr: Parameters: expr: Expression to convert. - target_type: InQL type spelling to cast to. Prefer primitive type tokens such as `int`, `float`, `str`, and + target_type: IncQL type spelling to cast to. Prefer primitive type tokens such as `int`, `float`, `str`, and `bool` when the target is a common scalar primitive. """ return _cast_application(expr, target_type) diff --git a/src/functions/casts/try_cast.incn b/src/functions/casts/try_cast.incn index a82598cc..355e9ca8 100644 --- a/src/functions/casts/try_cast.incn +++ b/src/functions/casts/try_cast.incn @@ -88,7 +88,7 @@ pub def try_cast(expr: ColumnExpr, target_type: str) -> ColumnExpr: Parameters: expr: Expression to convert. - target_type: InQL type spelling to cast to. Prefer primitive type tokens such as `int`, `float`, `str`, and + target_type: IncQL type spelling to cast to. Prefer primitive type tokens such as `int`, `float`, `str`, and `bool` when the target is a common scalar primitive. """ return _try_cast_application(expr, target_type) diff --git a/src/functions/formatting/display.incn b/src/functions/formatting/display.incn index b3560bf1..33b05704 100644 --- a/src/functions/formatting/display.incn +++ b/src/functions/formatting/display.incn @@ -21,7 +21,7 @@ pub def display[T with Clone](data: LazyFrame[T]) -> None: """ match data.clone().collect(): Ok(df) => _render_data_frame(df) - Err(err) => println(f"InQL display failed: {err.error_message()}") + Err(err) => println(f"IncQL display failed: {err.error_message()}") def _join_columns(columns: list[str]) -> str: @@ -33,7 +33,7 @@ def _render_data_frame[T with Clone](data: DataFrame[T]) -> None: """Render one collected DataFrame preview using structured materialization metadata.""" preview = data.preview_text() columns = data.columns() - println("InQL display") + println("IncQL display") if len(columns) > 0: println(f"columns: [{_join_columns(columns)}]") println(f"rows: {data.row_count()}") diff --git a/src/functions/mod.incn b/src/functions/mod.incn index 42fc2389..940a62d5 100644 --- a/src/functions/mod.incn +++ b/src/functions/mod.incn @@ -1,5 +1,5 @@ """ -Explicit builder helpers for the current InQL relational surface. +Explicit builder helpers for the current IncQL relational surface. Each helper lives in a logical family submodule with declaration-side registry metadata and local inline tests. This facade keeps the public import surface stable without owning a second machine-readable registry list. diff --git a/src/functions/operators/and_.incn b/src/functions/operators/and_.incn index dff7e97c..f2bac5ec 100644 --- a/src/functions/operators/and_.incn +++ b/src/functions/operators/and_.incn @@ -2,7 +2,7 @@ Boolean conjunction helper. `and_` uses a trailing underscore to avoid host-language keyword collisions while keeping the canonical helper spelling -stable for InQL users. +stable for IncQL users. """ from function_registry import ( diff --git a/src/functions/operators/mul.incn b/src/functions/operators/mul.incn index 01108598..104fbc61 100644 --- a/src/functions/operators/mul.incn +++ b/src/functions/operators/mul.incn @@ -1,7 +1,7 @@ """ Multiplication scalar helper. -`mul` uses the public InQL spelling while declaring the Substrait extension name `multiply` in its registry metadata. +`mul` uses the public IncQL spelling while declaring the Substrait extension name `multiply` in its registry metadata. """ from function_registry import ( diff --git a/src/functions/registry.incn b/src/functions/registry.incn index 223c534b..79b38669 100644 --- a/src/functions/registry.incn +++ b/src/functions/registry.incn @@ -1,5 +1,5 @@ """ -Shared declaration-side registry state for public InQL helpers. +Shared declaration-side registry state for public IncQL helpers. Helper modules attach metadata with `@register_function(...)` next to the helper they expose. This module owns the runtime projection for helpers that have actually been loaded in the current process. Checked API metadata remains the diff --git a/src/functions/sketches/hll_deserialize.incn b/src/functions/sketches/hll_deserialize.incn index 639c6219..1e8b4046 100644 --- a/src/functions/sketches/hll_deserialize.incn +++ b/src/functions/sketches/hll_deserialize.incn @@ -32,7 +32,7 @@ pub def hll_deserialize( payload: StrValueOrColumn, value_domain: SketchValueDomain = SketchValueDomain.StringIdentifier, precision: int = 14, - format: SketchSerializationFormat = SketchSerializationFormat.InqlHllV1, + format: SketchSerializationFormat = SketchSerializationFormat.IncqlHllV1, ) -> SketchExpr: """ Deserialize an explicit HyperLogLog payload into typed sketch state. diff --git a/src/generator_builders.incn b/src/generator_builders.incn index 3916de3f..7e2830b7 100644 --- a/src/generator_builders.incn +++ b/src/generator_builders.incn @@ -238,7 +238,7 @@ module tests: generator = explode(col("line_items"), "line_item") assert generator.kind == GeneratorKind.Explode assert generator.canonical_name == "explode" - assert generator.function_ref == "inql.functions.explode" + assert generator.function_ref == "incql.functions.explode" assert column_expr_name(generator.expr) == "line_items" assert len(generator.arguments) == 1 assert generator.output_columns[0] == "line_item" diff --git a/src/governance.incn b/src/governance.incn index 7b6a5445..5fb4f9f8 100644 --- a/src/governance.incn +++ b/src/governance.incn @@ -186,7 +186,7 @@ pub def governed_attribute( target: SemanticTarget, key: str, value: str, - value_schema: str = "inql.governance.string.v0.1", + value_schema: str = "incql.governance.string.v0.1", scope: GovernedAttributeScope = GovernedAttributeScope.Field, source: GovernedAttributeSource = GovernedAttributeSource.Inferred, confidence: GovernedAttributeConfidence = GovernedAttributeConfidence.Unknown, diff --git a/src/inspect.incn b/src/inspect.incn index 4df71433..d38a4642 100644 --- a/src/inspect.incn +++ b/src/inspect.incn @@ -34,7 +34,7 @@ from governance import ( governed_attribute, policy_checkpoint, ) -from metadata import inql_version +from metadata import incql_version from functions.registry import function_registry_entry_by_name from prism import prism_plan_id from prism.output_columns import authored_output_schema, rewritten_output_schema @@ -46,8 +46,8 @@ from scalar_schema import ScalarColumnSpec from std.collections import OrderedSet from window_builders import WindowProjection -pub const INSPECTION_SCHEMA_VERSION: str = "inql.inspection.v0.1" -pub const LINEAGE_RULE_VERSION: str = "inql.lineage.prism.v0.1" +pub const INSPECTION_SCHEMA_VERSION: str = "incql.inspection.v0.1" +pub const LINEAGE_RULE_VERSION: str = "incql.lineage.prism.v0.1" @derive(Clone) @@ -82,7 +82,7 @@ pub model PlanInspection: """Structured local inspection record for one Prism-backed lazy plan.""" pub schema_version: str - pub inql_version: str + pub incql_version: str pub plan_id: str pub plan_target: SemanticTarget pub root_target: SemanticTarget @@ -216,7 +216,7 @@ pub def inspect_plan[T with Clone](data: LazyFrame[T]) -> PlanInspection: ) return PlanInspection( schema_version=INSPECTION_SCHEMA_VERSION, - inql_version=inql_version(), + incql_version=incql_version(), plan_id=plan_id, plan_target=plan_target, root_target=_authored_node_target(plan_id, authored_context.node(tip_id)), @@ -1192,17 +1192,17 @@ def _metadata_attachments( attachments.append( _public_attachment( plan_target, - "inql.inspection", + "incql.inspection", "schema_version", INSPECTION_SCHEMA_VERSION, MetadataLifecycle.Analyzed, - MetadataSource.InQL, + MetadataSource.IncQL, ), ) attachments.append( _public_attachment( plan_target, - "inql.lineage", + "incql.lineage", "rule_version", LINEAGE_RULE_VERSION, MetadataLifecycle.Analyzed, @@ -1215,7 +1215,7 @@ def _metadata_attachments( attachments.append( _public_attachment( output_fields[idx], - "inql.schema", + "incql.schema", "substrait_primitive_kind", kind.value(), MetadataLifecycle.Analyzed, @@ -1229,13 +1229,13 @@ def _governed_attributes_from_metadata(attachments: list[MetadataAttachment]) -> """Promote local schema metadata into conservative governed attributes.""" mut attributes: list[GovernedAttribute] = [] for attachment in attachments: - if attachment.namespace == "inql.schema" and attachment.key == "substrait_primitive_kind": + if attachment.namespace == "incql.schema" and attachment.key == "substrait_primitive_kind": attributes.append( governed_attribute( attachment.target, "schema.substrait_primitive_kind", attachment.payload.value, - value_schema="inql.governance.schema-primitive-kind.v0.1", + value_schema="incql.governance.schema-primitive-kind.v0.1", scope=GovernedAttributeScope.Field, source=GovernedAttributeSource.Lineage, confidence=GovernedAttributeConfidence.Exact, @@ -1260,7 +1260,7 @@ def _policy_checkpoints_for_inspection( plan_target, PolicyCheckpointKind.Planning, PolicyCheckpointAction.Observe, - "inql.policy.local-inspection", + "incql.policy.local-inspection", "governed_evidence_observed", evidence_refs=evidence_refs, )] @@ -1284,7 +1284,7 @@ def _public_attachment( target=target, namespace=namespace, key=key, - payload=MetadataPayload(schema="inql.metadata.string.v0.1", value=value), + payload=MetadataPayload(schema="incql.metadata.string.v0.1", value=value), lifecycle=lifecycle, source=source, visibility=MetadataVisibility.Public, diff --git a/src/lib.incn b/src/lib.incn index 40b7dced..bb98cd0c 100644 --- a/src/lib.incn +++ b/src/lib.incn @@ -1,8 +1,8 @@ """ -InQL — typed query surface (Incan library). +IncQL — typed query surface (Incan library). Add modules (e.g. `dataset`, `functions`, `query`) and re-export them here with `pub from ... import ...`. -Consumers depend on this package via `[dependencies]` and import with `from pub::inql import ...`. +Consumers depend on this package via `[dependencies]` and import with `from pub::incql import ...`. """ pub from dataset import BoundedDataSet, DataFrame, DataSet, DataStream, LazyFrame, UnboundedDataSet @@ -405,7 +405,7 @@ pub from backends import ( datafusion_backend_selection, parquet_source, ) -pub from metadata import inql_version +pub from metadata import incql_version pub from session.domain import SinkKind, SinkTarget, csv_sink, parquet_sink, sink_kind_name pub from evidence import ( AdapterCoverageRecord, diff --git a/src/metadata.incn b/src/metadata.incn index 594e2d1f..6fec8d7c 100644 --- a/src/metadata.incn +++ b/src/metadata.incn @@ -1,6 +1,6 @@ -"""Library identity and version helpers for InQL consumers.""" +"""Library identity and version helpers for IncQL consumers.""" -pub def inql_version() -> str: - """Semantic version of this InQL package (keep in sync with `incan.toml`).""" +pub def incql_version() -> str: + """Semantic version of this IncQL package (keep in sync with `incan.toml`).""" return "0.1.0" diff --git a/src/prism/output_columns.incn b/src/prism/output_columns.incn index 6f2c6316..815b316d 100644 --- a/src/prism/output_columns.incn +++ b/src/prism/output_columns.incn @@ -166,7 +166,7 @@ def _expression_output_schema( expr: ColumnExpr, output_name: str, ) -> ScalarColumnSpec: - """Infer the output schema fact for one scalar expression when InQL can do so honestly.""" + """Infer the output schema fact for one scalar expression when IncQL can do so honestly.""" match expr: BoolColumnExpr(_) => return ScalarColumnSpec(name=output_name, kind=Some(SubstraitPrimitiveKind.Bool), nullable=true) diff --git a/src/projection_builders.incn b/src/projection_builders.incn index ba3d77f8..457470ab 100644 --- a/src/projection_builders.incn +++ b/src/projection_builders.incn @@ -1,5 +1,5 @@ """ -Explicit scalar-expression builder surface for the current InQL-only row-expression slice. +Explicit scalar-expression builder surface for the current IncQL-only row-expression slice. These builders are the semantic target for future compiler sugar such as `.amount * 1.2`, predicate expressions, grouping keys, aggregate inputs, and query-block lowering. Today they provide a library-owned way to express row-level @@ -80,49 +80,49 @@ pub type UntypedColumnExpr = Union[ColumnRefExpr, IntLiteralExpr, FloatLiteralEx @derive(Clone) pub model BoolColumnExpr: - """Scalar expression known by InQL's helper surface to produce boolean values.""" + """Scalar expression known by IncQL's helper surface to produce boolean values.""" pub expr: UntypedColumnExpr @derive(Clone) pub model IntColumnExpr: - """Scalar expression known by InQL's helper surface to produce integer values.""" + """Scalar expression known by IncQL's helper surface to produce integer values.""" pub expr: UntypedColumnExpr @derive(Clone) pub model FloatColumnExpr: - """Scalar expression known by InQL's helper surface to produce floating-point values.""" + """Scalar expression known by IncQL's helper surface to produce floating-point values.""" pub expr: UntypedColumnExpr @derive(Clone) pub model NumberColumnExpr: - """Scalar expression known by InQL's helper surface to produce numeric values.""" + """Scalar expression known by IncQL's helper surface to produce numeric values.""" pub expr: UntypedColumnExpr @derive(Clone) pub model StringColumnExpr: - """Scalar expression known by InQL's helper surface to produce string values.""" + """Scalar expression known by IncQL's helper surface to produce string values.""" pub expr: UntypedColumnExpr @derive(Clone) pub model TimestampColumnExpr: - """Scalar expression known by InQL's helper surface to produce timestamp values.""" + """Scalar expression known by IncQL's helper surface to produce timestamp values.""" pub expr: UntypedColumnExpr @derive(Clone) pub model TemporalColumnExpr: - """Scalar expression accepted by InQL's date/time helper surface as string or timestamp temporal input.""" + """Scalar expression accepted by IncQL's date/time helper surface as string or timestamp temporal input.""" pub expr: UntypedColumnExpr diff --git a/src/quality.incn b/src/quality.incn index e51a8d16..44300d61 100644 --- a/src/quality.incn +++ b/src/quality.incn @@ -68,7 +68,7 @@ pub model QualityAssertion: """ One typed relational quality assertion declaration. - `expression` and `group_by` carry ordinary InQL scalar expressions so assertions can be planned as relational work. + `expression` and `group_by` carry ordinary IncQL scalar expressions so assertions can be planned as relational work. Count thresholds use integer fields, and rate thresholds use the `max_rate` field where applicable. """ diff --git a/src/session/csv_schema.incn b/src/session/csv_schema.incn index f878ca33..45cda3c8 100644 --- a/src/session/csv_schema.incn +++ b/src/session/csv_schema.incn @@ -7,7 +7,7 @@ from substrait.schema import RowColumnSpec, SubstraitPrimitiveKind pub def infer_csv_schema_columns(uri: str) -> Result[list[RowColumnSpec], SessionError]: """Read one CSV file and infer schema columns (name, nullability, primitive kind) from header + rows.""" - # TODO(InQL RFC 010): This is still an intentionally high-level CSV inference pass. + # TODO(IncQL RFC 010): This is still an intentionally high-level CSV inference pass. # It only looks at header presence, empty cells, and coarse primitive-kind merging. # Revisit this once the fuller CSV ingestion contract lands so dialect rules, quoting/escaping, # richer typing, and stricter schema/header policies stop living behind this lightweight helper. diff --git a/src/session/datafusion_backend.incn b/src/session/datafusion_backend.incn index 105835a5..88588cff 100644 --- a/src/session/datafusion_backend.incn +++ b/src/session/datafusion_backend.incn @@ -232,7 +232,7 @@ pub async def datafusion_write_parquet_async( async def _dataframe_from_plan(ctx: SessionContext, plan: Plan) -> Result[RustDataFrame, BackendError]: - """Build a DataFusion DataFrame, including InQL-owned relation bridges.""" + """Build a DataFusion DataFrame, including IncQL-owned relation bridges.""" root = root_rel(plan.clone()) match _window_rel(root.clone()): Some(_) => return await _dataframe_from_window_root_plan(ctx, plan) @@ -264,7 +264,7 @@ async def _dataframe_from_window_root_plan(ctx: SessionContext, plan: Plan) -> R async def _dataframe_from_non_window_plan(ctx: SessionContext, plan: Plan) -> Result[RustDataFrame, BackendError]: """Build a DataFusion DataFrame for plans that do not have a window root.""" root = root_rel(plan.clone()) - # DataFusion's stock Substrait consumer cannot plan InQL generator ExtensionSingleRel nodes. Root generators can + # DataFusion's stock Substrait consumer cannot plan IncQL generator ExtensionSingleRel nodes. Root generators can # execute directly; nested generators are first materialized and then replaced by execution-only temp reads. match _generator_extension(root): Some(extension) => return await _dataframe_from_generator_extension(ctx, extension) @@ -328,13 +328,13 @@ async def _dataframe_from_standard_plan(ctx: SessionContext, plan: Plan) -> Resu """Build a DataFusion DataFrame through the stock Substrait consumer.""" match _typed_sketch_extension_function_name(plan.clone()): Some(function_name) => - message = f"DataFusion backend does not support typed sketch execution for `{function_name}`; InQL preserved the typed sketch plan, but this adapter has no sketch runtime implementation" + message = f"DataFusion backend does not support typed sketch execution for `{function_name}`; IncQL preserved the typed sketch plan, but this adapter has no sketch runtime implementation" return Err(backend_error(BackendErrorKind.BackendPlanningError, message)) None => pass match _typed_variant_extension_function_name(plan.clone()): Some(function_name) => - message = f"DataFusion backend does not support variant execution for `{function_name}`; InQL preserved the typed variant plan, but this adapter has no variant runtime implementation" + message = f"DataFusion backend does not support variant execution for `{function_name}`; IncQL preserved the typed variant plan, but this adapter has no variant runtime implementation" return Err(backend_error(BackendErrorKind.BackendPlanningError, message)) None => pass @@ -353,7 +353,7 @@ async def _dataframe_from_consumer_plan( async def _execute_logical_plan(ctx: SessionContext, logical_plan: LogicalPlan) -> Result[RustDataFrame, BackendError]: - """Execute one DataFusion logical plan while preserving InQL's backend error envelope.""" + """Execute one DataFusion logical plan while preserving IncQL's backend error envelope.""" match await ctx.execute_logical_plan(logical_plan): Ok(df) => return Ok(df) Err(err) => return Err(backend_error(BackendErrorKind.BackendExecutionError, err.to_string())) @@ -409,7 +409,7 @@ def _dataframe_from_window_api( window_rel: ConsistentPartitionWindowRel, output_columns: list[str], ) -> Result[RustDataFrame, BackendError]: - """Apply one InQL window relation through DataFusion's typed expression API.""" + """Apply one IncQL window relation through DataFusion's typed expression API.""" select_exprs = _window_select_exprs(ctx, child_columns, window_rel, output_columns)? match child_df.select(select_exprs): Ok(df) => return Ok(df) @@ -420,11 +420,11 @@ async def _dataframe_from_generator_extension( ctx: SessionContext, extension: ExtensionSingleRel, ) -> Result[RustDataFrame, BackendError]: - """Build a DataFusion DataFrame for one InQL generator relation-extension root.""" + """Build a DataFusion DataFrame for one IncQL generator relation-extension root.""" payload = _decode_generator_payload(extension.clone())? child = _generator_input_rel(extension.clone())? child_columns = relation_output_columns(child.clone()) - # Generator arguments are ordinary InQL scalar expressions. Project them into stable temp columns first so the + # Generator arguments are ordinary IncQL scalar expressions. Project them into stable temp columns first so the # DataFusion unnest API can operate on columns rather than on arbitrary Substrait expressions. temp_columns = _generator_temp_columns(len(payload.arguments)) projected_names = _generator_projected_root_names( @@ -533,7 +533,7 @@ def _extend_materializations( def _replace_materialized_generator_relations(rel: Rel, path: str) -> Rel: """Replace previously materialized generator relation nodes with temp-view reads.""" - # This is an adapter-only rewrite. The InQL semantic plan remains a generator extension relation; only DataFusion's + # This is an adapter-only rewrite. The IncQL semantic plan remains a generator extension relation; only DataFusion's # execution plan sees the temporary table scan. match _generator_extension(rel.clone()): Some(_) => return read_named_table_rel(_materialized_generator_table_name(path)) @@ -782,7 +782,7 @@ def _replace_extension_generator_child(original: Rel, extension: ExtensionSingle def _materialized_generator_table_name(path: str) -> str: """Return the internal DataFusion temp-view name for one materialized generator relation path.""" - return f"__inql_generator_materialized_{path}" + return f"__incql_generator_materialized_{path}" def _collect_materialized_window_relations(rel: Rel, path: str) -> list[MaterializedWindowRelation]: @@ -1048,7 +1048,7 @@ def _replace_extension_window_child(original: Rel, extension: ExtensionSingleRel def _materialized_window_table_name(path: str) -> str: """Return the internal DataFusion temp-view name for one materialized window relation path.""" - return f"__inql_window_materialized_{path}" + return f"__incql_window_materialized_{path}" async def _register_materialized_dataframe( @@ -1064,7 +1064,7 @@ async def _register_materialized_dataframe( _register_materialized_schema_from_batch(f"{table_name}", batches[0]) schema = batches[0].schema() # Non-empty outputs are frozen into a MemTable so the stock consumer reads a concrete table instead of an - # InQL-specific generator extension relation. + # IncQL-specific generator extension relation. match MemTable.try_new(schema, [batches]): Ok(table) => return _register_materialized_memtable(ctx, table_name, table) Err(err) => return Err(backend_error(BackendErrorKind.BackendPlanningError, err.to_string())) @@ -1107,7 +1107,7 @@ def _register_materialized_schema_from_batch(table_name: str, batch: RecordBatch def _substrait_kind_for_arrow_type_name(type_name: str) -> SubstraitPrimitiveKind: - """Map the current DataFusion primitive output type names into InQL's minimal Substrait schema kinds.""" + """Map the current DataFusion primitive output type names into IncQL's minimal Substrait schema kinds.""" if type_name.starts_with("List(") or type_name.starts_with("LargeList("): if type_name.contains("Int64") or type_name.contains("Int32") or type_name.contains("UInt64") or type_name.contains( "UInt32", @@ -1150,7 +1150,7 @@ def _generator_projected_root_names( def _generator_extension(rel: Rel) -> Option[ExtensionSingleRel]: - """Return the root generator extension relation when the plan root is an InQL generator.""" + """Return the root generator extension relation when the plan root is an IncQL generator.""" match rel.rel_type: Some(RelType.ExtensionSingle(extension)) => if let Some(detail) = extension.detail.clone(): @@ -1161,7 +1161,7 @@ def _generator_extension(rel: Rel) -> Option[ExtensionSingleRel]: def _window_rel(rel: Rel) -> Option[ConsistentPartitionWindowRel]: - """Return the root window relation when the plan root is an InQL analytic window.""" + """Return the root window relation when the plan root is an IncQL analytic window.""" match rel.rel_type: Some(RelType.Window(window_rel)) => return Some(window_rel.as_ref().clone()) _ => return None @@ -1180,7 +1180,7 @@ def _window_select_exprs( window_rel: ConsistentPartitionWindowRel, output_columns: list[str], ) -> Result[list[DataFusionExpr], BackendError]: - """Build the typed DataFusion projection for one lowered InQL window relation.""" + """Build the typed DataFusion projection for one lowered IncQL window relation.""" final_columns = _window_final_columns(input_columns.clone(), window_rel.clone(), output_columns) return Ok( [_window_select_expr(ctx.clone(), input_columns.clone(), window_rel.clone(), output_column)? for output_column in final_columns], @@ -1427,7 +1427,7 @@ def _datafusion_window_function_definition( ctx: SessionContext, function_name: str, ) -> Result[DataFusionWindowFunctionDefinition, BackendError]: - """Resolve one InQL window function against DataFusion's function registry.""" + """Resolve one IncQL window function against DataFusion's function registry.""" state = ctx.state() if let Ok(udwf) = state.udwf(function_name): return Ok(DataFusionWindowFunctionDefinition.WindowUDF(udwf)) @@ -1536,14 +1536,14 @@ def _datafusion_window_distinct(invocation: RustI32) -> bool: def _window_output_expr(function_name: str, expr: DataFusionExpr) -> DataFusionExpr: - """Cast DataFusion unsigned analytic outputs to InQL's signed integer surface.""" + """Cast DataFusion unsigned analytic outputs to IncQL's signed integer surface.""" if _window_output_needs_i64_cast(function_name): return DataFusionExpr.Cast(DataFusionCast.new(Box.new(expr), ArrowDataType.Int64)) return expr def _datafusion_alias(expr: DataFusionExpr, output_name: str) -> DataFusionExpr: - """Attach one InQL output name to a DataFusion expression.""" + """Attach one IncQL output name to a DataFusion expression.""" return DataFusionExpr.Alias(DataFusionAlias(expr=Box.new(expr), relation=None, name=f"{output_name}", metadata=None)) @@ -1553,7 +1553,7 @@ def _datafusion_column(column_name: str) -> DataFusionExpr: def _datafusion_null_treatment(options: list[FunctionOption]) -> Option[DataFusionNullTreatment]: - """Translate optional InQL null treatment metadata into DataFusion's window model.""" + """Translate optional IncQL null treatment metadata into DataFusion's window model.""" for option in options: if option.name == WINDOW_NULL_TREATMENT_OPTION and len(option.preference) > 0: if option.preference[0] == "ignore_nulls": @@ -1569,7 +1569,7 @@ def _unsupported_window_expr(message: str) -> BackendError: def _window_output_needs_i64_cast(function_name: str) -> bool: - """Return whether DataFusion's unsigned analytic result should be cast into InQL's i64 surface type.""" + """Return whether DataFusion's unsigned analytic result should be cast into IncQL's i64 surface type.""" return ( function_name == "count" or function_name == "row_number" @@ -1603,7 +1603,7 @@ def _generator_input_rel(extension: ExtensionSingleRel) -> Result[Rel, BackendEr def _decode_generator_payload(extension: ExtensionSingleRel) -> Result[GeneratorExtensionPayload, BackendError]: - """Decode one InQL generator extension payload.""" + """Decode one IncQL generator extension payload.""" match extension.detail: Some(detail) => match decode_generator_extension_payload(detail.value): @@ -1634,7 +1634,7 @@ def _apply_generator_dataframe_shape( def _unnest_columns(df: RustDataFrame, columns: list[str], preserve_nulls: bool) -> Result[RustDataFrame, BackendError]: - """Call DataFusion unnest with the InQL inner/outer null-preservation policy.""" + """Call DataFusion unnest with the IncQL inner/outer null-preservation policy.""" options = UnnestOptions.new().with_preserve_nulls(preserve_nulls) mut datafusion_columns: list[Column] = [] for column in columns: @@ -1651,28 +1651,28 @@ def _unnest_columns(df: RustDataFrame, columns: list[str], preserve_nulls: bool) async def _collect_dataframe_batches(df: RustDataFrame) -> Result[list[RecordBatch], BackendError]: - """Collect one DataFusion frame and preserve InQL's backend execution error envelope.""" + """Collect one DataFusion frame and preserve IncQL's backend execution error envelope.""" match await df.collect(): Ok(batches) => return Ok(batches) Err(err) => return Err(backend_error(BackendErrorKind.BackendExecutionError, err.to_string())) async def _render_dataframe_preview(df: RustDataFrame) -> Result[str, BackendError]: - """Render one DataFusion preview table while preserving InQL's backend error envelope.""" + """Render one DataFusion preview table while preserving IncQL's backend error envelope.""" match await df.to_string(): Ok(rendered) => return Ok(rendered) Err(err) => return Err(backend_error(BackendErrorKind.BackendExecutionError, err.to_string())) async def _write_dataframe_csv(df: RustDataFrame, uri: str) -> Result[None, BackendError]: - """Write one DataFusion frame to CSV while preserving InQL's sink error envelope.""" + """Write one DataFusion frame to CSV while preserving IncQL's sink error envelope.""" match await df.write_csv(uri, DataFrameWriteOptions.new(), None): Ok(_) => return Ok(None) Err(err) => return Err(backend_error(BackendErrorKind.BackendSinkError, err.to_string())) async def _write_dataframe_parquet(df: RustDataFrame, uri: str) -> Result[None, BackendError]: - """Write one DataFusion frame to Parquet while preserving InQL's sink error envelope.""" + """Write one DataFusion frame to Parquet while preserving IncQL's sink error envelope.""" match await df.write_parquet(uri, DataFrameWriteOptions.new(), None): Ok(_) => return Ok(None) Err(err) => return Err(backend_error(BackendErrorKind.BackendSinkError, err.to_string())) @@ -1683,14 +1683,14 @@ def _unnest_plan_builder( datafusion_columns: list[Column], options: UnnestOptions, ) -> Result[LogicalPlanBuilder, BackendError]: - """Create a DataFusion unnest builder while preserving InQL's planning error envelope.""" + """Create a DataFusion unnest builder while preserving IncQL's planning error envelope.""" match LogicalPlanBuilder.from(logical_plan).unnest_columns_with_options(datafusion_columns, options): Ok(builder) => return Ok(builder) Err(err) => return Err(backend_error(BackendErrorKind.BackendPlanningError, err.to_string())) def _build_logical_plan(builder: LogicalPlanBuilder) -> Result[LogicalPlan, BackendError]: - """Build a DataFusion logical plan while preserving InQL's planning error envelope.""" + """Build a DataFusion logical plan while preserving IncQL's planning error envelope.""" match builder.build(): Ok(plan) => return Ok(plan) Err(err) => return Err(backend_error(BackendErrorKind.BackendPlanningError, err.to_string())) @@ -1701,7 +1701,7 @@ def _rename_generator_output_column( source_name: str, output_name: str, ) -> Result[RustDataFrame, BackendError]: - """Rename one generated DataFusion output column while preserving InQL's planning error envelope.""" + """Rename one generated DataFusion output column while preserving IncQL's planning error envelope.""" match df.with_column_renamed(source_name, output_name): Ok(next_df) => return Ok(next_df) Err(err) => return Err(backend_error(BackendErrorKind.BackendPlanningError, err.to_string())) @@ -1712,7 +1712,7 @@ def _rename_flat_generator_outputs( temp_columns: list[str], output_columns: list[str], ) -> Result[RustDataFrame, BackendError]: - """Rename generated scalar/list outputs from temporary names to InQL aliases.""" + """Rename generated scalar/list outputs from temporary names to IncQL aliases.""" if len(temp_columns) != len(output_columns): return Err(backend_error(BackendErrorKind.BackendPlanningError, "generator payload/output arity mismatch")) mut current = df @@ -1726,9 +1726,9 @@ def _rename_struct_generator_outputs( temp_column: str, output_columns: list[str], ) -> Result[RustDataFrame, BackendError]: - """Rename generated struct fields from temporary qualified names to InQL aliases.""" + """Rename generated struct fields from temporary qualified names to IncQL aliases.""" mut current = df - # DataFusion names unnested struct fields as `temp.field`; the public InQL result should expose only the declared + # DataFusion names unnested struct fields as `temp.field`; the public IncQL result should expose only the declared # generator output aliases. for output_column in output_columns: field_name = f"{temp_column}.{output_column}" @@ -1746,7 +1746,7 @@ def _extended_columns(input_columns: list[str], appended_columns: list[str]) -> def _generator_temp_columns(argument_count: int) -> list[str]: """Return stable temporary column aliases for lowered generator argument expressions.""" - return [f"__inql_generator_arg_{idx}" for idx in range(argument_count)] + return [f"__incql_generator_arg_{idx}" for idx in range(argument_count)] def _preserve_generator_nulls(extension_uri: str) -> bool: @@ -1807,7 +1807,7 @@ def _consumer_plan_from_current_plan(plan: Plan) -> Result[ConsumerPlan, Backend def _datafusion_producer_plan(plan: Plan) -> Plan: """Return a producer plan with extension names adapted for the DataFusion consumer only.""" - # Keep InQL/Substrait plans semantically owned by InQL. This rewrite exists only because DataFusion's Substrait + # Keep IncQL/Substrait plans semantically owned by IncQL. This rewrite exists only because DataFusion's Substrait # consumer recognizes different implementation names for these aggregate functions. return Plan( version=plan.version, diff --git a/src/session/datafusion_common_scalar_functions.incn b/src/session/datafusion_common_scalar_functions.incn index 167c082c..fcf39893 100644 --- a/src/session/datafusion_common_scalar_functions.incn +++ b/src/session/datafusion_common_scalar_functions.incn @@ -25,11 +25,11 @@ const _SECONDS_PER_DAY: int = 24 * _SECONDS_PER_HOUR @derive(Clone) enum DataFusionCommonScalarFunction(str): - """RFC 018 scalar UDFs that InQL must provide to DataFusion itself.""" + """RFC 018 scalar UDFs that IncQL must provide to DataFusion itself.""" - # InQL's current encoding helpers are text-to-text until RFC 026 introduces richer logical values. + # IncQL's current encoding helpers are text-to-text until RFC 026 introduces richer logical values. # DataFusion exposes regexp_like/regexp_replace natively; capture extraction stays adapter-owned. - # These date/time helpers are InQL catalog names whose current DataFusion adapter behavior is implemented here. + # These date/time helpers are IncQL catalog names whose current DataFusion adapter behavior is implemented here. Encode = "encode" Decode = "decode" RegexpExtract = "regexp_extract" diff --git a/src/session/datafusion_format_functions.incn b/src/session/datafusion_format_functions.incn index 1c65a1dd..b03445e8 100644 --- a/src/session/datafusion_format_functions.incn +++ b/src/session/datafusion_format_functions.incn @@ -1,4 +1,4 @@ -"""DataFusion execution bridge for InQL-owned scalar format functions.""" +"""DataFusion execution bridge for IncQL-owned scalar format functions.""" from rust::crc32fast import Hasher as Crc32Hasher from rust::datafusion::arrow::array import Array, ArrayRef, BooleanArray, Int64Array, MapArray, StringArray @@ -19,7 +19,7 @@ from std.json import JsonError, JsonKind, JsonValue @derive(Clone) enum DataFusionFormatFunction(str): - """Format UDFs that InQL must provide to DataFusion itself.""" + """Format UDFs that IncQL must provide to DataFusion itself.""" Sha1 = "sha1" Crc32 = "crc32" diff --git a/src/session/types.incn b/src/session/types.incn index b68b9761..4446ca88 100644 --- a/src/session/types.incn +++ b/src/session/types.incn @@ -197,7 +197,7 @@ pub class Session: source = csv_source(source_uri.0) columns = planned_schema_columns_for_source(source)? self._register_parsed(name, source)? - # TODO(InQL RFC 004): Formalize logical-name schema binding as an explicit catalog/snapshot model and surface overwrite diagnostics. + # TODO(IncQL RFC 004): Formalize logical-name schema binding as an explicit catalog/snapshot model and surface overwrite diagnostics. register_named_table_schema(name.0, columns) return self.table(name.0) diff --git a/src/sketch_types.incn b/src/sketch_types.incn index 8919d531..da2f6c07 100644 --- a/src/sketch_types.incn +++ b/src/sketch_types.incn @@ -31,7 +31,7 @@ pub enum SketchValueDomain(str): pub enum SketchSerializationFormat(str): """Explicit serialized sketch format identity.""" - InqlHllV1 = "inql_hll_v1" + IncqlHllV1 = "incql_hll_v1" @derive(Clone) @@ -55,7 +55,7 @@ pub model SketchExpr: pub def hll_type( value_domain: SketchValueDomain = SketchValueDomain.StringIdentifier, precision: int = 14, - format: SketchSerializationFormat = SketchSerializationFormat.InqlHllV1, + format: SketchSerializationFormat = SketchSerializationFormat.IncqlHllV1, ) -> SketchLogicalType: """ Build HyperLogLog logical type metadata. @@ -146,7 +146,7 @@ module tests: assert sketch_type.family == SketchFamily.HyperLogLog assert sketch_type.value_domain == SketchValueDomain.IntegerIdentifier assert sketch_type.precision == 12 - assert sketch_type.format == SketchSerializationFormat.InqlHllV1 + assert sketch_type.format == SketchSerializationFormat.IncqlHllV1 def test_hll_type_rejects_non_positive_precision() -> None: assert_raises[ValueError](_invalid_hll_precision) def test_sketch_col_preserves_expression_and_type() -> None: diff --git a/src/substrait/conformance.incn b/src/substrait/conformance.incn index 06c986ba..29e8e61c 100644 --- a/src/substrait/conformance.incn +++ b/src/substrait/conformance.incn @@ -1,5 +1,5 @@ """ -Public facade for InQL's Substrait conformance surface. +Public facade for IncQL's Substrait conformance surface. This module re-exports the two production-owned conformance layers: @@ -8,7 +8,7 @@ This module re-exports the two production-owned conformance layers: The split keeps ownership clean: -- catalog modules describe what InQL claims +- catalog modules describe what IncQL claims - validator modules check whether plans satisfy those claims - tests own fixture-plan construction diff --git a/src/substrait/conformance_catalog.incn b/src/substrait/conformance_catalog.incn index 4ac6dea7..470d2ab8 100644 --- a/src/substrait/conformance_catalog.incn +++ b/src/substrait/conformance_catalog.incn @@ -1,7 +1,7 @@ """ -Scenario catalog for InQL's Substrait conformance corpus. +Scenario catalog for IncQL's Substrait conformance corpus. -This module owns the machine-readable contract metadata for the current InQL Substrait profile: +This module owns the machine-readable contract metadata for the current IncQL Substrait profile: - stable scenario identifiers - scenario taxonomy and portability classification @@ -12,7 +12,7 @@ It does not own synthetic fixture plans. Production code defines the contract; t validate them against this catalog. The catalog is intentionally narrower than full semantic correctness. A scenario entry describes what boundary facts -InQL claims to emit or preserve in a plan. It does not, by itself, prove that higher-level query semantics are +IncQL claims to emit or preserve in a plan. It does not, by itself, prove that higher-level query semantics are complete. """ @@ -163,7 +163,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: match key: CoreScenarioKey.ReadNamedTable => return _core_scenario( - scenario_id=str("inql.substrait.core.read_named_table.001"), + scenario_id=str("incql.substrait.core.read_named_table.001"), title=str("ReadRel named table without secret material"), capability_tags=_capability_tags([str("read"), str("named-table")]), root_rel=ConformanceRel.Read, @@ -177,7 +177,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.ReadLocalFiles => return _core_scenario( - scenario_id=str("inql.substrait.core.read_local_files.001"), + scenario_id=str("incql.substrait.core.read_local_files.001"), title=str("ReadRel local files with portable format fields"), capability_tags=_capability_tags([str("read"), str("local-files")]), root_rel=ConformanceRel.Read, @@ -189,7 +189,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.ReadVirtualTable => return _core_scenario( - scenario_id=str("inql.substrait.core.read_virtual_table.001"), + scenario_id=str("incql.substrait.core.read_virtual_table.001"), title=str("ReadRel virtual table with schema-aligned rows"), capability_tags=_capability_tags([str("read"), str("virtual-table"), str("literal-rows")]), root_rel=ConformanceRel.Read, @@ -201,7 +201,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.FilterRows => return _core_scenario( - scenario_id=str("inql.substrait.core.filter_rows.001"), + scenario_id=str("incql.substrait.core.filter_rows.001"), title=str("FilterRel boolean row predicate"), capability_tags=_capability_tags([str("filter"), str("predicate")]), root_rel=ConformanceRel.Filter, @@ -215,7 +215,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.ProjectComputedColumns => return _core_scenario( - scenario_id=str("inql.substrait.core.project_computed_columns.001"), + scenario_id=str("incql.substrait.core.project_computed_columns.001"), title=str("ProjectRel boundary scaffold"), capability_tags=_capability_tags([str("project"), str("shape-scaffold")]), root_rel=ConformanceRel.Project, @@ -229,7 +229,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.JoinRelVariants => return _core_scenario( - scenario_id=str("inql.substrait.core.join_rel_variants.001"), + scenario_id=str("incql.substrait.core.join_rel_variants.001"), title=str("JoinRel variant emission boundary"), capability_tags=_capability_tags( [str("join"), str("inner"), str("left"), str("semi"), str("anti"), str("single"), str("mark")], @@ -245,7 +245,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.CrossRelCartesian => return _core_scenario( - scenario_id=str("inql.substrait.core.cross_rel_cartesian.001"), + scenario_id=str("incql.substrait.core.cross_rel_cartesian.001"), title=str("CrossRel cartesian product semantics"), capability_tags=_capability_tags([str("cross"), str("cartesian-product")]), root_rel=ConformanceRel.Cross, @@ -257,7 +257,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.AggregateGroupingSets => return _core_scenario( - scenario_id=str("inql.substrait.core.aggregate_grouping_sets.001"), + scenario_id=str("incql.substrait.core.aggregate_grouping_sets.001"), title=str("AggregateRel boundary scaffold"), capability_tags=_capability_tags([str("aggregate"), str("shape-scaffold")]), root_rel=ConformanceRel.Aggregate, @@ -271,7 +271,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.SortRelOrdering => return _core_scenario( - scenario_id=str("inql.substrait.core.sort_rel_ordering.001"), + scenario_id=str("incql.substrait.core.sort_rel_ordering.001"), title=str("SortRel deterministic ordering contract"), capability_tags=_capability_tags([str("sort"), str("order-by"), str("collation")]), root_rel=ConformanceRel.Sort, @@ -283,7 +283,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.FetchRelLimitOffset => return _core_scenario( - scenario_id=str("inql.substrait.core.fetch_rel_limit_offset.001"), + scenario_id=str("incql.substrait.core.fetch_rel_limit_offset.001"), title=str("FetchRel offset and count windowing"), capability_tags=_capability_tags([str("fetch"), str("limit"), str("offset")]), root_rel=ConformanceRel.Fetch, @@ -295,7 +295,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.SetRelOperations => return _core_scenario( - scenario_id=str("inql.substrait.core.set_rel_operations.001"), + scenario_id=str("incql.substrait.core.set_rel_operations.001"), title=str("SetRel operation emission boundary"), capability_tags=_capability_tags([str("set"), str("union"), str("intersect"), str("except")]), root_rel=ConformanceRel.Set, @@ -309,7 +309,7 @@ pub def core_scenario(key: CoreScenarioKey) -> SubstraitConformanceScenario: ) CoreScenarioKey.ReferenceRelSharedSubplan => return _core_scenario( - scenario_id=str("inql.substrait.core.reference_rel_shared_subplan.001"), + scenario_id=str("incql.substrait.core.reference_rel_shared_subplan.001"), title=str("ReferenceRel ordinal emission boundary"), capability_tags=_capability_tags([str("reference"), str("shared-subplan"), str("plan-dag")]), root_rel=ConformanceRel.Reference, diff --git a/src/substrait/conformance_validate.incn b/src/substrait/conformance_validate.incn index 5ebff906..f3457f15 100644 --- a/src/substrait/conformance_validate.incn +++ b/src/substrait/conformance_validate.incn @@ -13,7 +13,7 @@ Those belong in tests so production conformance code remains about contract vali generation. These validators are designed to prove boundary facts about emitted plans. They are not a complete semantic oracle for -every relational operation InQL may eventually support. +every relational operation IncQL may eventually support. """ from rust::substrait::proto import Plan diff --git a/src/substrait/expr_lowering.incn b/src/substrait/expr_lowering.incn index 551b9180..3bbca9f8 100644 --- a/src/substrait/expr_lowering.incn +++ b/src/substrait/expr_lowering.incn @@ -514,7 +514,7 @@ def _known_expression_kind( bindings: list[ResolvedProjectionBinding], expr: ColumnExpr, ) -> Result[Option[SubstraitPrimitiveKind], SubstraitLoweringError]: - """Return a known primitive expression kind, or `None` when InQL cannot infer it honestly.""" + """Return a known primitive expression kind, or `None` when IncQL cannot infer it honestly.""" match expr: BoolColumnExpr(_) => return Ok(Some(SubstraitPrimitiveKind.Bool)) IntColumnExpr(_) => return Ok(Some(SubstraitPrimitiveKind.I64)) diff --git a/src/substrait/extensions.incn b/src/substrait/extensions.incn index 06aecac4..d82e4677 100644 --- a/src/substrait/extensions.incn +++ b/src/substrait/extensions.incn @@ -1,5 +1,5 @@ """ -Substrait extension and function-anchor bookkeeping for InQL. +Substrait extension and function-anchor bookkeeping for IncQL. This module derives extension declarations from registry metadata and collects required extension URNs over relation and expression trees. @@ -328,7 +328,7 @@ def _aggregate_function_uses_if_then(aggregate_function: AggregateFunction) -> b def _generator_payload_arguments(extension: ExtensionSingleRel) -> list[Expression]: - """Return lowered generator payload arguments when the extension carries InQL generator metadata.""" + """Return lowered generator payload arguments when the extension carries IncQL generator metadata.""" match extension.detail: Some(detail) => if not is_generator_extension_uri(detail.type_url): diff --git a/src/substrait/function_extensions.incn b/src/substrait/function_extensions.incn index 4d327273..d604fd2a 100644 --- a/src/substrait/function_extensions.incn +++ b/src/substrait/function_extensions.incn @@ -1,5 +1,5 @@ """ -Stable Substrait extension anchors and URIs for registry-backed InQL functions. +Stable Substrait extension anchors and URIs for registry-backed IncQL functions. Function helpers import anchors from this module when declaring their registry metadata. Runtime extension declarations derive names and kinds from those registry entries instead of keeping a second hand-written function catalog. @@ -196,15 +196,15 @@ pub const IS_STRING_FUNCTION_ANCHOR: u32 = 168 pub const IS_TIMESTAMP_FUNCTION_ANCHOR: u32 = 169 pub const IS_ARRAY_FUNCTION_ANCHOR: u32 = 170 pub const IS_OBJECT_FUNCTION_ANCHOR: u32 = 171 -pub const FUNCTION_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/functions.yaml" -pub const EXPLODE_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/unnest.yaml#explode" -pub const EXPLODE_OUTER_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/unnest.yaml#explode_outer" -pub const POSEXPLODE_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/unnest.yaml#posexplode" -pub const POSEXPLODE_OUTER_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/unnest.yaml#posexplode_outer" -pub const INLINE_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/unnest.yaml#inline" -pub const INLINE_OUTER_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/unnest.yaml#inline_outer" -pub const FLATTEN_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/unnest.yaml#flatten" -pub const STACK_EXTENSION_URI: str = "https://inql.io/extensions/v0.1/table_functions.yaml#stack" +pub const FUNCTION_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/functions.yaml" +pub const EXPLODE_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/unnest.yaml#explode" +pub const EXPLODE_OUTER_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/unnest.yaml#explode_outer" +pub const POSEXPLODE_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/unnest.yaml#posexplode" +pub const POSEXPLODE_OUTER_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/unnest.yaml#posexplode_outer" +pub const INLINE_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/unnest.yaml#inline" +pub const INLINE_OUTER_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/unnest.yaml#inline_outer" +pub const FLATTEN_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/unnest.yaml#flatten" +pub const STACK_EXTENSION_URI: str = "https://incql.io/extensions/v0.1/table_functions.yaml#stack" pub def registered_substrait_extension_uris() -> list[str]: @@ -213,7 +213,7 @@ pub def registered_substrait_extension_uris() -> list[str]: pub def is_generator_extension_uri(extension_uri: str) -> bool: - """Return whether an ExtensionSingle URI belongs to an InQL generator.""" + """Return whether an ExtensionSingle URI belongs to an IncQL generator.""" return ( extension_uri == EXPLODE_EXTENSION_URI or extension_uri == EXPLODE_OUTER_EXTENSION_URI diff --git a/src/substrait/generator_payload.incn b/src/substrait/generator_payload.incn index c25b234e..ae4614f3 100644 --- a/src/substrait/generator_payload.incn +++ b/src/substrait/generator_payload.incn @@ -1,4 +1,4 @@ -"""Binary payload for InQL generator relation-extension nodes.""" +"""Binary payload for IncQL generator relation-extension nodes.""" from rust::incan_stdlib::errors import raise_value_error from rust::prost import Message @@ -6,8 +6,8 @@ from rust::std::string import String as RustString from rust::substrait::proto import Expression from std.io import BytesIO, Endian, IoError, _BytesIO -# ASCII "INQLGEN1"; this lets decoders reject unrelated relation-extension payloads before reading length fields. -const _MAGIC: list[int] = [73, 78, 81, 76, 71, 69, 78, 49] +# ASCII "INCQLGEN1"; this lets decoders reject unrelated relation-extension payloads before reading length fields. +const _MAGIC: list[int] = [73, 78, 67, 81, 76, 71, 69, 78, 49] @derive(Clone) @@ -117,7 +117,7 @@ def _write_payload_bytes(out: _BytesIO, data: bytes) -> None: def _write_magic(out: _BytesIO) -> None: """Write the generator payload magic through std.io.""" - _write_payload_bytes(out, b"INQLGEN1") + _write_payload_bytes(out, b"INCQLGEN1") return diff --git a/src/substrait/inspect.incn b/src/substrait/inspect.incn index 007ab44d..bbc0583f 100644 --- a/src/substrait/inspect.incn +++ b/src/substrait/inspect.incn @@ -167,7 +167,7 @@ def _sketch_measure_suffix(options: list[FunctionOption]) -> str: suffix = suffix + "_" + value_domain if precision != "" and precision != "14": suffix = suffix + "_p" + precision - if format != "" and format != "inql_hll_v1": + if format != "" and format != "incql_hll_v1": suffix = suffix + "_" + format return suffix @@ -232,7 +232,7 @@ def _window_output_columns(window_rel: ConsistentPartitionWindowRel) -> list[str def _window_function_output_name(options: list[FunctionOption]) -> str: - """Return the InQL output alias encoded on one lowered window function.""" + """Return the IncQL output alias encoded on one lowered window function.""" for option in options: if option.name == WINDOW_OUTPUT_NAME_OPTION and len(option.preference) > 0: return option.preference[0] diff --git a/src/substrait/mod.incn b/src/substrait/mod.incn index 524e62c5..51798533 100644 --- a/src/substrait/mod.incn +++ b/src/substrait/mod.incn @@ -1,5 +1,5 @@ """ -Substrait support for InQL. +Substrait support for IncQL. The implementation is split into focused owner modules: diff --git a/src/substrait/plans.incn b/src/substrait/plans.incn index 48a9c467..16d8b302 100644 --- a/src/substrait/plans.incn +++ b/src/substrait/plans.incn @@ -1,5 +1,5 @@ """ -Top-level Substrait plan assembly helpers for InQL. +Top-level Substrait plan assembly helpers for IncQL. This module owns `Plan` envelopes, package-level producer/version metadata, and the entrypoints that wrap root relations into complete proto-backed plans. @@ -17,7 +17,7 @@ from substrait.relations import ( ) const DEFAULT_SUBSTRAIT_RELEASE_TAG: str = "v0.63.0" -const DEFAULT_SUBSTRAIT_PRODUCER_NAME: str = "inql-rfc002" +const DEFAULT_SUBSTRAIT_PRODUCER_NAME: str = "incql-rfc002" def _default_version() -> Version: diff --git a/src/substrait/relations.incn b/src/substrait/relations.incn index 91093ad6..4f29129b 100644 --- a/src/substrait/relations.incn +++ b/src/substrait/relations.incn @@ -462,7 +462,7 @@ def _resolved_window_projection( def _window_function_options(output_name: str, application: WindowFunctionApplication) -> list[FunctionOption]: - """Encode InQL-only window metadata that Substrait consumers must preserve.""" + """Encode IncQL-only window metadata that Substrait consumers must preserve.""" return [FunctionOption(name=WINDOW_OUTPUT_NAME_OPTION, preference=[output_name]), FunctionOption( name=WINDOW_NULL_TREATMENT_OPTION, preference=[application.null_treatment.value()], @@ -477,14 +477,14 @@ def _window_invocation(application: WindowFunctionApplication) -> AggregationInv def _window_bounds_type(frame: WindowFrame) -> BoundsType: - """Map InQL frame kind to Substrait bounds type.""" + """Map IncQL frame kind to Substrait bounds type.""" match frame.kind: WindowFrameKind.Rows => return BoundsType.Rows WindowFrameKind.Range => return BoundsType.Range def _window_bound(bound: WindowBound) -> Bound: - """Map one InQL frame bound to the Substrait window-bound oneof.""" + """Map one IncQL frame bound to the Substrait window-bound oneof.""" match bound.kind: WindowBoundKind.UnboundedPreceding => return Bound(kind=Some(BoundKind.Unbounded(SubstraitUnbounded()))) WindowBoundKind.Preceding => @@ -710,7 +710,7 @@ def _sort_field_direction_detail(application: ScalarFunctionApplicationExpr) -> def _sort_field(input_columns: list[str], key: ColumnExpr) -> Result[SortField, SubstraitLoweringError]: - """Lower one InQL sort-key expression into a Substrait SortField.""" + """Lower one IncQL sort-key expression into a Substrait SortField.""" return _sort_field_for_columns(scalar_columns_from_names(input_columns), key) @@ -718,7 +718,7 @@ def _sort_field_for_columns( input_columns: list[ScalarColumnSpec], key: ColumnExpr, ) -> Result[SortField, SubstraitLoweringError]: - """Lower one InQL sort-key expression against typed-or-name-only input columns.""" + """Lower one IncQL sort-key expression against typed-or-name-only input columns.""" key_expr = untyped_column_expr(key) match key_expr: ScalarFunctionApplicationExpr(application) => diff --git a/src/substrait/schema.incn b/src/substrait/schema.incn index 59063859..eeee5c4c 100644 --- a/src/substrait/schema.incn +++ b/src/substrait/schema.incn @@ -56,7 +56,7 @@ pub def row_shape_field_names(shape: RowShapeSpec) -> list[str]: def _primitive_kind_from_type_name(type_name: str) -> SubstraitPrimitiveKind: - """Map Incan field reflection type names to primitive schema kinds InQL can lower today.""" + """Map Incan field reflection type names to primitive schema kinds IncQL can lower today.""" if type_name == "bool": return SubstraitPrimitiveKind.Bool if type_name == "int": @@ -73,7 +73,7 @@ def _primitive_kind_from_type_name(type_name: str) -> SubstraitPrimitiveKind: return SubstraitPrimitiveKind.F64List if type_name == "list[str]" or type_name == "List[str]": return SubstraitPrimitiveKind.StringList - message = f"unsupported model field type `{type_name}` for InQL row shape" + message = f"unsupported model field type `{type_name}` for IncQL row shape" return raise_value_error(message) diff --git a/src/substrait/window_metadata.incn b/src/substrait/window_metadata.incn index 5bf6a451..db7f7af5 100644 --- a/src/substrait/window_metadata.incn +++ b/src/substrait/window_metadata.incn @@ -1,4 +1,4 @@ -"""Shared metadata keys for InQL window-function annotations in Substrait.""" +"""Shared metadata keys for IncQL window-function annotations in Substrait.""" -pub const WINDOW_OUTPUT_NAME_OPTION: str = "inql_output_name" +pub const WINDOW_OUTPUT_NAME_OPTION: str = "incql_output_name" pub const WINDOW_NULL_TREATMENT_OPTION: str = "null_treatment" diff --git a/src/window_builders.incn b/src/window_builders.incn index e525df4b..3a05c33f 100644 --- a/src/window_builders.incn +++ b/src/window_builders.incn @@ -419,7 +419,7 @@ module tests: def test_ranking_call_over_records_registry_identity() -> None: application = rank().over(window().order_by([col("amount")])) assert application.kind == WindowFunctionKind.Rank - assert application.function_ref == "inql.functions.rank" + assert application.function_ref == "incql.functions.rank" assert application.requires_ordering def test_offset_and_value_calls_record_arguments() -> None: lagged = lag(col("amount"), 2).ignore_nulls() diff --git a/tests/incan.lock b/tests/incan.lock index 48d410d4..1a386121 100644 --- a/tests/incan.lock +++ b/tests/incan.lock @@ -1851,7 +1851,7 @@ dependencies = [ ] [[package]] -name = "inql" +name = "incql" version = "0.2.0-rc8" dependencies = [ "datafusion", diff --git a/tests/test_dataset.incn b/tests/test_dataset.incn index 67e299d1..72bd0cb9 100644 --- a/tests/test_dataset.incn +++ b/tests/test_dataset.incn @@ -1,7 +1,7 @@ """Test: dataset types and RFC 001 contract (types, hierarchy, functions import).""" from std.testing import fail_t, parametrize -from metadata import inql_version +from metadata import incql_version from aggregate_builders import AggregateKind from dataset import DataSet, BoundedDataSet, UnboundedDataSet, DataFrame, LazyFrame, DataStream, lazy_frame_named_table from dataset.materialization import DataFrameMaterialization @@ -324,16 +324,16 @@ def test_type_contracts__concrete_and_trait_types_match_generic_arguments() -> N _touch(ub) -def test_version__inql_version_is_published() -> None: - """InQL version should be available from metadata.""" +def test_version__incql_version_is_published() -> None: + """IncQL version should be available from metadata.""" # -- Arrange -- expected_version = "0.1.0" # -- Act -- - version = inql_version() + version = incql_version() # -- Assert -- - assert version == expected_version, "InQL version should be 0.1.0" + assert version == expected_version, "IncQL version should be 0.1.0" def test_dataset_ops__method_wrapper_matches_canonical_function() -> None: diff --git a/tests/test_function_registry.incn b/tests/test_function_registry.incn index 0fe9e2ef..1710e3fb 100644 --- a/tests/test_function_registry.incn +++ b/tests/test_function_registry.incn @@ -1206,7 +1206,7 @@ def test_function_registry__extension_policy_is_separate_from_scalar_class() -> """Assert extension-only functions can be scalar while remaining explicitly namespaced.""" # -- Arrange -- mut registry = FunctionRegistry.new() - extension_namespace = "inql_ext.geo" + extension_namespace = "incql_ext.geo" # -- Act -- registry._add_entry( @@ -1225,7 +1225,7 @@ def test_function_registry__extension_policy_is_separate_from_scalar_class() -> # -- Assert -- assert extension_entry.namespace == extension_namespace, "extension-only entries should preserve explicit namespace" - assert extension_entry.function_ref == "inql_ext.geo.st_srid", "extension-only entries should derive namespaced function refs" + assert extension_entry.function_ref == "incql_ext.geo.st_srid", "extension-only entries should derive namespaced function refs" assert extension_entry.policy_category == FunctionPolicyCategory.ExtensionOnly, "extension-only entries should expose policy category" assert extension_entry.function_class == FunctionClass.Scalar, "extension-only policy should not replace semantic function class" assert extension_entry.substrait.kind == SubstraitMappingKind.ExtensionFunction, "extension-only entries may declare Substrait extension mappings" @@ -1257,7 +1257,7 @@ def test_function_registry__canonical_name_lookup_stays_core_scoped() -> None: """Assert explicit namespaces keep extension names from shadowing portable core names.""" # -- Arrange -- mut registry = FunctionRegistry.new() - extension_namespace = "inql_ext.crypto" + extension_namespace = "incql_ext.crypto" # -- Act -- registry._add_entry( diff --git a/tests/test_governance.incn b/tests/test_governance.incn index 737bb6a5..9b6ef00d 100644 --- a/tests/test_governance.incn +++ b/tests/test_governance.incn @@ -57,7 +57,7 @@ def test_governed_attribute__carries_provenance_confidence_status_and_lifetime() assert attribute.attribute_id == "governed:plan:fixture:field:customer_email:classification", "attribute identity should include target and key" assert attribute.target.name == "customer_email", "attribute should stay anchored to its semantic target" assert attribute.key == "classification", "attribute should carry its governed key" - assert attribute.value.schema == "inql.governance.string.v0.1", "string helpers should declare their payload schema" + assert attribute.value.schema == "incql.governance.string.v0.1", "string helpers should declare their payload schema" assert attribute.value.value == "personal_data", "attribute should carry its typed payload value" assert attribute.scope == GovernedAttributeScope.Field, "field attributes should declare their scope" assert attribute.source == GovernedAttributeSource.User, "attribute provenance should be explicit" diff --git a/tests/test_inspection.incn b/tests/test_inspection.incn index 26776d27..4079204c 100644 --- a/tests/test_inspection.incn +++ b/tests/test_inspection.incn @@ -157,7 +157,7 @@ def test_inspect_plan__exposes_stable_plan_targets_schema_and_artifacts() -> Non assert inspection.output_fields[1].name == "total_amount", "explicit aggregate alias should become output field target" assert inspection.output_fields[0].target_id != "customer_id", "field identity must not be the display name alone" assert len(inspection.attachments) >= 3, "inspection should emit typed metadata attachments for versions and known schema facts" - assert inspection.attachments[0].payload.schema == "inql.metadata.string.v0.1", "metadata payloads should declare a schema" + assert inspection.attachments[0].payload.schema == "incql.metadata.string.v0.1", "metadata payloads should declare a schema" assert len(inspection.governed_attributes) >= 1, "inspection should expose inferred governed attributes for known fields" assert len(inspection.policy_checkpoints) == 1, "inspection should expose the local planning checkpoint record" assert len(inspection.artifacts) == 9, "inspection should expose the full deterministic artifact family set" @@ -200,7 +200,7 @@ def test_inspect_plan__exposes_governed_attributes_and_policy_checkpoints() -> N for attribute in inspection.governed_attributes: if attribute.target.name == "customer_id" and attribute.key == "schema.substrait_primitive_kind": schema_attribute_found = true - assert attribute.value.schema == "inql.governance.schema-primitive-kind.v0.1", "schema governed attributes should use a governance payload schema" + assert attribute.value.schema == "incql.governance.schema-primitive-kind.v0.1", "schema governed attributes should use a governance payload schema" assert attribute.value.value == "i64", "schema governed attributes should preserve primitive kind" assert attribute.source == GovernedAttributeSource.Lineage, "inspection-derived field attributes should name lineage/schema evidence as source" assert attribute.status == GovernedAttributeStatus.Inferred, "inspection-derived field attributes should not claim asserted authority" diff --git a/tests/test_sketch_functions.incn b/tests/test_sketch_functions.incn index a07b9837..eca38017 100644 --- a/tests/test_sketch_functions.incn +++ b/tests/test_sketch_functions.incn @@ -35,7 +35,7 @@ def test_sketch_functions__hll_type_preserves_merge_compatibility_metadata() -> assert left.family == SketchFamily.HyperLogLog, "hll_type should identify the portable sketch family" assert left.value_domain == SketchValueDomain.IntegerIdentifier, "hll_type should preserve the value domain" assert left.precision == 12, "hll_type should preserve the precision" - assert left.format == SketchSerializationFormat.InqlHllV1, "hll_type should preserve the serialization format" + assert left.format == SketchSerializationFormat.IncqlHllV1, "hll_type should preserve the serialization format" assert sketch_types_compatible(left, right), "matching HLL logical types should be merge-compatible" assert not sketch_types_compatible(left, different_precision), "precision is part of sketch merge compatibility" @@ -119,7 +119,7 @@ def test_sketch_functions__hll_scalar_helpers_preserve_sketch_options() -> None: assert column_expr_option_value(estimate, SKETCH_FAMILY_OPTION) == "hyperloglog", "hll_estimate should carry sketch family" assert column_expr_option_value(estimate, SKETCH_VALUE_DOMAIN_OPTION) == "integer_identifier", "hll_estimate should carry value domain" assert column_expr_option_value(estimate, SKETCH_PRECISION_OPTION) == "12", "hll_estimate should carry precision" - assert column_expr_option_value(estimate, SKETCH_FORMAT_OPTION) == "inql_hll_v1", "hll_estimate should carry format" + assert column_expr_option_value(estimate, SKETCH_FORMAT_OPTION) == "incql_hll_v1", "hll_estimate should carry format" assert column_expr_function_name(serialized) == "hll_serialize", "hll_serialize should be a registry-backed scalar helper" assert column_expr_option_value(serialized, SKETCH_PRECISION_OPTION) == "12", "hll_serialize should carry precision" diff --git a/tests/test_substrait_plan.incn b/tests/test_substrait_plan.incn index 52952aaa..0b610bff 100644 --- a/tests/test_substrait_plan.incn +++ b/tests/test_substrait_plan.incn @@ -520,7 +520,7 @@ def test_plan__ordering_helper_is_invalid_outside_sort_context() -> None: match result: Err(err) => assert err.kind == SubstraitLoweringErrorKind.InvalidScalarExpression, "context errors should be invalid scalar expressions" - assert err.message.contains("inql.functions.asc"), "diagnostic should name the registered function reference" + assert err.message.contains("incql.functions.asc"), "diagnostic should name the registered function reference" assert err.message.contains("sort_field context"), "diagnostic should explain the valid lowering context" Ok(_) => return fail_t("asc should not lower as a standalone scalar expression") @@ -968,7 +968,7 @@ def test_plan__aggregate_rel_lowers_approximate_aggregates() -> None: assert relation_kind_name(aggregated) == "AggregateRel", "approximate aggregate lowering should emit AggregateRel" assert plan_has_extension_urn(plan, registered_substrait_extension_uris()[0]), "approximate aggregate plans should register aggregate extensions" assert output_columns == ["id", "approx_count_distinct_id", "approx_percentile_id_p0_5_a10000", "approx_percentile_id_p0_95_a2500"], "approximate aggregate output names should preserve percentile and accuracy parameters" - assert aggregate_functions == ["approx_count_distinct", "approx_percentile", "approx_percentile"], "approximate aggregates should keep their InQL Substrait extension names" + assert aggregate_functions == ["approx_count_distinct", "approx_percentile", "approx_percentile"], "approximate aggregates should keep their IncQL Substrait extension names" assert aggregate_invocations == ["All", "All", "All"], "approximate aggregates should not lower as exact DISTINCT invocations" @@ -991,7 +991,7 @@ def test_plan__aggregate_rel_lowers_typed_hll_sketches() -> None: # -- Assert -- assert relation_kind_name(aggregated) == "AggregateRel", "typed sketch aggregates should still lower through AggregateRel" assert output_columns == ["id", "hll_sketch_id_p12", "hll_merge_id_p12"], "typed sketch aggregate outputs should include non-default sketch compatibility metadata" - assert aggregate_functions == ["hll_sketch", "hll_merge"], "typed sketch aggregates should keep their InQL extension names" + assert aggregate_functions == ["hll_sketch", "hll_merge"], "typed sketch aggregates should keep their IncQL extension names" assert aggregate_measure_option_value(aggregated, 0, SKETCH_FAMILY_OPTION) == "hyperloglog", "hll_sketch should carry sketch family metadata" assert aggregate_measure_option_value(aggregated, 0, SKETCH_PRECISION_OPTION) == "12", "hll_sketch should carry sketch precision metadata" assert aggregate_measure_option_value(aggregated, 1, SKETCH_FAMILY_OPTION) == "hyperloglog", "hll_merge should carry sketch family metadata" @@ -1323,7 +1323,7 @@ def test_plan__revision_pin_and_extension_registry_are_exported() -> None: # -- Assert -- assert tag == "v0.63.0", "revision helpers should expose the currently targeted Substrait release tag" - assert producer == "inql-rfc002", "revision helpers should expose the package producer label" + assert producer == "incql-rfc002", "revision helpers should expose the package producer label" assert len(registered) == 9, "current package boundary should register function and generator extension URIs" assert registered[0] == FUNCTION_EXTENSION_URI, "registry should include the shared function extension URI first" assert registered[1] == EXPLODE_EXTENSION_URI, "registry should include the emitted explode extension URI" diff --git a/tests/test_window_functions.incn b/tests/test_window_functions.incn index bb0aa55c..ce322e67 100644 --- a/tests/test_window_functions.incn +++ b/tests/test_window_functions.incn @@ -55,7 +55,7 @@ def test_window_builders__ranking_helpers_return_unplaced_calls() -> None: assert rank_app.kind == WindowFunctionKind.Rank, "rank should keep typed window identity" assert dense_rank_app.kind == WindowFunctionKind.DenseRank, "dense_rank should keep typed window identity" assert row_number_app.requires_ordering, "ranking helpers should require explicit window ordering" - assert rank_app.function_ref == "inql.functions.rank", "rank should derive stable registry identity" + assert rank_app.function_ref == "incql.functions.rank", "rank should derive stable registry identity" assert dense_rank_app.canonical_name == "dense_rank", "dense_rank should expose its canonical name" diff --git a/vocab_companion/Cargo.lock b/vocab_companion/Cargo.lock index a7a37aa8..e6db8e6d 100644 --- a/vocab_companion/Cargo.lock +++ b/vocab_companion/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ ] [[package]] -name = "inql_vocab_companion" +name = "incql_vocab_companion" version = "0.1.0" dependencies = [ "incan_vocab", diff --git a/vocab_companion/Cargo.toml b/vocab_companion/Cargo.toml index e870d1e8..ce451c2c 100644 --- a/vocab_companion/Cargo.toml +++ b/vocab_companion/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "inql_vocab_companion" +name = "incql_vocab_companion" version = "0.1.0" edition = "2021" diff --git a/vocab_companion/src/desugar.rs b/vocab_companion/src/desugar.rs index c4a30499..b05d6b0b 100644 --- a/vocab_companion/src/desugar.rs +++ b/vocab_companion/src/desugar.rs @@ -8,7 +8,7 @@ use incan_vocab::{ use crate::{helper_exported, QUERY_FIELD_DESCRIPTOR, QUALITY_KW, QUERY_KW}; #[derive(Default)] -pub struct InqlVocabDesugarer; +pub struct IncqlVocabDesugarer; struct PendingJoin { source: IncanExpr, @@ -16,7 +16,7 @@ struct PendingJoin { method_name: &'static str, } -impl VocabDesugarer for InqlVocabDesugarer { +impl VocabDesugarer for IncqlVocabDesugarer { fn desugar(&self, node: &VocabSyntaxNode) -> Result { let declaration = match node { VocabSyntaxNode::Declaration(declaration) @@ -26,12 +26,12 @@ impl VocabDesugarer for InqlVocabDesugarer { } VocabSyntaxNode::Declaration(_) => { return Err(DesugarError::new( - "InQL desugarer expected a `query:` or `quality:` declaration", + "IncQL desugarer expected a `query:` or `quality:` declaration", )); } _ => { return Err(DesugarError::new( - "InQL query desugarer expected a declaration node", + "IncQL query desugarer expected a declaration node", )) } }; @@ -39,7 +39,7 @@ impl VocabDesugarer for InqlVocabDesugarer { QUERY_KW => Ok(DesugarOutput::Expression(lower_query(declaration)?)), QUALITY_KW => Ok(DesugarOutput::Expression(lower_quality(declaration)?)), _ => Err(DesugarError::new( - "InQL desugarer received an unsupported declaration keyword", + "IncQL desugarer received an unsupported declaration keyword", )), } } @@ -555,7 +555,7 @@ fn lower_expression_list(clause: &VocabClause) -> Result, Desugar fn lower_column_expr(expr: &IncanExpr) -> Result { // The vocab AST distinguishes query-only field shorthands from ordinary Incan expressions. Normalize every - // supported query expression into the public InQL helper surface before the compiler typechecks the generated call. + // supported query expression into the public IncQL helper surface before the compiler typechecks the generated call. match expr { IncanExpr::ScopedSurface(surface) if surface.descriptor_key == QUERY_FIELD_DESCRIPTOR => { if let IncanScopedSurfacePayload::LeadingDotPath { segments, .. } = &surface.payload { diff --git a/vocab_companion/src/lib.rs b/vocab_companion/src/lib.rs index e7161369..b1ce2ba0 100644 --- a/vocab_companion/src/lib.rs +++ b/vocab_companion/src/lib.rs @@ -1,7 +1,7 @@ -//! InQL vocabulary companion. +//! IncQL vocabulary companion. //! -//! The Incan compiler owns the generic vocabulary contract. InQL owns package-specific surfaces such as `query:` -//! and `quality:`, then lowers them into ordinary InQL helper/method calls that continue through Prism, Substrait, +//! The Incan compiler owns the generic vocabulary contract. IncQL owns package-specific surfaces such as `query:` +//! and `quality:`, then lowers them into ordinary IncQL helper/method calls that continue through Prism, Substrait, //! quality observation, and the active backend. mod desugar; @@ -12,15 +12,15 @@ use incan_vocab::{ ScopedSurfaceEligibility, ScopedSurfaceMisuseScope, ScopedSurfaceReceiver, VocabRegistration, }; -pub use desugar::InqlVocabDesugarer; +pub use desugar::IncqlVocabDesugarer; -pub const NAMESPACE: &str = "inql"; +pub const NAMESPACE: &str = "incql"; pub const QUERY_KW: &str = "query"; pub const QUALITY_KW: &str = "quality"; -pub const QUERY_FIELD_DESCRIPTOR: &str = "inql.query.field"; +pub const QUERY_FIELD_DESCRIPTOR: &str = "incql.query.field"; // Incan's current vocab manifest API requires desugarers to name helper bindings explicitly. Keep these lists in -// lock-step with the public helper surfaces so vocab blocks do not silently drift away from ordinary `pub::inql` +// lock-step with the public helper surfaces so vocab blocks do not silently drift away from ordinary `pub::incql` // helper calls. const QUERY_BLOCK_HELPER_EXPORTS: &[&str] = &[ "col", @@ -312,9 +312,9 @@ pub fn library_vocab() -> VocabRegistration { .with_misuse_scope(ScopedSurfaceMisuseScope::ActivatingFile) .with_diagnostic( ScopedSurfaceDiagnosticTemplate::new( - "inql-query-field-outside-scope", + "incql-query-field-outside-scope", ScopedSurfaceDiagnosticKind::OutsideScope, - "query field shorthand is only valid inside InQL query clauses", + "query field shorthand is only valid inside IncQL query clauses", ) .with_help( "move the leading-dot field reference into a `query:` clause", @@ -326,7 +326,7 @@ pub fn library_vocab() -> VocabRegistration { helper_bindings: helper_bindings(), ..LibraryManifest::default() }) - .with_desugarer(InqlVocabDesugarer) + .with_desugarer(IncqlVocabDesugarer) } fn helper_bindings() -> Vec { @@ -344,7 +344,7 @@ pub(crate) fn helper_exported(name: &str) -> bool { QUERY_BLOCK_HELPER_EXPORTS.contains(&name) || QUALITY_BLOCK_HELPER_EXPORTS.contains(&name) } -incan_vocab::export_wasm_desugarer!(InqlVocabDesugarer); +incan_vocab::export_wasm_desugarer!(IncqlVocabDesugarer); #[cfg(test)] mod tests {