diff --git a/.github/workflows/rfc-index.yml b/.github/workflows/rfc-index.yml new file mode 100644 index 00000000..ad3a8cd5 --- /dev/null +++ b/.github/workflows/rfc-index.yml @@ -0,0 +1,37 @@ +name: RFC index + +on: + push: + branches: [main] + paths: + - "docs/rfcs/**" + - "scripts/generate_rfc_index.py" + - ".github/workflows/rfc-index.yml" + pull_request: + branches: [main] + paths: + - "docs/rfcs/**" + - "scripts/generate_rfc_index.py" + - ".github/workflows/rfc-index.yml" + workflow_dispatch: + +concurrency: + group: rfc-index-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + name: Check generated RFC index + runs-on: ubuntu-latest + + steps: + - name: Check out IncQL + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Check RFC index + run: python3 scripts/generate_rfc_index.py --check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aab03c0b..8e2e3df0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,22 @@ See [docs/architecture.md][architecture] for a concise map. In short: 2. **Implement** in the right place: library APIs here when they are ordinary Incan code; compiler or stdlib changes in the Incan repo as needed. 3. **Keep the [README][readme] and docs accurate** for anything a new user or contributor would notice. +### RFC index + +RFC files are the source of truth for the RFC index. After adding or changing an RFC header, regenerate the index: + +```bash +make rfc-index +``` + +Use the check target before committing: + +```bash +make rfc-index-check +``` + +The generated table in `docs/rfcs/README.md` must not be edited by hand; edit the RFC file header instead. + ### Function docstrings - Every function or method with a body (`def ...`) in `.incn` files must include a docstring. diff --git a/Makefile b/Makefile index 495bd542..001f790c 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,16 @@ registry-metadata: ## Validate RFC 014 function registry checked API metadata @$(INCAN) tools metadata api . --format json > target/function_registry_api_metadata.json @RUSTFLAGS="-Awarnings" $(INCAN) run scripts/check_function_registry_metadata.incn +.PHONY: rfc-index +rfc-index: ## Regenerate docs/rfcs/README.md from RFC markdown files + @echo "\033[1mGenerating RFC index...\033[0m" + @python3 scripts/generate_rfc_index.py + +.PHONY: rfc-index-check +rfc-index-check: ## Check that docs/rfcs/README.md is in sync with RFC files + @echo "\033[1mChecking RFC index...\033[0m" + @python3 scripts/generate_rfc_index.py --check + .PHONY: build-locked build-locked: ## Build with `--locked` (stricter; requires current `incan.lock`) @echo "\033[1mBuilding IncQL library (locked)...\033[0m" @@ -110,15 +120,16 @@ fmt-check: ## Check formatting without writing (`incan fmt --check` per director # ============================================================================= .PHONY: check -check: fmt-check test-style vocab-companion-test registry-metadata build test ## Format check, style gate, metadata check, build, and test +check: rfc-index-check fmt-check test-style vocab-companion-test registry-metadata build test ## RFC index, format, style, metadata, build, and test @echo "\033[32m✓ check passed\033[0m" .PHONY: pre-commit -pre-commit: fmt-check test-style vocab-companion-test registry-metadata build test ## Fast gate before commit (same as `check`) +pre-commit: rfc-index-check fmt-check test-style vocab-companion-test registry-metadata build test ## Fast gate before commit (same as `check`) @echo "\033[32m✓ pre-commit gate passed\033[0m" .PHONY: ci ci: fmt-check test-style vocab-companion-test registry-metadata build test smoke-consumer ## Same steps as GitHub Actions `incql` job +ci: rfc-index-check fmt-check test-style vocab-companion-test registry-metadata build test smoke-consumer ## Same steps as GitHub Actions `incql` job @echo "\033[32m✓ ci gate passed\033[0m" .PHONY: verify diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md index ce62ec55..e6ce5905 100644 --- a/docs/rfcs/README.md +++ b/docs/rfcs/README.md @@ -4,60 +4,63 @@ IncQL uses its **own** RFC series (starting at 000), independent of the [Incan l **New RFC:** copy [TEMPLATE.md], name the file `NNN_short_slug.md`, pick the next number from the table (or from open issues), and open a PR. Section order and header fields follow that template. For workflow and conventions, see [Writing IncQL RFCs]. -| RFC | Status | Title | | -| -------------- | ----------- | ------------------------------------------------------------------------------------------------- | --- | -| [000][rfc-000] | Planned | Language specification — core model, naming, schema shapes, layer boundaries | | -| [001][rfc-001] | In Progress | Dataset types and carriers (`DataSet[T]`, `BoundedDataSet[T]`, `UnboundedDataSet[T]`) | | -| [002][rfc-002] | In Progress | Apache Substrait — `Rel`-level contract, mapping catalog, binding boundaries | | -| [003][rfc-003] | Implemented | `query {}` blocks — grammar, typing, Substrait lowering | | -| [004][rfc-004] | In Progress | Execution context — session, DataFusion, read/transform/write | | -| [005][rfc-005] | Blocked | Pipe-forward relational syntax (`\|>`) — optional surface | | -| [006][rfc-006] | Blocked | Promote unnest/explode to core Substrait lowering — blocked on upstream Substrait standardization | | -| [007][rfc-007] | In Progress | Prism logical planning and optimization engine | | -| [008][rfc-008] | Planned | Optimizer boundary, statistics, cost-based optimization, and adaptive execution | | -| [009][rfc-009] | Draft | Session format handler registry (plugin-style source format registration) | | -| [010][rfc-010] | Draft | CSV dialect and interpretation contract | | -| [011][rfc-011] | Draft | Source discovery and parse-unit expansion | | -| [012][rfc-012] | Implemented | Unified scalar expression surface | | -| [013][rfc-013] | Implemented | Function catalog program | | -| [014][rfc-014] | Implemented | Function registry and catalog governance | | -| [015][rfc-015] | Implemented | Core scalar functions and operators | | -| [016][rfc-016] | Implemented | Core aggregate functions | | -| [017][rfc-017] | Implemented | Aggregate modifiers | | -| [018][rfc-018] | Implemented | Common scalar function catalog | | -| [019][rfc-019] | Implemented | Window functions | | -| [020][rfc-020] | Implemented | Nested data functions | | -| [021][rfc-021] | Implemented | Generator and table-valued functions | | -| [022][rfc-022] | Implemented | Semi-structured and format functions | | -| [023][rfc-023] | Implemented | Approximate and sketch functions | | -| [024][rfc-024] | Implemented | Function extension policy | | -| [025][rfc-025] | Implemented | Typed sketch logical values | | -| [026][rfc-026] | Implemented | Semi-structured variant logical values | | -| [027][rfc-027] | In Progress | Relational evidence program | | -| [028][rfc-028] | In Progress | Semantic identity and target model | | -| [029][rfc-029] | In Progress | Typed metadata attachments | | -| [030][rfc-030] | In Progress | Prism lineage graph | | -| [031][rfc-031] | In Progress | Local inspection APIs and artifacts | | -| [032][rfc-032] | Implemented | Execution observations | | -| [033][rfc-033] | Implemented | Adapter requirements and coverage | | -| [034][rfc-034] | Implemented | Quality assertions and observations | | -| [035][rfc-035] | Draft | Governed attributes and policy checkpoints | | -| [036][rfc-036] | Draft | Governed plan bundle | | -| [037][rfc-037] | Draft | Plan diff and blast-radius inputs | | -| [038][rfc-038] | Draft | Evidence exchange bridges | | -| [039][rfc-039] | Draft | Pandas-familiar exploration API | | -| [040][rfc-040] | Draft | Interoperability semantic profiles | | -| [041][rfc-041] | Draft | Prism plan ingress and external client frontends | | -| [042][rfc-042] | Draft | Async verification evidence | | -| [043][rfc-043] | Draft | Canonical equality and digest profiles | | -| [044][rfc-044] | Draft | Verifier statements and proof artifacts | | -| [045][rfc-045] | Draft | Constraint evidence and verification-aware planning | | -| [046][rfc-046] | Draft | Data contract ingress and product topology | | -| [047][rfc-047] | Draft | Semantic evidence graph and agent query surface | | -| [048][rfc-048] | Draft | Cluster execution backend mode | | -| [050][rfc-050] | Draft | Addon component registry and package contract | | + + +| RFC | Status | Title | +| --- | --- | --- | +| [000](000_incql_syntax.md) | Planned | Language Specification | +| [001](001_incql_dataset.md) | In Progress | Dataset types and carriers (`DataSet[T]`) | +| [002](002_apache_substrait_integration.md) | In Progress | Apache Substrait integration | +| [003](003_incql_query_blocks.md) | Implemented | `query {}` blocks — syntax, typing, Substrait | +| [004](004_incql_execution_context.md) | In Progress | Execution context and DataFusion | +| [005](005_incql_pipe_forward.md) | Blocked | Pipe-forward relational syntax (`\|>`) | +| [006](006_unnest_core_substrait.md) | Blocked | Promote unnest/explode to core Substrait lowering | +| [007](007_prism_planning_engine.md) | In Progress | Prism logical planning and optimization engine | +| [008](008_optimizer_boundary_stats_cbo_aqe.md) | Planned | Optimizer boundary, statistics, cost-based optimization, and adaptive execution | +| [009](009_session_format_handler_registry.md) | Draft | Session Format Handler Registry | +| [010](010_csv_ingestion_contract.md) | Draft | CSV dialect and interpretation contract | +| [011](011_source_discovery_contract.md) | Draft | Source discovery and parse-unit expansion | +| [012](012_unified_scalar_expression_surface.md) | Implemented | Unified scalar expression surface | +| [013](013_function_catalog_program.md) | Implemented | Function catalog program | +| [014](014_function_registry.md) | Implemented | Function registry and catalog governance | +| [015](015_core_scalar_functions.md) | Implemented | Core scalar functions and operators | +| [016](016_core_aggregate_functions.md) | Implemented | Core aggregate functions | +| [017](017_aggregate_modifiers.md) | Implemented | Aggregate modifiers | +| [018](018_common_scalar_function_catalog.md) | Implemented | Common scalar function catalog | +| [019](019_window_functions.md) | Implemented | Window functions | +| [020](020_nested_data_functions.md) | Implemented | Nested data functions | +| [021](021_generator_table_functions.md) | Implemented | Generator and table-valued functions | +| [022](022_semi_structured_format_functions.md) | Implemented | Semi-structured and format functions | +| [023](023_approximate_sketch_functions.md) | Implemented | Approximate and sketch functions | +| [024](024_function_extension_policy.md) | Implemented | Function extension policy | +| [025](025_typed_sketch_logical_values.md) | Implemented | Typed sketch logical values | +| [026](026_semi_structured_variant_values.md) | Implemented | Semi-structured variant logical values | +| [027](027_relational_evidence_program.md) | In Progress | Relational evidence program | +| [028](028_semantic_identity_targets.md) | In Progress | Semantic identity and target model | +| [029](029_metadata_attachments.md) | In Progress | Typed metadata attachments | +| [030](030_prism_lineage_graph.md) | In Progress | Prism lineage graph | +| [031](031_inspection_artifacts.md) | In Progress | Local inspection APIs and artifacts | +| [032](032_execution_observations.md) | Implemented | Execution observations | +| [033](033_adapter_requirements_coverage.md) | Implemented | Adapter requirements and coverage | +| [034](034_quality_assertions_observations.md) | Implemented | Quality assertions and observations | +| [035](035_governed_attributes_policy_checkpoints.md) | Implemented | Governed attributes and policy checkpoints | +| [036](036_governed_plan_bundle.md) | Draft | Governed plan bundle | +| [037](037_plan_diff_blast_radius_inputs.md) | Draft | Plan diff and blast-radius inputs | +| [038](038_evidence_exchange_bridges.md) | Draft | Evidence exchange bridges | +| [039](039_pandas_familiar_exploration_api.md) | Draft | Pandas-familiar exploration API | +| [040](040_interoperability_semantic_profiles.md) | Draft | Interoperability semantic profiles | +| [041](041_prism_plan_ingress_frontends.md) | Draft | Prism plan ingress and external client frontends | +| [042](042_async_verification_evidence.md) | Draft | Async verification evidence | +| [043](043_canonical_equality_digest_profiles.md) | Draft | Canonical equality and digest profiles | +| [044](044_verifier_statements_proof_artifacts.md) | Draft | Verifier statements and proof artifacts | +| [045](045_constraint_evidence_verification_planning.md) | Draft | Constraint evidence and verification-aware planning | +| [046](046_data_contract_ingress.md) | Draft | Data contract ingress and product topology | +| [047](047_semantic_evidence_graph_agent_surface.md) | Draft | Semantic evidence graph and agent query surface | +| [048](048_cluster_execution_backend_mode.md) | Draft | Cluster execution backend mode | +| [050](050_addon_component_registry.md) | Draft | Addon component registry and package contract | + + - **v0.1 tracking:** RFCs 000–004 plus RFC 007 remain the foundation that defines when IncQL v0.1 is complete: authors can read data, write typed queries, lower through Prism to Substrait, execute through DataFusion, and write results. The table also marks additional v0.1-shipped slices that landed before the whole foundation is closed, including the function catalog and evidence/observation work. @@ -67,54 +70,4 @@ New RFCs should follow [TEMPLATE.md] (aligned with Incan’s RFC structure, adap [TEMPLATE.md]: TEMPLATE.md [Writing IncQL RFCs]: ../contributing/writing_rfcs.md -[rfc-000]: 000_incql_syntax.md -[rfc-001]: 001_incql_dataset.md -[rfc-002]: 002_apache_substrait_integration.md -[rfc-003]: 003_incql_query_blocks.md -[rfc-004]: 004_incql_execution_context.md -[rfc-005]: 005_incql_pipe_forward.md -[rfc-006]: 006_unnest_core_substrait.md -[rfc-007]: 007_prism_planning_engine.md -[rfc-008]: 008_optimizer_boundary_stats_cbo_aqe.md -[rfc-009]: 009_session_format_handler_registry.md -[rfc-010]: 010_csv_ingestion_contract.md -[rfc-011]: 011_source_discovery_contract.md -[rfc-012]: 012_unified_scalar_expression_surface.md -[rfc-013]: 013_function_catalog_program.md -[rfc-014]: 014_function_registry.md -[rfc-015]: 015_core_scalar_functions.md -[rfc-016]: 016_core_aggregate_functions.md -[rfc-017]: 017_aggregate_modifiers.md -[rfc-018]: 018_common_scalar_function_catalog.md -[rfc-019]: 019_window_functions.md -[rfc-020]: 020_nested_data_functions.md -[rfc-021]: 021_generator_table_functions.md -[rfc-022]: 022_semi_structured_format_functions.md -[rfc-023]: 023_approximate_sketch_functions.md -[rfc-024]: 024_function_extension_policy.md -[rfc-025]: 025_typed_sketch_logical_values.md -[rfc-026]: 026_semi_structured_variant_values.md -[rfc-027]: 027_relational_evidence_program.md -[rfc-028]: 028_semantic_identity_targets.md -[rfc-029]: 029_metadata_attachments.md -[rfc-030]: 030_prism_lineage_graph.md -[rfc-031]: 031_inspection_artifacts.md -[rfc-032]: 032_execution_observations.md -[rfc-033]: 033_adapter_requirements_coverage.md -[rfc-034]: 034_quality_assertions_observations.md -[rfc-035]: 035_governed_attributes_policy_checkpoints.md -[rfc-036]: 036_governed_plan_bundle.md -[rfc-037]: 037_plan_diff_blast_radius_inputs.md -[rfc-038]: 038_evidence_exchange_bridges.md -[rfc-039]: 039_pandas_familiar_exploration_api.md -[rfc-040]: 040_interoperability_semantic_profiles.md -[rfc-041]: 041_prism_plan_ingress_frontends.md -[rfc-042]: 042_async_verification_evidence.md -[rfc-043]: 043_canonical_equality_digest_profiles.md -[rfc-044]: 044_verifier_statements_proof_artifacts.md -[rfc-045]: 045_constraint_evidence_verification_planning.md -[rfc-046]: 046_data_contract_ingress.md -[rfc-047]: 047_semantic_evidence_graph_agent_surface.md -[rfc-048]: 048_cluster_execution_backend_mode.md -[rfc-050]: 050_addon_component_registry.md [incan-rfcs]: https://github.com/encero-systems/incan/tree/main/workspaces/docs-site/docs/RFCs diff --git a/scripts/generate_rfc_index.py b/scripts/generate_rfc_index.py new file mode 100644 index 00000000..31bd81e7 --- /dev/null +++ b/scripts/generate_rfc_index.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Generate the IncQL RFC index table from RFC markdown files.""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +RFC_DIR = ROOT / "docs" / "rfcs" +README = RFC_DIR / "README.md" +START = "" +END = "" + + +@dataclass(frozen=True) +class RfcEntry: + number: str + status: str + title: str + path: Path + + +def _escape_table_cell(value: str) -> str: + return value.replace("|", r"\|") + + +def _read_entry(path: Path) -> RfcEntry: + text = path.read_text(encoding="utf-8") + heading = re.search(r"^# IncQL RFC (?P\d{3}): (?P.+)$", text, re.MULTILINE) + if heading is None: + raise ValueError(f"{path}: missing '# IncQL RFC NNN: ...' heading") + + status = re.search(r"^- \*\*Status:\*\* (?P<status>.+)$", text, re.MULTILINE) + if status is None: + raise ValueError(f"{path}: missing '- **Status:** ...' header") + + number = heading.group("number") + filename_number = path.name.split("_", 1)[0] + if filename_number != number: + raise ValueError(f"{path}: filename number {filename_number} does not match heading {number}") + + return RfcEntry( + number=number, + status=status.group("status").strip(), + title=heading.group("title").strip(), + path=path.relative_to(RFC_DIR), + ) + + +def _entries() -> list[RfcEntry]: + entries = [_read_entry(path) for path in sorted(RFC_DIR.glob("[0-9][0-9][0-9]_*.md"))] + numbers = [entry.number for entry in entries] + duplicates = sorted({number for number in numbers if numbers.count(number) > 1}) + if duplicates: + joined = ", ".join(duplicates) + raise ValueError(f"duplicate RFC number(s): {joined}") + return entries + + +def _render_table(entries: list[RfcEntry]) -> str: + rows = [ + START, + "", + "| RFC | Status | Title |", + "| --- | --- | --- |", + ] + for entry in entries: + rows.append( + f"| [{entry.number}]({entry.path.as_posix()}) | " + f"{_escape_table_cell(entry.status)} | " + f"{_escape_table_cell(entry.title)} |" + ) + rows.extend(["", END]) + return "\n".join(rows) + + +def _replace_generated_region(readme: str, generated: str) -> str: + if START in readme and END in readme: + before, rest = readme.split(START, 1) + _, after = rest.split(END, 1) + return before.rstrip() + "\n\n" + generated + after + + table_match = re.search( + r"(?ms)^\| RFC\s+\|.*?^<!-- TODO: #7: auto populate this table.*?-->\n", + readme, + ) + if table_match is None: + raise ValueError(f"{README}: missing generated markers and bootstrap table") + return readme[: table_match.start()].rstrip() + "\n\n" + generated + "\n\n" + readme[table_match.end() :] + + +def render_readme() -> str: + current = README.read_text(encoding="utf-8") + generated = _render_table(_entries()) + return _replace_generated_region(current, generated) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="fail if docs/rfcs/README.md is stale") + args = parser.parse_args() + + try: + rendered = render_readme() + except ValueError as exc: + print(exc, file=sys.stderr) + return 2 + + current = README.read_text(encoding="utf-8") + if args.check: + if current != rendered: + print("docs/rfcs/README.md is stale; run `python3 scripts/generate_rfc_index.py`.", file=sys.stderr) + return 1 + return 0 + + README.write_text(rendered, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())