diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 6c6728848e2b..17479c236c1b 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -123,6 +123,14 @@ Requirements: - #17376: **(Ingestion / Hex)** Three recipe fields are removed and now emit a deprecation warning when set: `lineage_start_time`, `lineage_end_time`, and `datahub_page_size`. These belonged to the old lineage path that searched DataHub for Hex-tagged Query entities. Lineage now comes directly from the Hex REST API (the `queriedTables` API on Hex Enterprise workspaces, or SQL parsing of project/component cells on all workspaces), so these fields no longer have any effect. **Migration:** remove them from your recipe. +- #18133 **(Ingestion / BigQuery)** The `usage.start_time`, `usage.end_time`, and `usage.bucket_duration` recipe fields are deprecated in favor of the top-level `start_time`, `end_time`, and `bucket_duration` fields, which control the ingestion window for lineage, usage, and operations together. Previously, setting these fields only under `usage` had no effect at all — they were silently ignored, and only the default "last full day" window was ingested. **Behavior change:** if your recipe sets one of these fields only under `usage`, it will now actually take effect and be applied at the top level (with a deprecation warning logged); this may widen the range of data ingested. Since BigQuery usage ingestion scans audit logs, a wider window can increase query cost. **Action:** move `start_time`, `end_time`, and `bucket_duration` out of the `usage` block and set them at the top level of your recipe instead. Setting the same field in both places is now a configuration error. + +- #18133 **(Ingestion / BigQuery)** The `usage.max_query_duration` recipe field is deprecated in favor of the top-level `max_query_duration` field. Unlike the time-window fields above, `max_query_duration` only takes effect with the older, legacy extraction path (`use_queries_v2: False`) — it has no effect on the default queries-v2 path either way. **Action:** move `max_query_duration` out of the `usage` block and set it at the top level of your recipe instead. Setting the same field in both places is now a configuration error. + +- #18133 **(Ingestion / BigQuery)** `usage.format_sql_queries`, `usage.include_top_n_queries`, and `usage.queries_character_limit` now take effect when using the default queries-v2 extraction path (`use_queries_v2: True`); previously they were silently ignored on that path and only worked with the older, legacy extraction path. **Behavior change:** if your recipe already sets `usage.format_sql_queries: true`, DataHub will now actually reformat SQL queries before displaying them (this was silently skipped before). If you don't set these fields, nothing changes. + +- #18133 **(Ingestion / BigQuery)** `usage.include_read_operational_stats` and `usage.apply_view_usage_to_tables` only work with the older, legacy extraction path (`use_queries_v2: False`) — the default queries-v2 path has no equivalent setting. Setting either to a non-default value while using queries-v2 now logs a warning explaining that it will be ignored. + ### Other Notable Changes - #18001 **(Ingestion / Redshift)** `table_pattern` is now applied to SQL-parsing-derived lineage and usage, not just to the catalog walk. Previously, lineage/usage/query entities produced by the SQL parsing aggregator were emitted for all tables regardless of `table_pattern`; they are now filtered consistently with the other SQL connectors (Snowflake, BigQuery, Databricks). The default `table_pattern` allows everything, so most setups are unaffected. **If you set `table_pattern`** and relied on lineage/usage edges to excluded tables, widen the pattern to keep them. diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index bcc8d266494a..56cb8e199a98 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -265,6 +265,45 @@ def _warn_deprecated_configs(self): title="Config option deprecation warning", ) + # Unlike the forwarded usage.start_time/end_time/bucket_duration/max_query_duration + # fields, these two are never popped from self.config.usage, so we can still see + # here whether the user set them - surface it in the structured report (visible + # in the UI), not just the config-validation-time logger.warning. + if self.config.use_queries_v2: + if self.config.usage.include_read_operational_stats: + self.report.report_warning( + message="`usage.include_read_operational_stats` is only supported with the legacy extraction path " + "(`use_queries_v2: False`) and is ignored under queries-v2.", + context="Config option deprecation warning", + title="Config option deprecation warning", + ) + if self.config.usage.apply_view_usage_to_tables: + self.report.report_warning( + message="`usage.apply_view_usage_to_tables` is only supported with the legacy extraction path " + "(`use_queries_v2: False`) and is ignored under queries-v2.", + context="Config option deprecation warning", + title="Config option deprecation warning", + ) + + def _build_queries_extractor_config(self) -> BigQueryQueriesExtractorConfig: + return BigQueryQueriesExtractorConfig( + window=self.config, + user_email_pattern=self.config.usage.user_email_pattern, + pushdown_deny_usernames=self.config.pushdown_deny_usernames, + pushdown_allow_usernames=self.config.pushdown_allow_usernames, + include_lineage=self.config.include_table_lineage, + include_usage_statistics=self.config.include_usage_statistics, + include_operations=self.config.usage.include_operational_stats, + include_queries=self.config.include_queries, + include_query_usage_statistics=self.config.include_query_usage_statistics, + top_n_queries=self.config.usage.top_n_queries, + format_sql_queries=self.config.usage.format_sql_queries, + include_top_n_queries=self.config.usage.include_top_n_queries, + queries_character_limit=self.config.usage.queries_character_limit, + region_qualifiers=self.config.region_qualifiers, + region_qualifiers_auto_discovery=self.config.region_qualifiers_auto_discovery, + ) + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: self._warn_deprecated_configs() projects = get_projects( @@ -311,20 +350,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: BigQueryQueriesExtractor( connection=self.config.get_bigquery_client(), schema_api=self.bq_schema_extractor.schema_api, - config=BigQueryQueriesExtractorConfig( - window=self.config, - user_email_pattern=self.config.usage.user_email_pattern, - pushdown_deny_usernames=self.config.pushdown_deny_usernames, - pushdown_allow_usernames=self.config.pushdown_allow_usernames, - include_lineage=self.config.include_table_lineage, - include_usage_statistics=self.config.include_usage_statistics, - include_operations=self.config.usage.include_operational_stats, - include_queries=self.config.include_queries, - include_query_usage_statistics=self.config.include_query_usage_statistics, - top_n_queries=self.config.usage.top_n_queries, - region_qualifiers=self.config.region_qualifiers, - region_qualifiers_auto_discovery=self.config.region_qualifiers_auto_discovery, - ), + config=self._build_queries_extractor_config(), structured_report=self.report, filters=self.filters, identifiers=self.identifiers, diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index 0a78d477320a..063d1bc69721 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -1,7 +1,7 @@ import logging import re from copy import deepcopy -from datetime import timedelta +from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, Tuple, Union from pydantic import ( @@ -20,6 +20,10 @@ LowerCaseDatasetUrnConfigMixin, PlatformInstanceConfigMixin, ) +from datahub.configuration.time_window_config import ( + BaseTimeWindowConfig, + BucketDuration, +) from datahub.configuration.validate_field_removal import pydantic_removed_field from datahub.ingestion.glossary.classification_mixin import ( ClassificationSourceConfigMixin, @@ -44,6 +48,16 @@ # Tuple (not list) so in-place mutation cannot silently drift the value used by callers. DEFAULT_REGION_QUALIFIERS: Tuple[str, ...] = ("region-us", "region-eu") +# Fields inherited from BaseTimeWindowConfig/BaseUsageConfig that are duplicated +# on the nested `usage` config but only ever read from the top level of +# BigQueryV2Config. See forward_deprecated_usage_fields below. +_DEPRECATED_USAGE_TOP_LEVEL_FIELDS: Tuple[str, ...] = ( + "start_time", + "end_time", + "bucket_duration", + "max_query_duration", +) + # Regexp for sharded tables. # A sharded table is a table that has a suffix of the form _yyyymmdd or yyyymmdd, where yyyymmdd is a date. # The regexp checks for valid dates in the suffix (e.g. 20200101, 20200229, 20201231) and if the date is not valid @@ -106,18 +120,51 @@ class BigQueryUsageConfig(BaseUsageConfig): "query_log_delay", month="April", year=2023 ) + # start_time/end_time/bucket_duration are inherited from BaseTimeWindowConfig but + # redeclared here (rather than editing the shared base class, which other connectors + # also use) solely to surface the BigQuery-specific deprecation in generated docs. + # Descriptions are composed from the base class's text plus a deprecation suffix, + # rather than duplicated verbatim, so they can't silently drift out of sync. + # See forward_deprecated_usage_fields on BigQueryV2Config for the runtime behavior. + start_time: datetime = Field( + default=None, # type: ignore + description=f"{BaseTimeWindowConfig.model_fields['start_time'].description} " + "**Deprecated**: set the top-level `start_time` instead - it governs lineage, " + "usage, and operations together.", + ) + end_time: datetime = Field( + default_factory=lambda: datetime.now(tz=timezone.utc), + description=f"{BaseTimeWindowConfig.model_fields['end_time'].description} " + "**Deprecated**: set the top-level `end_time` instead - it governs lineage, " + "usage, and operations together.", + ) + bucket_duration: BucketDuration = Field( + default=BucketDuration.DAY, + description=f"{BaseTimeWindowConfig.model_fields['bucket_duration'].description} " + "**Deprecated**: set the top-level `bucket_duration` instead - it governs " + "lineage, usage, and operations together.", + ) + max_query_duration: timedelta = Field( default=timedelta(minutes=15), description="Correction to pad start_time and end_time with. For handling the case where the read happens " "within our time range but the query completion event is delayed and happens after the configured" - " end time.", + " end time. **Deprecated**: set the top-level `max_query_duration` instead. Note it only takes " + "effect with the legacy extraction path (`use_queries_v2: False`).", ) apply_view_usage_to_tables: bool = Field( default=False, description="Whether to apply view's usage to its base tables. If set to False, uses sql parser and applies " "usage to views / tables mentioned in the query. If set to True, usage is applied to base tables " - "only.", + "only. Only applied with the legacy extraction path (`use_queries_v2: False`); ignored under " + "queries-v2.", + ) + + include_read_operational_stats: bool = Field( + default=False, + description="Whether to report read operational stats. Experimental. Only applied with the " + "legacy extraction path (`use_queries_v2: False`); ignored under queries-v2.", ) @@ -531,6 +578,46 @@ def have_table_data_read_permission(self) -> bool: "lineage_parse_view_ddl", month="January", year=2025 ) + @model_validator(mode="before") + @classmethod + def forward_deprecated_usage_fields(cls, values: Any) -> Any: + # `usage.start_time`/`end_time`/`bucket_duration`/`max_query_duration` are + # inherited from BaseTimeWindowConfig/BaseUsageConfig via BigQueryUsageConfig + # but were never read by either the queries-v2 or legacy code paths, which + # both use the top-level copies. These are connector-wide settings governing + # lineage, usage, and operations together, so they belong at the top level only. + if not isinstance(values, dict) or not isinstance(values.get("usage"), dict): + # usage.pop() below requires a dict; skip if usage is already a + # BigQueryUsageConfig object. Accepted since recipes always come from YAML. + return values + # Copy first: usage.pop() below must not mutate the caller's dict. + values = deepcopy(values) + usage = values["usage"] + for field in _DEPRECATED_USAGE_TOP_LEVEL_FIELDS: + if field not in usage: + continue + if field in values and values[field] is not None: + raise ValueError( + f"`{field}` is set both at the top level and under `usage`. " + f"The top-level `{field}` is the only valid setting - remove `usage.{field}` from your recipe." + ) + if field == "max_query_duration": + # Unlike start_time/end_time/bucket_duration, the top-level max_query_duration + # is only read on the legacy (non-queries-v2) extraction path - it has no effect + # under the default use_queries_v2=True, so don't claim otherwise. + logger.warning( + "`usage.max_query_duration` is deprecated and will be ignored in a future release. " + "Please set `max_query_duration` at the top level instead - note it only takes " + "effect with the legacy extraction path (`use_queries_v2: False`)." + ) + else: + logger.warning( + f"`usage.{field}` is deprecated and will be ignored in a future release. " + f"Please set `{field}` at the top level instead - it applies to lineage, usage, and operations together." + ) + values[field] = usage.pop(field) + return values + @model_validator(mode="before") @classmethod def set_include_schema_metadata(cls, values: Dict) -> Dict: @@ -601,6 +688,26 @@ def validate_queries_v2_stateful_ingestion(self) -> "BigQueryV2Config": ) return self + @model_validator(mode="after") + def warn_legacy_only_usage_fields_under_queries_v2(self) -> "BigQueryV2Config": + # `include_read_operational_stats` and `apply_view_usage_to_tables` are only + # implemented in the legacy (non-queries-v2) usage extraction path; qv2 attributes + # usage purely via SQL parsing and has no equivalent mechanism. We can't tell + # whether the user "explicitly" set a field post-validation, so we pragmatically + # warn whenever it differs from its (both False) default. + if self.use_queries_v2: + if self.usage.include_read_operational_stats: + logger.warning( + "`usage.include_read_operational_stats` is only supported with the legacy " + "extraction path (`use_queries_v2: False`) and is ignored under queries-v2." + ) + if self.usage.apply_view_usage_to_tables: + logger.warning( + "`usage.apply_view_usage_to_tables` is only supported with the legacy " + "extraction path (`use_queries_v2: False`) and is ignored under queries-v2." + ) + return self + @field_validator( "pushdown_deny_usernames", "pushdown_allow_usernames", mode="after" ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py index 71918816c7d0..f08b529a1100 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/queries_extractor.py @@ -7,7 +7,7 @@ from typing import Collection, Dict, Iterable, List, Optional, Set, TypedDict from google.cloud.bigquery import Client -from pydantic import Field, PositiveInt +from pydantic import Field, PositiveInt, model_validator from datahub.configuration.common import AllowDenyPattern, HiddenFromDocs from datahub.configuration.time_window_config import ( @@ -43,7 +43,11 @@ from datahub.ingestion.source.state.redundant_run_skip_handler import ( RedundantQueriesRunSkipHandler, ) -from datahub.ingestion.source.usage.usage_common import BaseUsageConfig +from datahub.ingestion.source.usage.usage_common import ( + DEFAULT_QUERIES_CHARACTER_LIMIT, + BaseUsageConfig, + validate_top_n_queries_character_budget, +) from datahub.metadata.urns import CorpUserUrn from datahub.sql_parsing.schema_resolver import SchemaResolver from datahub.sql_parsing.sql_parsing_aggregator import ( @@ -122,6 +126,21 @@ class BigQueryQueriesExtractorConfig(BigQueryBaseConfig): default=10, description="Number of top queries to save to each table." ) + format_sql_queries: bool = Field( + default=False, description="Whether to format the SQL queries." + ) + + include_top_n_queries: bool = Field( + default=True, description="Whether to ingest the top_n_queries." + ) + + queries_character_limit: HiddenFromDocs[int] = Field( + default=DEFAULT_QUERIES_CHARACTER_LIMIT, + description="Total character limit for all queries in a single database call. This is a " + "low-level config property which should be touched with care. " + "Queries will be truncated to length `queries_character_limit / top_n_queries`.", + ) + include_lineage: bool = True include_queries: bool = True include_usage_statistics: bool = True @@ -142,6 +161,19 @@ class BigQueryQueriesExtractorConfig(BigQueryBaseConfig): "Set to True if your project has datasets in regions beyond `region-us` and `region-eu`.", ) + @model_validator(mode="after") + def check_top_n_queries_character_budget(self) -> "BigQueryQueriesExtractorConfig": + # The main BigQuery source builds this config from an already-validated + # BigQueryUsageConfig (which shares this check via BaseUsageConfig), but the + # standalone bigquery-queries source constructs this class directly, so it + # needs its own copy of the check to fail at recipe parse time rather than + # mid-ingestion inside BaseUsageConfig(...). + validate_top_n_queries_character_budget( + top_n_queries=self.top_n_queries, + queries_character_limit=self.queries_character_limit, + ) + return self + class BigQueryQueriesExtractor(Closeable): """ @@ -216,11 +248,14 @@ def __init__( end_time=self.end_time, user_email_pattern=self.config.user_email_pattern, top_n_queries=self.config.top_n_queries, + format_sql_queries=self.config.format_sql_queries, + include_top_n_queries=self.config.include_top_n_queries, + queries_character_limit=self.config.queries_character_limit, ), generate_operations=self.config.include_operations, is_temp_table=self.is_temp_table, is_allowed_table=self.is_allowed_table, - format_queries=False, + format_queries=self.config.format_sql_queries, ) self.report.sql_aggregator = self.aggregator.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py b/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py index 4cd583b21ac2..bd2e931a4b5d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/usage/usage_common.py @@ -49,6 +49,19 @@ DEFAULT_QUERIES_CHARACTER_LIMIT = 24000 +def validate_top_n_queries_character_budget( + top_n_queries: int, queries_character_limit: int +) -> None: + """Shared by BaseUsageConfig and BigQueryQueriesExtractorConfig, which both + accept these two fields but don't share a common base class.""" + minimum_query_size = 20 + max_queries = int(queries_character_limit / minimum_query_size) + if top_n_queries > max_queries: + raise ValueError( + f"top_n_queries is set to {top_n_queries} but it can be maximum {max_queries}" + ) + + def default_user_urn_builder(email: str) -> str: return builder.make_user_urn(email.split("@")[0]) @@ -258,13 +271,10 @@ class BaseUsageConfig(BaseTimeWindowConfig): @field_validator("top_n_queries", mode="after") @classmethod def ensure_top_n_queries_is_not_too_big(cls, v: int, info: ValidationInfo) -> int: - minimum_query_size = 20 - values = info.data - max_queries = int(values["queries_character_limit"] / minimum_query_size) - if v > max_queries: - raise ValueError( - f"top_n_queries is set to {v} but it can be maximum {max_queries}" - ) + validate_top_n_queries_character_budget( + top_n_queries=v, + queries_character_limit=info.data["queries_character_limit"], + ) return v diff --git a/metadata-ingestion/tests/unit/bigquery/test_bigquery_queries_extractor.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_queries_extractor.py index b6c04890e970..c7064e7a5113 100644 --- a/metadata-ingestion/tests/unit/bigquery/test_bigquery_queries_extractor.py +++ b/metadata-ingestion/tests/unit/bigquery/test_bigquery_queries_extractor.py @@ -23,12 +23,25 @@ import re from datetime import datetime, timezone -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +import pytest +from pydantic import ValidationError + +from datahub.ingestion.source.bigquery_v2.bigquery_config import ( + BigQueryFilterConfig, + BigQueryIdentifierConfig, +) from datahub.ingestion.source.bigquery_v2.bigquery_report import ( BigQueryQueriesExtractorReport, ) +from datahub.ingestion.source.bigquery_v2.common import ( + BigQueryFilter, + BigQueryIdentifierBuilder, +) from datahub.ingestion.source.bigquery_v2.queries_extractor import ( + BigQueryQueriesExtractor, + BigQueryQueriesExtractorConfig, _all_scanned_regions_empty, _build_enriched_query_log_query, _build_user_filter, @@ -1239,3 +1252,55 @@ def test_region_qualifiers_auto_discovery_defaults_to_false(self): assert ( BigQueryQueriesExtractorConfig().region_qualifiers_auto_discovery is False ) + + +class TestQueriesExtractorUsageConfigWiring: + """Tests for usage.format_sql_queries / include_top_n_queries / queries_character_limit + being forwarded from BigQueryQueriesExtractorConfig into the SqlParsingAggregator.""" + + def _build_extractor(self, config): + filters = BigQueryFilter(BigQueryFilterConfig(), MagicMock()) + identifiers = BigQueryIdentifierBuilder(BigQueryIdentifierConfig(), MagicMock()) + return BigQueryQueriesExtractor( + connection=MagicMock(), + schema_api=MagicMock(), + config=config, + structured_report=MagicMock(), + filters=filters, + identifiers=identifiers, + ) + + def test_top_n_queries_too_big_for_character_limit_rejected_at_parse_time(self): + # The standalone bigquery-queries source uses BigQueryQueriesExtractorConfig + # directly (not via BigQueryUsageConfig, which already validates this combo), + # so this class needs its own copy of the check - otherwise an inconsistent + # combo only blows up later inside BaseUsageConfig(...) mid-ingestion. + with pytest.raises(ValidationError) as excinfo: + BigQueryQueriesExtractorConfig(top_n_queries=2, queries_character_limit=20) + assert "top_n_queries is set to 2 but it can be maximum 1" in str(excinfo.value) + + @pytest.mark.parametrize("value", [True, False]) + def test_format_sql_queries_forwarded_to_aggregator(self, value): + # format_sql_queries is dual-target: it drives both the aggregator's own + # format_queries kwarg and usage_config.format_sql_queries. + with patch( + "datahub.ingestion.source.bigquery_v2.queries_extractor.SqlParsingAggregator" + ) as mock_aggregator_cls: + self._build_extractor( + BigQueryQueriesExtractorConfig(format_sql_queries=value) + ) + _, kwargs = mock_aggregator_cls.call_args + assert kwargs["format_queries"] is value + assert kwargs["usage_config"].format_sql_queries is value + + @pytest.mark.parametrize( + "field,value", + [("include_top_n_queries", False), ("queries_character_limit", 1000)], + ) + def test_usage_config_field_forwarded_to_aggregator(self, field, value): + with patch( + "datahub.ingestion.source.bigquery_v2.queries_extractor.SqlParsingAggregator" + ) as mock_aggregator_cls: + self._build_extractor(BigQueryQueriesExtractorConfig(**{field: value})) + _, kwargs = mock_aggregator_cls.call_args + assert getattr(kwargs["usage_config"], field) == value diff --git a/metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py b/metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py index fe1a255a7366..87ec527eb027 100644 --- a/metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py +++ b/metadata-ingestion/tests/unit/bigquery/test_bigquery_source.py @@ -8,8 +8,10 @@ import time_machine from google.api_core.exceptions import GoogleAPICallError from google.cloud.bigquery.table import Row, TableListItem +from pydantic import ValidationError from datahub.configuration.common import AllowDenyPattern +from datahub.configuration.time_window_config import BucketDuration from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.source.bigquery_v2.bigquery import BigqueryV2Source @@ -20,6 +22,7 @@ BigQueryTableRef, ) from datahub.ingestion.source.bigquery_v2.bigquery_config import ( + BigQueryUsageConfig, BigQueryV2Config, ) from datahub.ingestion.source.bigquery_v2.bigquery_connection import ( @@ -1401,6 +1404,177 @@ def test_bigquery_config_deprecated_schema_pattern(): ) # dataset_pattern +@pytest.mark.parametrize( + "field,usage_value,expected", + [ + ( + "start_time", + "2023-01-01T00:00:00Z", + datetime(2023, 1, 1, tzinfo=timezone.utc), + ), + ("end_time", "2023-01-01T00:00:00Z", datetime(2023, 1, 1, tzinfo=timezone.utc)), + ("bucket_duration", "HOUR", BucketDuration.HOUR), + ("max_query_duration", "PT30M", timedelta(minutes=30)), + ], +) +def test_bigquery_config_usage_field_forwarded_to_top_level( + field: str, usage_value: str, expected: object +) -> None: + config = BigQueryV2Config.model_validate({"usage": {field: usage_value}}) + assert getattr(config, field) == expected + + +@pytest.mark.parametrize( + "field,top_level_value,usage_value", + [ + ("start_time", "2023-01-01T00:00:00Z", "2023-02-01T00:00:00Z"), + ("end_time", "2023-01-01T00:00:00Z", "2023-02-01T00:00:00Z"), + ("bucket_duration", "DAY", "HOUR"), + ("max_query_duration", "PT30M", "PT45M"), + ], +) +def test_bigquery_config_usage_and_top_level_field_conflict_raises( + field: str, top_level_value: str, usage_value: str +) -> None: + with pytest.raises(ValidationError): + BigQueryV2Config.model_validate( + {field: top_level_value, "usage": {field: usage_value}} + ) + + +def test_bigquery_config_usage_time_window_field_emits_deprecation_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.clear() + with caplog.at_level(logging.WARNING): + BigQueryV2Config.model_validate({"usage": {"end_time": "2023-01-01T00:00:00Z"}}) + assert any("deprecated" in record.msg for record in caplog.records) + + +def test_bigquery_config_usage_max_query_duration_warns_legacy_only( + caplog: pytest.LogCaptureFixture, +) -> None: + # max_query_duration, unlike start_time/end_time/bucket_duration, is only ever + # read on the legacy (non-queries-v2) extraction path, so the warning must not + # claim it affects lineage/usage/operations generally under queries-v2. + caplog.clear() + with caplog.at_level(logging.WARNING): + BigQueryV2Config.model_validate({"usage": {"max_query_duration": "PT30M"}}) + assert any( + "legacy" in record.msg or "use_queries_v2" in record.msg + for record in caplog.records + ) + + +def test_bigquery_config_usage_forwarded_field_cleared_from_nested_usage(): + config = BigQueryV2Config.model_validate({"usage": {"max_query_duration": "PT30M"}}) + assert config.usage.max_query_duration == timedelta(minutes=15) # default + + +@pytest.mark.parametrize( + "field", ["start_time", "end_time", "bucket_duration", "max_query_duration"] +) +def test_bigquery_usage_config_field_description_mentions_deprecation( + field: str, +) -> None: + # The generated connector docs page renders these `description=` strings, so the + # deprecation must be visible there too, not just in the runtime warning. + description = BigQueryUsageConfig.model_fields[field].description + assert description is not None + assert "deprecated" in description.lower() + + +def test_bigquery_source_builds_queries_extractor_config_from_usage_fields(): + # Guards the bigquery.py seam that maps self.config.usage.* into + # BigQueryQueriesExtractorConfig(...) - a typo'd kwarg here would otherwise only + # surface if some other test happened to exercise the full queries-v2 codepath. + config = BigQueryV2Config.model_validate( + { + "usage": { + "format_sql_queries": True, + "include_top_n_queries": False, + "queries_character_limit": 1000, + "top_n_queries": 5, + } + } + ) + fake_source = BigqueryV2Source.__new__(BigqueryV2Source) + fake_source.config = config + queries_config = fake_source._build_queries_extractor_config() + + assert queries_config.format_sql_queries is True + assert queries_config.include_top_n_queries is False + assert queries_config.queries_character_limit == 1000 + assert queries_config.top_n_queries == 5 + + +def test_bigquery_source_reports_legacy_only_usage_fields_under_queries_v2(): + # The logger.warning fired at config-validation time is easy to miss; this also + # surfaces in the structured ingestion report, which is what shows up in the UI. + config = BigQueryV2Config.model_validate( + {"use_queries_v2": True, "usage": {"apply_view_usage_to_tables": True}} + ) + fake_source = BigqueryV2Source.__new__(BigqueryV2Source) + fake_source.config = config + fake_source.report = BigQueryV2Report() + fake_source._warn_deprecated_configs() + + assert any( + "apply_view_usage_to_tables" in w.message for w in fake_source.report.warnings + ) + + +def test_bigquery_source_no_legacy_only_usage_field_report_warning_under_legacy_path(): + config = BigQueryV2Config.model_validate( + {"use_queries_v2": False, "usage": {"apply_view_usage_to_tables": True}} + ) + fake_source = BigqueryV2Source.__new__(BigqueryV2Source) + fake_source.config = config + fake_source.report = BigQueryV2Report() + fake_source._warn_deprecated_configs() + + assert not any( + "apply_view_usage_to_tables" in w.message for w in fake_source.report.warnings + ) + + +@pytest.mark.parametrize( + "field", ["apply_view_usage_to_tables", "include_read_operational_stats"] +) +def test_bigquery_config_legacy_only_usage_field_warns_under_queries_v2( + field: str, caplog: pytest.LogCaptureFixture +) -> None: + caplog.clear() + with caplog.at_level(logging.WARNING): + BigQueryV2Config.model_validate( + {"use_queries_v2": True, "usage": {field: True}} + ) + assert any( + "use_queries_v2" in record.msg or "legacy" in record.msg + for record in caplog.records + ) + + +def test_bigquery_config_legacy_only_usage_fields_no_warning_under_legacy_path( + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.clear() + with caplog.at_level(logging.WARNING): + BigQueryV2Config.model_validate( + { + "use_queries_v2": False, + "usage": { + "apply_view_usage_to_tables": True, + "include_read_operational_stats": True, + }, + } + ) + assert not any( + "use_queries_v2" in record.msg or "legacy" in record.msg + for record in caplog.records + ) + + @patch.object(BigQueryV2Config, "get_bigquery_client") @patch.object(BigQueryV2Config, "get_projects_client") def test_get_projects_with_project_labels(