From 8bd37eba770c974bf08fd62c2e568de7796a6a6e Mon Sep 17 00:00:00 2001 From: Lakshay Nasa <76848292+lakshay-nasa@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:14:05 +0530 Subject: [PATCH] feat(docs-archive): add 1.5.0 and switch to prebuilt Vercel deploy - Add DataHub 1.5.0 to the archive: versioned_docs, autogenerated sidebar, versions.json, and docusaurus.config.js (set as lastVersion) - Fix 1.5.0 source issues: -> MDX typos (3 files) and a stray broken local image link in how-to-set-up-graphql.md - De-hardcode navbar/footer "Latest" links and point footer quicklinks at 1.5.0 - Add scripts/deploy: local build + Vercel Build Output API packaging for `vercel deploy --prebuilt` - Remove docs-archive-deploy.yml: the multi-version build exceeds hosted runner memory; deploys move to local prebuilt - gitignore /.vercel --- .github/workflows/docs-archive-deploy.yml | 57 - docs-archive/.gitignore | 3 + docs-archive/docusaurus.config.js | 13 +- docs-archive/scripts/deploy/README.md | 43 + docs-archive/scripts/deploy/build-archive.sh | 23 + .../scripts/deploy/package-prebuilt.sh | 39 + .../versioned_docs/version-1.5.0/AGENTS.md | 603 + .../versioned_docs/version-1.5.0/CLAUDE.md | 8 + .../versioned_docs/version-1.5.0/README.md | 891 + .../versioned_docs/version-1.5.0/SECURITY.md | 23 + .../version-1.5.0/datahub-actions/README.md | 248 + .../plugin/action/propagation/docs/README.md | 90 + .../plugin/action/tag/README.md | 43 + .../plugin/action/term/README.md | 62 + .../datahub-agent-context/README.md | 176 + .../datahub_agent_context/snowflake/README.md | 182 + .../version-1.5.0/datahub-frontend/README.md | 97 + .../datahub-graphql-core/README.md | 93 + .../datahub-web-react/BASE_PATH.md | 160 + .../version-1.5.0/datahub-web-react/CLAUDE.md | 216 + .../version-1.5.0/datahub-web-react/README.md | 187 + .../src/app/analytics/README.md | 161 + .../document/changeHistory/ARCHITECTURE.md | 296 + .../changeHistory/changeMessages/README.md | 178 + .../src/app/mfeframework/README-MFE.md | 122 + .../version-1.5.0/docker/README.md | 302 + .../docker/airflow/local_airflow.md | 208 + .../docker/datahub-upgrade/README.md | 168 + .../version-1.5.0/docs/CODE_OF_CONDUCT.md | 81 + .../version-1.5.0/docs/CONTRIBUTING.md | 67 + .../version-1.5.0/docs/PYSPARK.md | 325 + .../version-1.5.0/docs/SECURITY_STANCE.md | 86 + .../version-1.5.0/docs/_api-guide-template.md | 76 + .../docs/_feature-guide-template.md | 87 + .../version-1.5.0/docs/act-on-metadata.md | 20 + .../docs/act-on-metadata/impact-analysis.md | 106 + .../version-1.5.0/docs/actions/README.md | 249 + .../docs/actions/actions/executor.md | 86 + .../docs/actions/actions/hello_world.md | 62 + .../docs/actions/actions/slack.md | 280 + .../version-1.5.0/docs/actions/concepts.md | 105 + .../events/audit-events-search-guide.md | 284 + .../actions/events/entity-change-event.md | 732 + .../events/metadata-change-log-event.md | 154 + .../guides/developing-a-transformer.md | 135 + .../actions/guides/developing-an-action.md | 134 + .../version-1.5.0/docs/actions/quickstart.md | 175 + .../sources/datahub-cloud-event-source.md | 125 + .../actions/sources/kafka-event-source.md | 134 + .../docs/advanced/api-tracing.md | 439 + .../docs/advanced/aspect-versioning.md | 63 + .../docs/advanced/backfilling.md | 9 + .../docs/advanced/bootstrap-mcps.md | 189 + .../docs/advanced/browse-paths-upgrade.md | 143 + .../docs/advanced/db-retention.md | 87 + .../docs/advanced/derived-aspects.md | 9 + .../docs/advanced/entity-hierarchy.md | 9 + .../docs/advanced/field-path-spec-v2.md | 372 + .../docs/advanced/high-cardinality.md | 52 + .../jar-extraction-tmpfs-optimization.md | 345 + .../version-1.5.0/docs/advanced/mcp-mcl.md | 298 + .../advanced/micrometer-best-practices.md | 271 + .../version-1.5.0/docs/advanced/monitoring.md | 1144 + .../docs/advanced/partial-update.md | 9 + .../version-1.5.0/docs/advanced/patch.md | 695 + .../docs/advanced/pdl-best-practices.md | 9 + .../docs/advanced/writing-mcps.md | 165 + .../version-1.5.0/docs/api/datahub-apis.md | 110 + .../docs/api/graphql/getting-started.md | 171 + .../api/graphql/graphql-best-practices.md | 1082 + .../graphql/graphql-endpoint-development.md | 65 + .../docs/api/graphql/how-to-set-up-graphql.md | 118 + .../docs/api/graphql/overview.md | 45 + .../docs/api/graphql/token-management.md | 139 + .../docs/api/openapi/openapi-usage-guide.md | 922 + .../docs/api/restli/evaluate-tests.md | 27 + .../api/restli/get-elastic-task-status.md | 58 + .../docs/api/restli/get-index-sizes.md | 25 + .../docs/api/restli/restli-overview.md | 1442 + .../docs/api/restli/restore-indices.md | 35 + .../api/restli/truncate-time-series-aspect.md | 54 + .../docs/api/tutorials/applications.md | 356 + .../docs/api/tutorials/assertions.md | 1820 + .../docs/api/tutorials/container.md | 57 + .../docs/api/tutorials/custom-assertions.md | 360 + .../docs/api/tutorials/custom-properties.md | 452 + .../docs/api/tutorials/dashboard-chart.md | 216 + .../docs/api/tutorials/data-contracts.md | 224 + .../docs/api/tutorials/dataflow-datajob.md | 179 + .../docs/api/tutorials/datasets.md | 261 + .../docs/api/tutorials/deprecation.md | 219 + .../docs/api/tutorials/descriptions.md | 434 + .../docs/api/tutorials/documents.md | 1022 + .../docs/api/tutorials/domains.md | 413 + .../version-1.5.0/docs/api/tutorials/forms.md | 281 + .../docs/api/tutorials/incidents.md | 169 + .../docs/api/tutorials/lineage.md | 619 + .../version-1.5.0/docs/api/tutorials/ml.md | 996 + .../docs/api/tutorials/ml_feature_store.md | 793 + .../api/tutorials/mlmodel-mlmodelgroup.md | 200 + .../docs/api/tutorials/operations.md | 182 + .../docs/api/tutorials/owners.md | 470 + .../api/tutorials/sdk/bulk-assertions-sdk.md | 688 + .../docs/api/tutorials/sdk/search_client.md | 315 + .../api/tutorials/structured-properties.md | 2080 ++ .../docs/api/tutorials/subscriptions.md | 195 + .../version-1.5.0/docs/api/tutorials/tags.md | 509 + .../version-1.5.0/docs/api/tutorials/terms.md | 509 + .../docs/architecture/architecture.md | 48 + .../docs/architecture/docker-containers.md | 26 + .../docs/architecture/metadata-ingestion.md | 42 + .../docs/architecture/metadata-serving.md | 69 + .../docs/assertions/open-assertions-spec.md | 488 + .../assertions/snowflake/snowflake_dmfs.md | 347 + .../docs/authentication/README.md | 77 + .../changing-default-credentials.md | 161 + .../docs/authentication/concepts.md | 229 + .../external-oauth-providers.md | 267 + .../docs/authentication/guides/add-users.md | 238 + .../docs/authentication/guides/jaas.md | 76 + .../guides/sso/configure-oidc-behind-proxy.md | 72 + .../guides/sso/configure-oidc-react.md | 214 + .../guides/sso/initialize-oidc.md | 229 + ...oducing-metadata-service-authentication.md | 200 + .../authentication/personal-access-tokens.md | 114 + .../docs/authorization/README.md | 26 + .../authorization/access-policies-guide.md | 278 + .../docs/authorization/groups.md | 38 + .../docs/authorization/policies.md | 448 + .../version-1.5.0/docs/authorization/roles.md | 208 + .../version-1.5.0/docs/automations/ai-docs.md | 63 + .../docs/automations/ai-term-suggestion.md | 90 + .../automations/bigquery-metadata-sync.md | 198 + .../automations/databricks-metadata-sync.md | 286 + .../docs/automations/docs-propagation.md | 119 + .../automations/glossary-term-propagation.md | 71 + .../automations/snowflake-tag-propagation.md | 117 + .../docs/browseV2/browse-paths-v2.md | 57 + .../version-1.5.0/docs/businessattributes.md | 108 + .../docs/cli-commands/dataset.md | 317 + .../docs/cli-commands/graphql.md | 509 + .../version-1.5.0/docs/cli-commands/search.md | 786 + .../versioned_docs/version-1.5.0/docs/cli.md | 1203 + .../version-1.5.0/docs/components.md | 65 + .../version-1.5.0/docs/datahub_lite.md | 621 + .../version-1.5.0/docs/dataproducts.md | 423 + .../docs/deploy/BASE_PATH_CONFIGURATION.md | 154 + .../version-1.5.0/docs/deploy/aws.md | 500 + .../version-1.5.0/docs/deploy/azure.md | 231 + .../docs/deploy/confluent-cloud.md | 239 + .../docs/deploy/environment-vars.md | 1211 + .../version-1.5.0/docs/deploy/gcp.md | 115 + .../version-1.5.0/docs/deploy/kubernetes.md | 164 + .../version-1.5.0/docs/deploy/telemetry.md | 35 + .../dev-guides/agent-context/agent-context.md | 308 + .../agent-context/copilot-studio.md | 130 + .../dev-guides/agent-context/google-adk.md | 368 + .../agent-context/google-vertex-ai.md | 86 + .../dev-guides/agent-context/langchain.md | 280 + .../dev-guides/agent-context/snowflake.md | 372 + .../semantic-search/ARCHITECTURE.md | 446 + .../semantic-search/CONFIGURATION.md | 545 + .../docs/dev-guides/semantic-search/README.md | 183 + .../semantic-search/SWITCHING_PROVIDERS.md | 172 + .../version-1.5.0/docs/dev-guides/timeline.md | 265 + .../version-1.5.0/docs/developers.md | 485 + .../docs/developers/java-sdk-v2-design.md | 1278 + .../docs/developers/java-sdk-v2-philosophy.md | 372 + .../version-1.5.0/docs/docker/development.md | 167 + .../version-1.5.0/docs/domains.md | 275 + .../version-1.5.0/docs/features.md | 56 + .../dataset-usage-and-query-history.md | 88 + .../features/feature-guides/access-roles.md | 238 + .../features/feature-guides/applications.md | 124 + .../ask-datahub-plugins/bigquery.md | 151 + .../ask-datahub-plugins/databricks.md | 122 + .../feature-guides/ask-datahub-plugins/dbt.md | 148 + .../ask-datahub-plugins/github.md | 166 + .../ask-datahub-plugins/glean.md | 107 + .../ask-datahub-plugins/overview.md | 174 + .../ask-datahub-plugins/snowflake.md | 199 + .../features/feature-guides/ask-datahub.md | 149 + .../compliance-forms/analytics.md | 148 + .../compliance-forms/complete-a-form.md | 181 + .../compliance-forms/create-a-form.md | 205 + .../compliance-forms/overview.md | 52 + .../context/context-documents.md | 106 + .../feature-guides/custom-asset-summaries.md | 181 + .../feature-guides/custom-home-page.md | 256 + .../feature-guides/file-upload-download.md | 206 + .../docs/features/feature-guides/lineage.md | 112 + .../logical-models/centralized-management.md | 142 + .../feature-guides/logical-models/overview.md | 163 + .../docs/features/feature-guides/mcp.md | 362 + .../properties/create-a-property.md | 273 + .../feature-guides/properties/overview.md | 59 + .../feature-guides/search-access-controls.md | 306 + .../feature-guides/service-accounts.md | 359 + .../features/feature-guides/ui-lineage.md | 67 + .../docs/generated/ingestion/sources/abs.md | 1059 + .../generated/ingestion/sources/athena.md | 1185 + .../generated/ingestion/sources/azure-ad.md | 492 + .../ingestion/sources/azure-data-factory.md | 369 + .../generated/ingestion/sources/bigquery.md | 2150 ++ .../ingestion/sources/business-glossary.md | 433 + .../generated/ingestion/sources/cassandra.md | 726 + .../generated/ingestion/sources/clickhouse.md | 2556 ++ .../ingestion/sources/cockroachdb.md | 1452 + .../generated/ingestion/sources/confluence.md | 1771 + .../ingestion/sources/csv-enricher.md | 196 + .../generated/ingestion/sources/databricks.md | 2119 ++ .../ingestion/sources/datahub-documents.md | 1303 + .../generated/ingestion/sources/datahub.md | 582 + .../ingestion/sources/datahubapply.md | 303 + .../ingestion/sources/datahubdebug.md | 144 + .../generated/ingestion/sources/datahubgc.md | 696 + .../generated/ingestion/sources/dataplex.md | 741 + .../docs/generated/ingestion/sources/db2.md | 1177 + .../docs/generated/ingestion/sources/dbt.md | 2574 ++ .../generated/ingestion/sources/delta-lake.md | 728 + .../generated/ingestion/sources/demo-data.md | 126 + .../docs/generated/ingestion/sources/doris.md | 1288 + .../generated/ingestion/sources/dremio.md | 925 + .../docs/generated/ingestion/sources/druid.md | 1064 + .../generated/ingestion/sources/dynamodb.md | 729 + .../ingestion/sources/elasticsearch.md | 519 + .../docs/generated/ingestion/sources/excel.md | 1286 + .../ingestion/sources/fabric-onelake.md | 1071 + .../docs/generated/ingestion/sources/feast.md | 234 + .../ingestion/sources/file-based-lineage.md | 203 + .../generated/ingestion/sources/fivetran.md | 1425 + .../docs/generated/ingestion/sources/gcs.md | 633 + .../docs/generated/ingestion/sources/glue.md | 960 + .../generated/ingestion/sources/grafana.md | 600 + .../docs/generated/ingestion/sources/hana.md | 1059 + .../docs/generated/ingestion/sources/hex.md | 431 + .../ingestion/sources/hive-metastore.md | 3355 ++ .../docs/generated/ingestion/sources/hive.md | 1459 + .../generated/ingestion/sources/iceberg.md | 894 + .../ingestion/sources/json-schema.md | 286 + .../ingestion/sources/kafka-connect.md | 1375 + .../docs/generated/ingestion/sources/kafka.md | 732 + .../docs/generated/ingestion/sources/ldap.md | 347 + .../generated/ingestion/sources/looker.md | 2119 ++ .../generated/ingestion/sources/mariadb.md | 1312 + .../generated/ingestion/sources/metabase.md | 404 + .../ingestion/sources/metadata-file.md | 232 + .../generated/ingestion/sources/mlflow.md | 353 + .../docs/generated/ingestion/sources/mode.md | 480 + .../generated/ingestion/sources/mongodb.md | 490 + .../docs/generated/ingestion/sources/mssql.md | 2005 + .../docs/generated/ingestion/sources/mysql.md | 1339 + .../docs/generated/ingestion/sources/neo4j.md | 246 + .../docs/generated/ingestion/sources/nifi.md | 440 + .../generated/ingestion/sources/notion.md | 1385 + .../docs/generated/ingestion/sources/okta.md | 443 + .../generated/ingestion/sources/openapi.md | 550 + .../generated/ingestion/sources/oracle.md | 1493 + .../generated/ingestion/sources/postgres.md | 1921 + .../sources/powerbi-report-server.md | 375 + .../generated/ingestion/sources/powerbi.md | 1125 + .../generated/ingestion/sources/preset.md | 493 + .../generated/ingestion/sources/presto.md | 1579 + .../generated/ingestion/sources/pulsar.md | 465 + .../generated/ingestion/sources/qlik-sense.md | 344 + .../docs/generated/ingestion/sources/rdf.md | 490 + .../generated/ingestion/sources/redash.md | 315 + .../generated/ingestion/sources/redshift.md | 1989 + .../docs/generated/ingestion/sources/s3.md | 1263 + .../docs/generated/ingestion/sources/sac.md | 478 + .../generated/ingestion/sources/sagemaker.md | 504 + .../generated/ingestion/sources/salesforce.md | 549 + .../docs/generated/ingestion/sources/sigma.md | 520 + .../docs/generated/ingestion/sources/slack.md | 225 + .../generated/ingestion/sources/snaplogic.md | 297 + .../generated/ingestion/sources/snowflake.md | 2292 ++ .../generated/ingestion/sources/snowplow.md | 1609 + .../ingestion/sources/sql-queries.md | 640 + .../generated/ingestion/sources/sqlalchemy.md | 1005 + .../generated/ingestion/sources/superset.md | 455 + .../generated/ingestion/sources/tableau.md | 1010 + .../generated/ingestion/sources/teradata.md | 1362 + .../docs/generated/ingestion/sources/trino.md | 2391 ++ .../generated/ingestion/sources/vertexai.md | 930 + .../generated/ingestion/sources/vertica.md | 1126 + .../lineage/automatic-lineage-extraction.md | 141 + .../metamodel/entities/application.md | 2429 ++ .../generated/metamodel/entities/assertion.md | 4134 ++ .../metamodel/entities/businessAttribute.md | 1222 + .../generated/metamodel/entities/chart.md | 4324 +++ .../generated/metamodel/entities/container.md | 2867 ++ .../generated/metamodel/entities/corpGroup.md | 2220 ++ .../generated/metamodel/entities/corpuser.md | 2815 ++ .../generated/metamodel/entities/dashboard.md | 4635 +++ .../metamodel/entities/dataContract.md | 994 + .../generated/metamodel/entities/dataFlow.md | 3231 ++ .../metamodel/entities/dataHubAccessToken.md | 140 + .../metamodel/entities/dataHubConnection.md | 191 + .../entities/dataHubExecutionRequest.md | 390 + .../metamodel/entities/dataHubFile.md | 359 + .../entities/dataHubIngestionSource.md | 584 + .../entities/dataHubOpenAPISchema.md | 234 + .../metamodel/entities/dataHubPageModule.md | 387 + .../metamodel/entities/dataHubPageTemplate.md | 347 + .../metamodel/entities/dataHubPersona.md | 77 + .../metamodel/entities/dataHubPolicy.md | 419 + .../metamodel/entities/dataHubRetention.md | 165 + .../metamodel/entities/dataHubRole.md | 107 + .../metamodel/entities/dataHubSecret.md | 171 + .../metamodel/entities/dataHubStepState.md | 146 + .../metamodel/entities/dataHubUpgrade.md | 161 + .../metamodel/entities/dataHubView.md | 350 + .../generated/metamodel/entities/dataJob.md | 4672 +++ .../metamodel/entities/dataPlatform.md | 612 + .../entities/dataPlatformInstance.md | 1584 + .../metamodel/entities/dataProcess.md | 1431 + .../metamodel/entities/dataProcessInstance.md | 2563 ++ .../metamodel/entities/dataProduct.md | 2496 ++ .../generated/metamodel/entities/dataType.md | 284 + .../generated/metamodel/entities/dataset.md | 7373 ++++ .../generated/metamodel/entities/document.md | 2186 ++ .../generated/metamodel/entities/domain.md | 2009 + .../metamodel/entities/entityType.md | 284 + .../metamodel/entities/erModelRelationship.md | 2173 ++ .../docs/generated/metamodel/entities/form.md | 1453 + .../metamodel/entities/globalSettings.md | 491 + .../metamodel/entities/glossaryNode.md | 2371 ++ .../metamodel/entities/glossaryTerm.md | 3775 ++ .../generated/metamodel/entities/incident.md | 1178 + .../metamodel/entities/inviteToken.md | 97 + .../generated/metamodel/entities/mlFeature.md | 3334 ++ .../metamodel/entities/mlFeatureTable.md | 2812 ++ .../generated/metamodel/entities/mlModel.md | 4880 +++ .../metamodel/entities/mlModelDeployment.md | 1651 + .../metamodel/entities/mlModelGroup.md | 3043 ++ .../metamodel/entities/mlPrimaryKey.md | 2734 ++ .../generated/metamodel/entities/notebook.md | 2608 ++ .../metamodel/entities/ownershipType.md | 631 + .../metamodel/entities/platformResource.md | 284 + .../docs/generated/metamodel/entities/post.md | 393 + .../generated/metamodel/entities/query.md | 1256 + .../docs/generated/metamodel/entities/role.md | 609 + .../metamodel/entities/schemaField.md | 2388 ++ .../metamodel/entities/structuredProperty.md | 1368 + .../docs/generated/metamodel/entities/tag.md | 1169 + .../generated/metamodel/entities/telemetry.md | 79 + .../docs/generated/metamodel/entities/test.md | 190 + .../metamodel/entities/versionSet.md | 879 + .../docs/glossary/business-glossary.md | 177 + .../how-to/semantic-search-configuration.md | 199 + .../docs/how/add-custom-data-platform.md | 117 + .../docs/how/add-custom-ingestion-source.md | 47 + .../version-1.5.0/docs/how/add-new-aspect.md | 27 + .../version-1.5.0/docs/how/add-user-data.md | 99 + .../version-1.5.0/docs/how/backup-datahub.md | 219 + .../version-1.5.0/docs/how/configure-cdc.md | 1206 + ...guring-authorization-with-apache-ranger.md | 267 + .../docs/how/debug-ingestion-recording.md | 234 + .../version-1.5.0/docs/how/delete-metadata.md | 295 + .../how/elasticsearch-search-client-shim.md | 355 + .../docs/how/extract-container-logs.md | 141 + .../version-1.5.0/docs/how/jattach-guide.md | 91 + .../version-1.5.0/docs/how/kafka-config.md | 336 + .../version-1.5.0/docs/how/load-indices.md | 488 + .../how/migrating-elasticsearch-opensearch.md | 161 + .../migrating-graph-service-implementation.md | 56 + .../version-1.5.0/docs/how/restore-indices.md | 338 + .../version-1.5.0/docs/how/search.md | 754 + .../version-1.5.0/docs/how/ui-tabs-guide.md | 25 + .../docs/how/updating-datahub.md | 1008 + .../version-1.5.0/docs/iceberg-catalog.md | 564 + .../version-1.5.0/docs/incidents/incidents.md | 434 + .../version-1.5.0/docs/lineage/airflow.md | 547 + .../version-1.5.0/docs/lineage/dagster.md | 242 + .../version-1.5.0/docs/lineage/openlineage.md | 200 + .../version-1.5.0/docs/lineage/prefect.md | 142 + .../version-1.5.0/docs/lineage/sql_parsing.md | 69 + .../version-1.5.0/docs/links.md | 72 + .../docs/managed-datahub/change-proposals.md | 368 + .../docs/managed-datahub/chrome-extension.md | 99 + ...ing-identity-provisioning-with-ms-entra.md | 100 + ...iguring-identity-provisioning-with-okta.md | 117 + .../datahub-api/entity-events-api.md | 909 + .../graphql-api/getting-started.md | 48 + .../integrations/aws-privatelink.md | 38 + .../integrations/oidc-sso-integration.md | 50 + .../managed-datahub-overview.md | 77 + .../observe/assertion-backfill.md | 196 + .../observe/assertion-notes.md | 37 + .../observe/assertion-query-attribution.md | 41 + .../managed-datahub/observe/assertions.md | 113 + .../observe/column-assertions.md | 520 + .../observe/custom-sql-assertions.md | 297 + .../managed-datahub/observe/data-contract.md | 95 + .../observe/data-health-dashboard.md | 103 + .../observe/freshness-assertions.md | 382 + .../observe/schema-assertions.md | 267 + .../observe/smart-assertions.md | 95 + .../observe/volume-assertions.md | 468 + ...etting-up-events-api-on-aws-eventbridge.md | 143 + .../setting-up-remote-ingestion-executor.md | 402 + .../managed-datahub/release-notes/next.md | 42 + .../managed-datahub/release-notes/v_0_1_69.md | 21 + .../managed-datahub/release-notes/v_0_1_70.md | 22 + .../managed-datahub/release-notes/v_0_1_72.md | 27 + .../managed-datahub/release-notes/v_0_1_73.md | 22 + .../managed-datahub/release-notes/v_0_2_0.md | 34 + .../managed-datahub/release-notes/v_0_2_1.md | 24 + .../managed-datahub/release-notes/v_0_2_10.md | 40 + .../managed-datahub/release-notes/v_0_2_11.md | 82 + .../managed-datahub/release-notes/v_0_2_12.md | 38 + .../managed-datahub/release-notes/v_0_2_13.md | 29 + .../managed-datahub/release-notes/v_0_2_14.md | 25 + .../managed-datahub/release-notes/v_0_2_15.md | 84 + .../managed-datahub/release-notes/v_0_2_16.md | 25 + .../managed-datahub/release-notes/v_0_2_2.md | 25 + .../managed-datahub/release-notes/v_0_2_3.md | 27 + .../managed-datahub/release-notes/v_0_2_4.md | 30 + .../managed-datahub/release-notes/v_0_2_5.md | 30 + .../managed-datahub/release-notes/v_0_2_6.md | 28 + .../managed-datahub/release-notes/v_0_2_7.md | 38 + .../managed-datahub/release-notes/v_0_2_8.md | 42 + .../managed-datahub/release-notes/v_0_2_9.md | 54 + .../managed-datahub/release-notes/v_0_3_1.md | 25 + .../managed-datahub/release-notes/v_0_3_10.md | 129 + .../managed-datahub/release-notes/v_0_3_11.md | 127 + .../managed-datahub/release-notes/v_0_3_12.md | 143 + .../managed-datahub/release-notes/v_0_3_13.md | 176 + .../managed-datahub/release-notes/v_0_3_14.md | 76 + .../managed-datahub/release-notes/v_0_3_15.md | 81 + .../managed-datahub/release-notes/v_0_3_16.md | 179 + .../managed-datahub/release-notes/v_0_3_17.md | 81 + .../managed-datahub/release-notes/v_0_3_2.md | 43 + .../managed-datahub/release-notes/v_0_3_3.md | 30 + .../managed-datahub/release-notes/v_0_3_4.md | 49 + .../managed-datahub/release-notes/v_0_3_5.md | 43 + .../managed-datahub/release-notes/v_0_3_6.md | 111 + .../managed-datahub/release-notes/v_0_3_7.md | 208 + .../managed-datahub/release-notes/v_0_3_8.md | 117 + .../managed-datahub/release-notes/v_0_3_9.md | 88 + .../managed-datahub/remote-executor/about.md | 91 + .../remote-executor/monitoring.md | 98 + .../managed-datahub/slack/saas-slack-app.md | 114 + .../managed-datahub/slack/saas-slack-setup.md | 279 + .../slack/saas-slack-troubleshoot.md | 111 + .../subscription-and-notification.md | 207 + .../managed-datahub/teams/saas-teams-app.md | 57 + .../managed-datahub/teams/saas-teams-setup.md | 79 + .../managed-datahub/upgrade_core_to_cloud.md | 268 + .../docs/managed-datahub/welcome-acryl.md | 63 + .../workflows/access-workflows.md | 430 + .../docs/metadata-ingestion-security.md | 124 + .../version-1.5.0/docs/metadata-standards.md | 69 + .../modeling/extending-the-metadata-model.md | 806 + .../docs/modeling/metadata-model.md | 630 + .../docs/ownership/ownership-types.md | 194 + .../version-1.5.0/docs/platform-instances.md | 224 + .../version-1.5.0/docs/plugins.md | 323 + .../version-1.5.0/docs/posts.md | 156 + .../bigquery/configuration.md | 151 + .../bigquery/overview.md | 42 + .../quick-ingestion-guides/bigquery/setup.md | 69 + .../looker/configuration.md | 213 + .../quick-ingestion-guides/looker/overview.md | 56 + .../quick-ingestion-guides/looker/setup.md | 160 + .../powerbi/configuration.md | 142 + .../powerbi/overview.md | 35 + .../quick-ingestion-guides/powerbi/setup.md | 108 + .../redshift/configuration.md | 139 + .../redshift/overview.md | 41 + .../quick-ingestion-guides/redshift/setup.md | 135 + .../snowflake/configuration.md | 149 + .../snowflake/overview.md | 51 + .../quick-ingestion-guides/snowflake/setup.md | 78 + .../tableau/configuration.md | 155 + .../tableau/overview.md | 41 + .../quick-ingestion-guides/tableau/setup.md | 68 + .../version-1.5.0/docs/quickstart.md | 330 + .../versioned_docs/version-1.5.0/docs/rfc.md | 126 + .../version-1.5.0/docs/rfcs/README.md | 48 + .../version-1.5.0/docs/roadmap.md | 177 + .../version-1.5.0/docs/schema-history.md | 67 + .../version-1.5.0/docs/slack.md | 53 + .../version-1.5.0/docs/sync-status.md | 49 + .../versioned_docs/version-1.5.0/docs/tags.md | 114 + .../docs/tests/metadata-tests.md | 303 + .../version-1.5.0/docs/townhall-history.md | 557 + .../version-1.5.0/docs/townhalls.md | 60 + .../docs/troubleshooting/build.md | 48 + .../docs/troubleshooting/general.md | 18 + .../docs/troubleshooting/quickstart.md | 313 + .../version-1.5.0/docs/ui-ingestion.md | 379 + .../docs/what-is-datahub/datahub-concepts.md | 213 + .../version-1.5.0/docs/what/aspect.md | 51 + .../version-1.5.0/docs/what/delta.md | 66 + .../version-1.5.0/docs/what/entity.md | 9 + .../version-1.5.0/docs/what/gma.md | 18 + .../version-1.5.0/docs/what/gms.md | 10 + .../version-1.5.0/docs/what/graph.md | 21 + .../version-1.5.0/docs/what/mxe.md | 431 + .../version-1.5.0/docs/what/relationship.md | 80 + .../docs/what/search-document.md | 69 + .../version-1.5.0/docs/what/search-index.md | 24 + .../version-1.5.0/docs/what/snapshot.md | 54 + .../version-1.5.0/docs/what/urn.md | 62 + .../version-1.5.0/graphql/enums.md | 4254 +++ .../version-1.5.0/graphql/inputObjects.md | 11450 ++++++ .../version-1.5.0/graphql/interfaces.md | 418 + .../version-1.5.0/graphql/mutations.md | 4004 ++ .../version-1.5.0/graphql/objects.md | 31150 ++++++++++++++++ .../version-1.5.0/graphql/queries.md | 2259 ++ .../version-1.5.0/graphql/scalars.md | 27 + .../version-1.5.0/graphql/unions.md | 72 + .../airflow-plugin/AIRFLOW_3_MIGRATION.md | 1209 + .../airflow-plugin/DOCKER_TEST_GUIDE.md | 483 + .../airflow-plugin/README.docker.md | 137 + .../airflow-plugin/README.md | 232 + .../airflow-plugin/scripts/README.md | 15 + .../example_dags/README.md | 152 + .../dagster-plugin/README.md | 9 + .../gx-plugin/README.md | 9 + .../prefect-plugin/README.md | 138 + .../metadata-ingestion/CLAUDE.md | 141 + .../KAFKA_CONNECT_LINEAGE.md | 532 + .../metadata-ingestion/README.md | 46 + .../metadata-ingestion/adding-source.md | 271 + .../metadata-ingestion/as-a-library.md | 138 + .../metadata-ingestion/cli-ingestion.md | 134 + .../metadata-ingestion/datahub-skills.md | 211 + .../metadata-ingestion/developing.md | 359 + .../add_stateful_ingestion_to_source.md | 202 + .../docs/dev_guides/classification.md | 460 + .../docs/dev_guides/profiling_ingestions.md | 92 + .../docs/dev_guides/reporting_telemetry.md | 112 + .../docs/dev_guides/sql_profiles.md | 62 + .../docs/dev_guides/stateful.md | 187 + .../docs/transformer/dataset_transformer.md | 1871 + .../docs/transformer/intro.md | 54 + .../transformer/universal_transformers.md | 114 + .../examples/library/README.md | 220 + .../examples/structured_properties/README.md | 57 + .../examples/transforms/README.md | 11 + .../integration_docs/great-expectations.md | 72 + .../metadata-ingestion/recipe_overview.md | 159 + .../schedule_docs/airflow.md | 48 + .../metadata-ingestion/schedule_docs/cron.md | 35 + .../schedule_docs/datahub.md | 9 + .../metadata-ingestion/schedule_docs/intro.md | 37 + .../schedule_docs/kubernetes.md | 54 + .../metadata-ingestion/sink_docs/console.md | 35 + .../metadata-ingestion/sink_docs/datahub.md | 229 + .../sink_docs/metadata-file.md | 43 + .../metadata-ingestion/sink_overview.md | 39 + .../source-docs-template.md | 157 + .../metadata-ingestion/source_overview.md | 42 + .../cli/resources/GRAPHQL_AGENT_CONTEXT.md | 133 + .../cli/resources/INIT_AGENT_CONTEXT.md | 72 + .../cli/resources/SEARCH_AGENT_CONTEXT.md | 162 + .../src/datahub/ingestion/recording/README.md | 625 + .../datahub/ingestion/source/rdf/README.md | 230 + .../source/rdf/docs/ENTITY_PLUGIN_CONTRACT.md | 445 + .../ingestion/source/rdf/docs/README.md | 145 + .../ingestion/source/rdf/docs/background.md | 208 + .../source/rdf/docs/rdf-specification.md | 561 + .../user-stories-and-acceptance-criteria.md | 584 + .../source/rdf/entities/domain/SPEC.md | 163 + .../source/rdf/entities/glossary_term/SPEC.md | 553 + .../source/rdf/entities/relationship/SPEC.md | 166 + .../ingestion/source/rdf/ingestion/README.md | 170 + .../ingestion/source/snowplow/README.md | 416 + .../java/acryl-spark-lineage/README.md | 491 + .../java/as-a-library-v2.md | 199 + .../metadata-integration/java/as-a-library.md | 257 + .../java/datahub-client/CLAUDE.md | 93 + .../java/datahub-protobuf/README.md | 693 + .../java/datahub-schematron/README.md | 87 + .../java/docs/sdk-v2/chart-entity.md | 1046 + .../java/docs/sdk-v2/client.md | 483 + .../java/docs/sdk-v2/container-entity.md | 521 + .../java/docs/sdk-v2/dashboard-entity.md | 1194 + .../java/docs/sdk-v2/dataflow-entity.md | 415 + .../java/docs/sdk-v2/datajob-entity.md | 1791 + .../java/docs/sdk-v2/dataset-entity.md | 813 + .../java/docs/sdk-v2/design-principles.md | 466 + .../java/docs/sdk-v2/entities-overview.md | 345 + .../java/docs/sdk-v2/getting-started.md | 460 + .../java/docs/sdk-v2/migration-from-v1.md | 494 + .../java/docs/sdk-v2/mlmodel-entity.md | 483 + .../java/docs/sdk-v2/mlmodelgroup-entity.md | 588 + .../java/docs/sdk-v2/patch-operations.md | 440 + .../java/openlineage-converter/README.md | 23 + .../java/spark-lineage-legacy/README.md | 278 + .../version-1.5.0/metadata-io/README.md | 28 + .../elasticsearch/client/shim/README.md | 152 + .../version-1.5.0/metadata-jobs/README.md | 18 + .../metadata-jobs/mae-consumer-job/README.md | 68 + .../metadata-jobs/mce-consumer-job/README.md | 102 + .../metadata-models-custom/README.md | 573 + .../version-1.5.0/metadata-service/README.md | 1536 + .../schema-registry-servlet/scripts/README.md | 45 + .../metadata-service/services/README.md | 11 + .../version-1.5.0/perf-test/README.md | 98 + .../python-sdk/builder/mce-builder.mdx | 1103 + .../python-sdk/builder/mcp-builder.mdx | 896 + .../python-sdk/clients/graph-client.mdx | 803 + .../python-sdk/clients/kafka-emitter.mdx | 97 + .../python-sdk/clients/rest-emitter.mdx | 297 + .../version-1.5.0/python-sdk/models.mdx | 19873 ++++++++++ .../python-sdk/sdk-v2/entities.mdx | 1678 + .../python-sdk/sdk-v2/entity-client.mdx | 96 + .../python-sdk/sdk-v2/lineage-client.mdx | 199 + .../python-sdk/sdk-v2/main-client.mdx | 105 + .../python-sdk/sdk-v2/resolver-client.mdx | 49 + .../python-sdk/sdk-v2/search-client.mdx | 234 + .../version-1.5.0/python-sdk/urns.mdx | 8272 ++++ .../versioned_docs/version-1.5.0/releases.md | 557 + .../version-1.5.0/smoke-test/README.md | 201 + .../analytics_backfill/README.md | 499 + .../smoke-test/tests/cypress/README.md | 40 + .../tests/library_examples/README.md | 255 + .../smoke-test/tests/openapi/README.md | 38 + .../smoke-test/tests/semantic/README.md | 88 + .../version-1.5.0-sidebars.json | 8 + docs-archive/versions.json | 2 +- 624 files changed, 379181 insertions(+), 64 deletions(-) delete mode 100644 .github/workflows/docs-archive-deploy.yml create mode 100644 docs-archive/scripts/deploy/README.md create mode 100755 docs-archive/scripts/deploy/build-archive.sh create mode 100755 docs-archive/scripts/deploy/package-prebuilt.sh create mode 100644 docs-archive/versioned_docs/version-1.5.0/AGENTS.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/CLAUDE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/SECURITY.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-actions/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/propagation/docs/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/tag/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/term/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/src/datahub_agent_context/snowflake/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-frontend/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-graphql-core/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-web-react/BASE_PATH.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-web-react/CLAUDE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-web-react/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/analytics/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/ARCHITECTURE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/changeMessages/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/mfeframework/README-MFE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docker/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docker/airflow/local_airflow.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docker/datahub-upgrade/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/CODE_OF_CONDUCT.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/CONTRIBUTING.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/PYSPARK.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/SECURITY_STANCE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/_api-guide-template.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/_feature-guide-template.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata/impact-analysis.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/executor.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/hello_world.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/slack.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/concepts.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/events/audit-events-search-guide.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/events/entity-change-event.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/events/metadata-change-log-event.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-a-transformer.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-an-action.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/quickstart.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/datahub-cloud-event-source.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/kafka-event-source.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/api-tracing.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/aspect-versioning.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/backfilling.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/bootstrap-mcps.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/browse-paths-upgrade.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/db-retention.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/derived-aspects.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/entity-hierarchy.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/field-path-spec-v2.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/high-cardinality.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/jar-extraction-tmpfs-optimization.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/mcp-mcl.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/micrometer-best-practices.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/monitoring.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/partial-update.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/patch.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/pdl-best-practices.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/advanced/writing-mcps.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/datahub-apis.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/getting-started.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-best-practices.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-endpoint-development.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/how-to-set-up-graphql.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/token-management.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/openapi/openapi-usage-guide.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/restli/evaluate-tests.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-elastic-task-status.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-index-sizes.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restli-overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restore-indices.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/restli/truncate-time-series-aspect.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/applications.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/container.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-properties.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dashboard-chart.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/data-contracts.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dataflow-datajob.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/datasets.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/deprecation.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/descriptions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/documents.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/domains.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/forms.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/incidents.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/lineage.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml_feature_store.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/mlmodel-mlmodelgroup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/operations.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/owners.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/bulk-assertions-sdk.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/search_client.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/structured-properties.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/subscriptions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/tags.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/terms.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/architecture/architecture.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/architecture/docker-containers.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-ingestion.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-serving.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/assertions/open-assertions-spec.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/assertions/snowflake/snowflake_dmfs.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/changing-default-credentials.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/concepts.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/external-oauth-providers.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/add-users.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/jaas.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-behind-proxy.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-react.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/initialize-oidc.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/introducing-metadata-service-authentication.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authentication/personal-access-tokens.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authorization/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authorization/access-policies-guide.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authorization/groups.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authorization/policies.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/authorization/roles.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-docs.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-term-suggestion.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/automations/bigquery-metadata-sync.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/automations/databricks-metadata-sync.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/automations/docs-propagation.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/automations/glossary-term-propagation.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/automations/snowflake-tag-propagation.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/browseV2/browse-paths-v2.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/businessattributes.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/dataset.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/graphql.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/search.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/cli.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/components.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/datahub_lite.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dataproducts.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/BASE_PATH_CONFIGURATION.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/aws.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/azure.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/confluent-cloud.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/environment-vars.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/gcp.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/kubernetes.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/deploy/telemetry.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/agent-context.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/copilot-studio.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-adk.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-vertex-ai.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/langchain.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/snowflake.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/ARCHITECTURE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/CONFIGURATION.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/SWITCHING_PROVIDERS.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/timeline.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/developers.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-design.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-philosophy.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/docker/development.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/domains.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/dataset-usage-and-query-history.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/access-roles.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/applications.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/bigquery.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/databricks.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/dbt.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/github.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/glean.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/snowflake.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/analytics.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/complete-a-form.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/create-a-form.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/context/context-documents.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-asset-summaries.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-home-page.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/file-upload-download.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/lineage.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/centralized-management.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/mcp.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/create-a-property.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/search-access-controls.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/service-accounts.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ui-lineage.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/abs.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/athena.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-ad.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-data-factory.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/bigquery.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/business-glossary.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cassandra.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/clickhouse.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cockroachdb.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/confluence.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/csv-enricher.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/databricks.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub-documents.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubapply.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubdebug.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubgc.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dataplex.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/db2.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dbt.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/delta-lake.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/demo-data.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/doris.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dremio.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/druid.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dynamodb.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/elasticsearch.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/excel.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fabric-onelake.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/feast.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/file-based-lineage.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fivetran.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/gcs.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/glue.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/grafana.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hana.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hex.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive-metastore.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/iceberg.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/json-schema.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka-connect.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/ldap.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/looker.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/mariadb.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/metabase.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/metadata-file.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/mlflow.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/mode.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/mongodb.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/mssql.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/mysql.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/neo4j.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/nifi.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/notion.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/okta.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/openapi.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/oracle.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/postgres.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/powerbi-report-server.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/powerbi.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/preset.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/presto.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/pulsar.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/qlik-sense.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/rdf.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/redash.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/redshift.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/s3.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/sac.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/sagemaker.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/salesforce.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/sigma.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/slack.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/snaplogic.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/snowflake.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/snowplow.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/sql-queries.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/sqlalchemy.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/superset.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/tableau.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/teradata.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/trino.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/vertexai.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/vertica.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/lineage/automatic-lineage-extraction.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/application.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/assertion.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/businessAttribute.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/chart.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/container.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/corpGroup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/corpuser.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dashboard.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataContract.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataFlow.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubAccessToken.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubConnection.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubExecutionRequest.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubFile.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubIngestionSource.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubOpenAPISchema.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubPageModule.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubPageTemplate.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubPersona.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubPolicy.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubRetention.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubRole.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubSecret.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubStepState.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubUpgrade.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataHubView.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataJob.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataPlatform.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataPlatformInstance.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataProcess.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataProcessInstance.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataProduct.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataType.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/dataset.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/document.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/domain.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/entityType.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/erModelRelationship.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/form.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/globalSettings.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/glossaryNode.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/glossaryTerm.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/incident.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/inviteToken.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/mlFeature.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/mlFeatureTable.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/mlModel.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/mlModelDeployment.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/mlModelGroup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/mlPrimaryKey.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/notebook.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/ownershipType.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/platformResource.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/post.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/query.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/role.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/schemaField.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/structuredProperty.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/tag.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/telemetry.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/test.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/generated/metamodel/entities/versionSet.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/glossary/business-glossary.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how-to/semantic-search-configuration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/add-custom-data-platform.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/add-custom-ingestion-source.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/add-new-aspect.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/add-user-data.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/backup-datahub.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/configure-cdc.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/configuring-authorization-with-apache-ranger.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/debug-ingestion-recording.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/delete-metadata.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/elasticsearch-search-client-shim.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/extract-container-logs.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/jattach-guide.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/kafka-config.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/load-indices.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/migrating-elasticsearch-opensearch.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/migrating-graph-service-implementation.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/restore-indices.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/search.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/ui-tabs-guide.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/how/updating-datahub.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/iceberg-catalog.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/incidents/incidents.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/lineage/airflow.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/lineage/dagster.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/lineage/openlineage.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/lineage/prefect.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/lineage/sql_parsing.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/links.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/change-proposals.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/chrome-extension.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-ms-entra.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-okta.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/entity-events-api.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/graphql-api/getting-started.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/aws-privatelink.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/oidc-sso-integration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/managed-datahub-overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-backfill.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-notes.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-query-attribution.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/column-assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/custom-sql-assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-contract.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-health-dashboard.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/freshness-assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/schema-assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/smart-assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/volume-assertions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/next.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_69.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_70.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_72.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_73.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_0.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_1.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_10.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_11.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_12.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_13.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_14.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_15.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_16.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_2.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_3.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_4.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_5.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_6.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_7.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_8.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_9.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_1.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_10.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_11.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_12.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_13.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_14.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_15.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_16.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_17.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_2.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_3.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_4.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_5.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_6.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_7.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_8.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_3_9.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/remote-executor/about.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/remote-executor/monitoring.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/slack/saas-slack-app.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/slack/saas-slack-setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/slack/saas-slack-troubleshoot.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/subscription-and-notification.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-app.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/upgrade_core_to_cloud.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/welcome-acryl.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/workflows/access-workflows.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/metadata-ingestion-security.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/metadata-standards.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/modeling/extending-the-metadata-model.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/modeling/metadata-model.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/ownership/ownership-types.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/platform-instances.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/plugins.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/posts.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/configuration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/configuration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/configuration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/configuration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/configuration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/configuration.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/setup.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/quickstart.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/rfc.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/rfcs/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/roadmap.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/schema-history.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/slack.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/sync-status.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/tags.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/tests/metadata-tests.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/townhall-history.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/townhalls.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/build.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/general.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/quickstart.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/ui-ingestion.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what-is-datahub/datahub-concepts.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/aspect.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/delta.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/gma.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/gms.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/graph.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/mxe.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/relationship.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/search-document.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/search-index.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/snapshot.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/docs/what/urn.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/enums.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/inputObjects.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/interfaces.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/mutations.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/objects.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/queries.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/scalars.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/graphql/unions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/AIRFLOW_3_MIGRATION.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/DOCKER_TEST_GUIDE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.docker.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/scripts/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/dagster-plugin/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/gx-plugin/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/prefect-plugin/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/CLAUDE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/KAFKA_CONNECT_LINEAGE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/adding-source.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/as-a-library.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/cli-ingestion.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/datahub-skills.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/developing.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/classification.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/profiling_ingestions.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/reporting_telemetry.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/sql_profiles.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/stateful.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/dataset_transformer.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/intro.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/universal_transformers.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/library/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/structured_properties/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/transforms/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/integration_docs/great-expectations.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/recipe_overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/airflow.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/cron.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/datahub.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/intro.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/kubernetes.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/console.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/datahub.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/metadata-file.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/source-docs-template.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/source_overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/cli/resources/GRAPHQL_AGENT_CONTEXT.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/cli/resources/INIT_AGENT_CONTEXT.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/cli/resources/SEARCH_AGENT_CONTEXT.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/recording/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/docs/ENTITY_PLUGIN_CONTRACT.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/docs/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/docs/background.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/docs/rdf-specification.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/docs/user-stories-and-acceptance-criteria.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/entities/domain/SPEC.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/entities/glossary_term/SPEC.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/entities/relationship/SPEC.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/rdf/ingestion/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/src/datahub/ingestion/source/snowplow/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/acryl-spark-lineage/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/as-a-library-v2.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/as-a-library.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/datahub-client/CLAUDE.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/datahub-protobuf/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/datahub-schematron/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/chart-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/client.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/container-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/dashboard-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/dataflow-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/datajob-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/dataset-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/design-principles.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/entities-overview.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/getting-started.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/migration-from-v1.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/mlmodel-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/mlmodelgroup-entity.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/docs/sdk-v2/patch-operations.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/openlineage-converter/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-integration/java/spark-lineage-legacy/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-io/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/client/shim/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-jobs/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-jobs/mae-consumer-job/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-jobs/mce-consumer-job/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-models-custom/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-service/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-service/schema-registry-servlet/scripts/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/metadata-service/services/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/perf-test/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/builder/mce-builder.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/builder/mcp-builder.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/clients/graph-client.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/clients/kafka-emitter.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/clients/rest-emitter.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/models.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/sdk-v2/entities.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/sdk-v2/entity-client.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/sdk-v2/lineage-client.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/sdk-v2/main-client.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/sdk-v2/resolver-client.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/sdk-v2/search-client.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/python-sdk/urns.mdx create mode 100644 docs-archive/versioned_docs/version-1.5.0/releases.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/smoke-test/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/smoke-test/test_resources/analytics_backfill/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/smoke-test/tests/cypress/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/smoke-test/tests/library_examples/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/smoke-test/tests/openapi/README.md create mode 100644 docs-archive/versioned_docs/version-1.5.0/smoke-test/tests/semantic/README.md create mode 100644 docs-archive/versioned_sidebars/version-1.5.0-sidebars.json diff --git a/.github/workflows/docs-archive-deploy.yml b/.github/workflows/docs-archive-deploy.yml deleted file mode 100644 index f413152b..00000000 --- a/.github/workflows/docs-archive-deploy.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Docs Archive Vercel Deploy - -on: - push: - branches: - - main - paths: - - "docs-archive/**" - - ".github/workflows/docs-archive-deploy.yml" - workflow_dispatch: - -jobs: - build-and-deploy: - runs-on: ubuntu-latest - timeout-minutes: 120 - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: npm - cache-dependency-path: docs-archive/package-lock.json - - - name: Install dependencies - working-directory: docs-archive - run: npm ci - - - name: Allocate Swap Space - run: | - sudo swapoff -a || true - sudo rm -f /swapfile - - sudo fallocate -l 10G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=10240 - sudo chmod 600 /swapfile - sudo mkswap /swapfile - sudo swapon /swapfile - sudo swapon --show - free -h - - - name: Build Docusaurus site - working-directory: docs-archive - run: | - export NODE_OPTIONS="--max-old-space-size=10240" - export DOCUSAURUS_SSR_CONCURRENCY=1 - npm run build - - - name: Deploy to Vercel (prebuilt) - working-directory: docs-archive - env: - VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }} - VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }} - run: | - npx vercel deploy --prebuilt --prod --yes --token=${{ secrets.VERCEL_TOKEN }} \ No newline at end of file diff --git a/docs-archive/.gitignore b/docs-archive/.gitignore index 14f91acd..3e92493f 100644 --- a/docs-archive/.gitignore +++ b/docs-archive/.gitignore @@ -4,6 +4,9 @@ # Production /build +# Vercel prebuilt output (generated by scripts/deploy/package-prebuilt.sh) +/.vercel + # Generated files .docusaurus .cache-loader diff --git a/docs-archive/docusaurus.config.js b/docs-archive/docusaurus.config.js index 49cd8e9e..22c3d5f3 100644 --- a/docs-archive/docusaurus.config.js +++ b/docs-archive/docusaurus.config.js @@ -31,9 +31,10 @@ const config = { routeBasePath: '/docs', includeCurrentVersion: false, sidebarPath: require.resolve('./sidebars.js'), - // 1.3.0 is "latest" in this archive — served without version prefix - lastVersion: '1.3.0', + // 1.5.0 is the newest version in this archive + lastVersion: '1.5.0', versions: { + '1.5.0': { label: '1.5.0', banner: 'none', path: '1.5.0' }, '1.3.0': { label: '1.3.0', banner: 'none', path: '1.3.0' }, '1.1.0': { label: '1.1.0', banner: 'none', path: '1.1.0' }, '1.0.0': { label: '1.0.0', banner: 'none', path: '1.0.0' }, @@ -79,7 +80,7 @@ const config = { dropdownItemsBefore: [ { href: 'https://docs.datahub.com', - label: '1.4.0 (Latest) ↗', + label: 'Latest ↗', }, { type: 'html', @@ -125,8 +126,8 @@ const config = { { title: 'Docs', items: [ - { label: 'Introduction', to: 'docs/1.3.0/features' }, - { label: 'Quickstart', to: 'docs/1.3.0/quickstart' }, + { label: 'Introduction', to: 'docs/1.5.0/features' }, + { label: 'Quickstart', to: 'docs/1.5.0/quickstart' }, ], }, { @@ -141,7 +142,7 @@ const config = { { title: 'More', items: [ - { label: 'Latest Docs (1.4.0)', href: 'https://docs.datahub.com' }, + { label: 'Latest Docs ↗', href: 'https://docs.datahub.com' }, { label: 'Roadmap', href: 'https://feature-requests.datahubproject.io/roadmap' }, { label: 'GitHub', href: 'https://github.com/datahub-project/datahub' }, ], diff --git a/docs-archive/scripts/deploy/README.md b/docs-archive/scripts/deploy/README.md new file mode 100644 index 00000000..f92ad69a --- /dev/null +++ b/docs-archive/scripts/deploy/README.md @@ -0,0 +1,43 @@ +# Docs archive — deploy scripts + +This archive builds many documentation versions with Docusaurus in a single +site. That build is memory-intensive and can exceed the memory available on +hosted build environments. To avoid that, we build the static site in an +environment we control and then upload the finished output to Vercel, so Vercel +serves it without rebuilding. + +## Flow + +```bash +# 1. Build the static site -> build/ +./scripts/deploy/build-archive.sh + +# 2. Wrap build/ into Vercel's Build Output API layout -> .vercel/output/ +./scripts/deploy/package-prebuilt.sh + +# 3. Upload the prebuilt output (Vercel serves it as-is, no rebuild). +export VERCEL_TOKEN=... # provide via a secret; never commit it +export VERCEL_ORG_ID=... +export VERCEL_PROJECT_ID=... +npx vercel deploy --prebuilt --prod --yes --token="$VERCEL_TOKEN" +``` + +## Why `--prebuilt` + +`--prebuilt` tells Vercel to upload the contents of `.vercel/output/` directly +instead of building from source. This keeps the build in an environment with +enough memory and avoids rebuilding on deploy. + +## Vercel project settings + +- The archive is its own Vercel project, served at `archive.docs.datahub.com`. +- Git-triggered deploys should be disabled for that project so a push does not + start a build from source. Deploys are expected to run via the CLI with + `--prebuilt`. + +## Committed vs generated + +- **Committed:** documentation source (`versioned_docs/`, configs) and these + scripts. +- **Generated (git-ignored):** `build/`, `.docusaurus/`, `node_modules/`, + `.vercel/`. diff --git a/docs-archive/scripts/deploy/build-archive.sh b/docs-archive/scripts/deploy/build-archive.sh new file mode 100755 index 00000000..7e70268d --- /dev/null +++ b/docs-archive/scripts/deploy/build-archive.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Build the docs archive with the memory settings needed for the full +# multi-version Docusaurus build. +# +# Output lands in docs-archive/build/ (Docusaurus static output). +# Run scripts/deploy/package-prebuilt.sh next to wrap it for Vercel. +# +set -euo pipefail + +ARCHIVE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ARCHIVE_DIR" + +# Raise the V8 heap limit and serialize SSR to keep peak memory manageable. +# The server compile needs well above the default heap; run this on a machine +# (or runner) with enough RAM to back the limit below, or provide swap. +export NODE_OPTIONS="--max-old-space-size=20480" +export DOCUSAURUS_SSR_CONCURRENCY=1 + +echo ">> Building docs archive in $ARCHIVE_DIR" +rm -rf build .docusaurus +npm run build +echo ">> Build complete -> $ARCHIVE_DIR/build" diff --git a/docs-archive/scripts/deploy/package-prebuilt.sh b/docs-archive/scripts/deploy/package-prebuilt.sh new file mode 100755 index 00000000..178422ba --- /dev/null +++ b/docs-archive/scripts/deploy/package-prebuilt.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# +# Wrap the finished Docusaurus build/ into Vercel's Build Output API (v3) +# layout at docs-archive/.vercel/output/, so that: +# +# npx vercel deploy --prebuilt --prod +# +# uploads the already-built site without Vercel rebuilding it from source. +# +# Run build-archive.sh first to produce build/. +# +set -euo pipefail + +ARCHIVE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BUILD_DIR="$ARCHIVE_DIR/build" +OUTPUT_DIR="$ARCHIVE_DIR/.vercel/output" + +if [ ! -d "$BUILD_DIR" ] || [ ! -f "$BUILD_DIR/index.html" ]; then + echo "ERROR: no build found at $BUILD_DIR — run scripts/deploy/build-archive.sh first." >&2 + exit 1 +fi + +echo ">> Packaging $BUILD_DIR -> $OUTPUT_DIR" +rm -rf "$OUTPUT_DIR" +mkdir -p "$OUTPUT_DIR/static" + +# Copy the entire static site into the Build Output API's static/ folder. +cp -R "$BUILD_DIR"/. "$OUTPUT_DIR/static/" + +# Minimal valid config for a purely static site. Vercel serves static/ as-is +# and uses the generated 404.html for not-found routes automatically. +cat > "$OUTPUT_DIR/config.json" <<'JSON' +{ + "version": 3 +} +JSON + +echo ">> Prebuilt output ready at $OUTPUT_DIR" +echo ">> Deploy with: (cd $ARCHIVE_DIR && npx vercel deploy --prebuilt --prod --yes --token=\$VERCEL_TOKEN)" diff --git a/docs-archive/versioned_docs/version-1.5.0/AGENTS.md b/docs-archive/versioned_docs/version-1.5.0/AGENTS.md new file mode 100644 index 00000000..2d74ac3d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/AGENTS.md @@ -0,0 +1,603 @@ +--- +title: DataHub — Agent Development Guide +sidebar_label: — Agent Development Guide +slug: /agents +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/AGENTS.md' +--- +# DataHub — Agent Development Guide + +This is the canonical reference for working with the DataHub codebase. It applies to all coding +agents (Claude Code, Cursor, Codex CLI, Devin, etc.) and human developers alike. + +## Essential Commands + +**Build and test:** + +```bash +./gradlew build # Build entire project +./gradlew check # Run all tests and linting +./gradlew format # Format all code (Java, Markdown, GraphQL, YAML) + +# Note that each directory typically has a build.gradle file, but the available tasks follow similar conventions. + +# Java code. +./gradlew spotlessApply # Java code formatting + +# Python code. +./gradlew :metadata-ingestion:testQuick # Fast Python unit tests +./gradlew :metadata-ingestion:lint # Python linting (ruff, mypy) +./gradlew :metadata-ingestion:lintFix # Python linting auto-fix (ruff only) + +# Markdown, GraphQL, YAML formatting +./gradlew :datahub-web-react:mdPrettierWrite # Format markdown files +./gradlew :datahub-web-react:graphqlPrettierWrite # Format GraphQL schemas +./gradlew :datahub-web-react:githubActionsPrettierWrite # Format GitHub Actions +``` + +**IMPORTANT: Verifying Python code changes:** + +- **ALWAYS use `./gradlew :metadata-ingestion:lintFix`** to verify Python code changes +- **NEVER use `python3 -m py_compile`** - it doesn't catch style issues or type errors +- **NEVER use `ruff` or `mypy` commands directly** - use the Gradle task instead +- lintFix runs ruff formatting and fixing automatically, ensuring code quality +- For smoke-test changes, the lintFix command will also check those files + +## Code Formatting and Linting + +**CRITICAL: Always use Gradle tasks for formatting and linting. Never use npm/yarn/npx commands directly.** + +### Available Formatting Tasks + +**Format everything:** + +```bash +./gradlew format # Format all code (Java, Markdown, GraphQL, YAML) +./gradlew formatChanged # Format only changed files (faster) +``` + +**Format specific file types:** + +```bash +# Markdown files +./gradlew :datahub-web-react:mdPrettierWrite # Format all markdown +./gradlew :datahub-web-react:mdPrettierCheck # Check markdown formatting + +# GraphQL schemas +./gradlew :datahub-web-react:graphqlPrettierWrite # Format GraphQL files +./gradlew :datahub-web-react:graphqlPrettierCheck # Check GraphQL formatting + +# GitHub Actions YAML +./gradlew :datahub-web-react:githubActionsPrettierWrite # Format workflow files +./gradlew :datahub-web-react:githubActionsPrettierCheck # Check workflow files + +# Java code +./gradlew spotlessApply # Format Java code + +# Python code +./gradlew :metadata-ingestion:lintFix # Format and fix Python code +./gradlew :metadata-ingestion:lint # Check Python formatting +``` + +### When CI Formatting Checks Fail + +If you see CI failures like: + +- `markdown_format / markdown_format_check (pull_request)` - Use `./gradlew :datahub-web-react:mdPrettierWrite` +- `graphql_prettier_check` - Use `./gradlew :datahub-web-react:graphqlPrettierWrite` +- `spotlessJavaCheck` - Use `./gradlew spotlessApply` +- Python linting failures - Use `./gradlew :metadata-ingestion:lintFix` + +**Never do this:** + +```bash +npx prettier --write "docs/**/*.md" # WRONG - bypasses Gradle +yarn prettier --write # WRONG - bypasses Gradle +npm run format # WRONG - bypasses Gradle +``` + +**Always do this:** + +```bash +./gradlew :datahub-web-react:mdPrettierWrite # CORRECT - uses Gradle +./gradlew format # CORRECT - formats everything +``` + +### Why Use Gradle Tasks? + +1. **Consistent configuration**: Gradle tasks use the project's Prettier config +2. **Pre-commit hook integration**: Gradle tasks match what CI runs +3. **Dependency management**: Ensures correct tool versions +4. **Cross-platform**: Works reliably across all environments + +**Java SDK v2 integration tests:** + +See [metadata-integration/java/datahub-client/CLAUDE.md](metadata-integration/java/datahub-client/CLAUDE.md) for detailed integration test documentation. + +## Architecture Overview + +DataHub is a **schema-first, event-driven metadata platform** with three core layers: + +### Core Services + +- **GMS (Generalized Metadata Service)**: Java/Spring backend handling metadata storage and REST/GraphQL APIs +- **Frontend**: React/TypeScript application consuming GraphQL APIs +- **Ingestion Framework**: Python CLI and connectors for extracting metadata from data sources +- **Event Streaming**: Kafka-based real-time metadata change propagation + +### Key Modules + +- `metadata-models/`: Avro/PDL schemas defining the metadata model +- `metadata-service/`: Backend services, APIs, and business logic +- `datahub-web-react/`: Frontend React application +- `metadata-ingestion/`: Python ingestion framework and CLI +- `datahub-graphql-core/`: GraphQL schema and resolvers + +Most of the non-frontend modules are written in Java. The modules written in Python are: + +- `metadata-ingestion/` +- `datahub-actions/` +- `metadata-ingestion-modules/airflow-plugin/` +- `metadata-ingestion-modules/gx-plugin/` +- `metadata-ingestion-modules/dagster-plugin/` +- `metadata-ingestion-modules/prefect-plugin/` + +Each Python module has a gradle setup similar to `metadata-ingestion/` (documented above) + +### Metadata Model Concepts + +- **Entities**: Core objects (Dataset, Dashboard, Chart, CorpUser, etc.) +- **Aspects**: Metadata facets (Ownership, Schema, Documentation, etc.) +- **URNs**: Unique identifiers (`urn:li:dataset:(urn:li:dataPlatform:mysql,db.table,PROD)`) +- **MCE/MCL**: Metadata Change Events/Logs for updates +- **Entity Registry**: YAML config defining entity-aspect relationships (`metadata-models/src/main/resources/entity-registry.yml`) + +### Validation Architecture + +**IMPORTANT**: Validation must work across all APIs (GraphQL, OpenAPI, RestLI). + +- **Never add validation in API-specific layers** (GraphQL resolvers, REST controllers) - this only protects one API +- **Always implement AspectPayloadValidators** in `metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/` +- **Register as Spring beans** in `SpringStandardPluginConfiguration.java` +- **Follow existing patterns**: See `SystemPolicyValidator.java` and `PolicyFieldTypeValidator.java` as examples + +## Development Flow + +1. **Schema changes** in `metadata-models/` trigger code generation across all languages +2. **Backend changes** in `metadata-service/` and other Java modules expose new REST/GraphQL APIs +3. **Frontend changes** in `datahub-web-react/` consume GraphQL APIs +4. **Ingestion changes** in `metadata-ingestion/` emit metadata to backend APIs + +## Working on Docs + +The docs site is a **Docusaurus 2** app in `docs-website/`. It runs on **port 3001** (not 3000, to avoid +conflicting with the frontend dev server). + +### Quick start + +```bash +scripts/dev/datahub-dev.sh docs # fast start (assumes prior build) +scripts/dev/datahub-dev.sh docs --build # full rebuild (runs docGen + yarnGenerate first) +``` + +Or via Gradle directly: `./gradlew :docs-website:yarnStart` (always does a full build). + +### How the docs site is assembled + +The final site is served from `docs-website/genDocs/` (gitignored). It is assembled at build time +from multiple hand-authored sources plus several generation steps: + +1. **Gradle generation tasks** produce `docs/generated/` (connector docs, entity reference, schemas) +2. **`generateDocsDir.ts`** discovers all markdown in the repo, applies transformations (frontmatter, + link rewriting, `{{ inline }}` directives), and writes the result to `genDocs/` +3. **Docusaurus** serves from `genDocs/`, additionally generating GraphQL API docs and Python SDK docs + +See `docs-website/AGENTS.md` for full pipeline details. + +### Where docs live + +| Path | What to edit | Detail guide | +| ---------------------------------------------- | ----------------------------------------------- | ------------------------------------------- | +| `docs/` | Hand-authored feature guides, API docs, how-tos | _(this section)_ | +| `metadata-ingestion/docs/sources//` | Connector docs (`*_pre.md`, `*_post.md`, etc.) | `metadata-ingestion/docs/sources/AGENTS.md` | +| `metadata-models/docs/entities/` | Entity descriptions (input to `modelDocGen`) | `metadata-models/docs/AGENTS.md` | +| `docs-website/src/pages/` | Custom React pages (e.g. `/integrations`) | `docs-website/AGENTS.md` | +| `docs-website/src/learn/` | Blog / learning articles (served at `/learn`) | `docs-website/AGENTS.md` | +| `docs-website/sidebars.js` | Sidebar navigation tree | `docs-website/AGENTS.md` | +| `docs-website/static/` | Images, logos, static assets | `docs-website/AGENTS.md` | +| `docs/generated/` | **Never edit** — auto-generated | | +| `docs-website/genDocs/` | **Never edit** — assembled output | | + +### Adding or editing a hand-authored doc + +1. Create/edit the markdown file in `docs/` +2. Add an entry in `docs-website/sidebars.js` (the doc ID is the file path minus `.md`) +3. Run `scripts/dev/datahub-dev.sh docs` to preview + +If `sidebars.js` is missing the entry, the build will warn about an unaccounted file. + +## Code Standards + +### General Principles + +- This is production code - maintain high quality +- Follow existing patterns within each module +- Generate appropriate unit tests +- Use type annotations everywhere (Python/TypeScript) + +### Language-Specific + +- **Java**: Use Spotless formatting, Spring Boot patterns, TestNG/JUnit Jupiter for tests +- **Python**: Use ruff for linting/formatting, pytest for testing, pydantic for configs + - **Type Safety**: Everything must have type annotations, avoid `Any` type, use specific types (`Dict[str, int]`, `TypedDict`) + - **Data Structures**: Prefer dataclasses/pydantic for internal data, return dataclasses over tuples + - **Code Quality**: Avoid global state, use named arguments, don't re-export in `__init__.py`, refactor repetitive code + - **Error Handling**: Robust error handling with layers of protection for known failure points +- **TypeScript**: Use Prettier formatting, strict types (no `any`), React Testing Library + +### Frontend Theming (Colors) + +**Always use semantic color tokens** from `datahub-web-react/src/conf/theme/colorThemes/types.ts`. Never use hardcoded hex values, `REDESIGN_COLORS`, `ANTD_GRAY`, or direct alchemy `colors.gray[X]` imports. + +**In styled-components** (no import needed — `theme` is available via props): + +```typescript +background: ${(props) => props.theme.colors.bg}; +color: ${(props) => props.theme.colors.text}; +border: 1px solid ${(props) => props.theme.colors.border}; +``` + +**In React component bodies:** + +```typescript +import { useTheme } from 'styled-components'; +const theme = useTheme(); + +``` + +**For alchemy components** (``, ``, etc.) — do not pass `color`/`colorLevel` props. Let them inherit from themed parent styled-components. + +**Do not import from:** + +- `datahub-web-react/src/alchemy-components/theme/foundations/colors.ts` (raw palette, only used internally by the theme) +- `REDESIGN_COLORS` or `ANTD_GRAY` from `entityV2/shared/constants.ts` + +### Code Comments + +Only add comments that provide real value beyond what the code already expresses. + +**Do NOT** add comments for: + +- Obvious operations (`# Get user by ID`, `// Create connection`) +- What the code does when it's self-evident (`# Loop through items`, `// Set variable to true`) +- Restating parameter names or return types already in signatures +- Basic language constructs (`# Import modules`, `// End of function`) + +**DO** add comments for: + +- **Why** something is done, especially non-obvious business logic or workarounds +- **Context** about external constraints, API quirks, or domain knowledge +- **Warnings** about gotchas, performance implications, or side effects +- **References** to tickets, RFCs, or external documentation that explain decisions +- **Complex algorithms** or mathematical formulas that aren't immediately clear +- **Temporary solutions** with TODOs and context for future improvements + +Examples: + +```python +# Good: Explains WHY and provides context +# Use a 30-second timeout because Snowflake's query API can hang indefinitely +# on large result sets. See issue #12345. +connection_timeout = 30 + +# Bad: Restates what's obvious from code +# Set connection timeout to 30 seconds +connection_timeout = 30 +``` + +### Testing Strategy + +- Python: Tests go in the `tests/` directory alongside `src/`, use `assert` statements +- Java: Tests alongside source in `src/test/` +- Frontend: Tests in `__tests__/` or `.test.tsx` files +- Smoke tests go in the `smoke-test/` directory + +#### Testing Principles: Focus on Value Over Coverage + +**IMPORTANT**: Quality over quantity. Avoid AI-generated test anti-patterns that create maintenance burden without providing real value. + +**Focus on behavior, not implementation**: + +- Test what the code does (business logic, edge cases that occur in production) +- Don't test how it does it (implementation details, private fields via reflection) +- Don't test third-party libraries work correctly (Spring, Micrometer, Kafka clients, etc.) +- Don't test Java/Python language features (`synchronized` methods are thread-safe, `@Nonnull` parameters reject nulls) + +**Avoid these specific anti-patterns**: + +- Testing null inputs on `@Nonnull`/`@NonNull` annotated parameters +- Verifying exact error message wording (creates brittleness during refactoring) +- Testing every possible input variation (case sensitivity x whitespace x special chars = maintenance nightmare) +- Using reflection to verify private implementation details +- Redundant concurrency testing on `synchronized` methods +- Testing obvious getter/setter behavior without business logic +- Testing Lombok-generated code (`@Data`, `@Builder`, `@Value` classes) - you're testing Lombok's code generator, not your logic +- Testing that annotations exist on classes - if required annotations are missing, the framework/compiler will fail at startup, not in your tests + +**Appropriate test scope**: + +- **Simple utilities** (enums, string parsing, formatters): ~50-100 lines of focused tests + - Happy path for each method + - One example of invalid input per method + - Edge cases likely to occur in production +- **Complex business logic**: Test proportional to risk and complexity + - Integration points and system boundaries + - Security-critical operations + - Error handling for realistic failure scenarios +- **Warning sign**: If tests are 5x+ the size of implementation, reconsider scope + +**Examples of low-value tests to avoid**: + +```java +// BAD: Testing @Nonnull contract (framework's job) +@Test +public void testNullParameterThrowsException() { + assertThrows(NullPointerException.class, + () -> service.process(null)); // parameter is @Nonnull +} + +// BAD: Testing Lombok-generated code +@Test +public void testBuilderSetsAllFields() { + MyConfig config = MyConfig.builder() + .field1("value1") + .field2("value2") + .build(); + assertEquals(config.getField1(), "value1"); + assertEquals(config.getField2(), "value2"); +} + +// BAD: Testing that annotations exist +@Test +public void testConfigurationAnnotations() { + assertNotNull(MyConfig.class.getAnnotation(Configuration.class)); + assertNotNull(MyConfig.class.getAnnotation(ComponentScan.class)); +} +// If @Configuration is missing, Spring won't load the context - you don't need a test for this + +// BAD: Exact error message (brittle) +assertEquals(exception.getMessage(), + "Unsupported database type 'oracle'. Only PostgreSQL and MySQL variants are supported."); + +// BAD: Redundant variations +assertEquals(DatabaseType.fromString("postgresql"), DatabaseType.POSTGRES); +assertEquals(DatabaseType.fromString("PostgreSQL"), DatabaseType.POSTGRES); +assertEquals(DatabaseType.fromString("POSTGRESQL"), DatabaseType.POSTGRES); +assertEquals(DatabaseType.fromString(" postgresql "), DatabaseType.POSTGRES); +// ... 10 more case/whitespace variations + +// GOOD: Focused behavioral test +@Test +public void testFromString_ValidInputsCaseInsensitive() { + assertEquals(DatabaseType.fromString("postgresql"), DatabaseType.POSTGRES); + assertEquals(DatabaseType.fromString("POSTGRESQL"), DatabaseType.POSTGRES); + assertEquals(DatabaseType.fromString(" postgresql "), DatabaseType.POSTGRES); +} + +@Test +public void testFromString_InvalidInputThrows() { + assertThrows(IllegalArgumentException.class, + () -> DatabaseType.fromString("oracle")); +} + +// GOOD: Testing YOUR custom validation logic on a Lombok class +@Test +public void testCustomValidation() { + assertThrows(IllegalArgumentException.class, + () -> MyConfig.builder().field1("invalid").build().validate()); +} +``` + +**When in doubt**: Ask "Does this test protect against a realistic regression?" If not, skip it. + +#### Security Testing: Configuration Property Classification + +**Critical test**: `metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java` + +This test prevents sensitive data leaks by requiring explicit classification of all configuration properties as either sensitive (redacted) or non-sensitive (visible in system info). + +**When adding new configuration properties**: The test will fail with clear instructions on which classification list to add your property to. Refer to the test file's comprehensive documentation for template syntax and examples. + +This is a mandatory security guardrail - never disable or skip this test. + +### Commits + +- Follow Conventional Commits format for commit messages +- Breaking Changes: Always update `docs/how/updating-datahub.md` for breaking changes. Write entries for non-technical audiences, reference the PR number, and focus on what users need to change rather than internal implementation details + +### Pull Requests + +When creating PRs, follow the template in `.github/pull_request_template.md`: + +**PR Title Format** (from [Contributing Guide](docs/CONTRIBUTING.md#pr-title-format)): + +``` +[optional scope]: +``` + +Types: `feat`, `fix`, `refactor`, `docs`, `test`, `perf`, `style`, `build`, `ci` + +Example: `feat(parser): add ability to parse arrays` + +**Checklist** (verify before submitting): + +- [ ] PR conforms to the Contributing Guideline (especially PR Title Format) +- [ ] Links to related issues (if applicable) +- [ ] Tests added/updated (if applicable) +- [ ] Docs added/updated (if applicable) +- [ ] Breaking changes documented in `docs/how/updating-datahub.md` + +## Starting / Operating DataHub + +Use `scripts/dev/datahub-dev.sh` for **ALL** environment operations. +**Do NOT use `./gradlew quickstartDebug` directly** — always use the wrapper script. + +### `datahub-dev` CLI Tool + +A stdlib-only Python CLI for agent-driven development. No venv needed — runs with system `python3`. + +**Always use the shell wrapper as the entry point:** + +```bash +scripts/dev/datahub-dev.sh +``` + +Run `scripts/dev/datahub-dev.sh --help` to see all available subcommands (`start`, `stop`, `setup`, +`frontend`, `docs`, `status`, `wait`, `rebuild`, `test`, `flag list/get`, `env`, `sync-flags`, `reset`, `nuke`). + +### End-to-End Workflow + +0. **Setup** (once): `scripts/dev/datahub-dev.sh setup` — installs Python dev environment (provides `datahub` CLI). For frontend work, also run `scripts/dev/datahub-dev.sh setup frontend`. +1. **Start**: `scripts/dev/datahub-dev.sh start` +2. **Code**: Make changes to Java/Python/frontend code +3. **Rebuild**: `scripts/dev/datahub-dev.sh rebuild --wait` +4. **Test**: `scripts/dev/datahub-dev.sh test ` +5. **Iterate**: Repeat steps 2–4 + +**Frontend hot-reload:** Run `scripts/dev/datahub-dev.sh frontend` to start the React dev server with hot-reload (instead of rebuilding the frontend container). + +### Module-to-Container Mapping + +| Source directory | Container | +| --------------------------------- | --------------------------------------------- | +| `metadata-service/` | `datahub-gms` | +| `datahub-graphql-core/` | `datahub-gms` | +| `metadata-io/` | `datahub-gms` | +| `datahub-frontend/` | `datahub-frontend-react` | +| `metadata-jobs/mce-consumer-job/` | `datahub-mce-consumer` | +| `metadata-jobs/mae-consumer-job/` | `datahub-mae-consumer` | +| `metadata-models/` | All (triggers full rebuild + code generation) | + +### Environment Variables + +Set any env var for DataHub containers via `env set` + `env restart`: + +```bash +scripts/dev/datahub-dev.sh env set KEY=VALUE +scripts/dev/datahub-dev.sh env restart # required — changes take effect on restart +scripts/dev/datahub-dev.sh env list # show current vars and pending_restart status +``` + +**Do NOT** manually edit `.env` files, use `docker compose -e`, or `export` — always use the wrapper. + +### Feature Flag Lifecycle + +**All flag changes require a container restart.** Use `env set` + `env restart`: + +```bash +scripts/dev/datahub-dev.sh env set SHOW_BROWSE_V2=true +scripts/dev/datahub-dev.sh env restart +``` + +`flag list` and `flag get` are read-only inspection tools — they show the current live values from +the running server but do not change anything. + +The flag manifest at `scripts/generated/flag-classification.json` is **auto-generated** +(gitignored). Run `scripts/dev/datahub-dev.sh sync-flags` after adding fields to `FeatureFlags.java` +or after a fresh clone. + +### Stopping DataHub + +`scripts/dev/datahub-dev.sh stop` shuts down all containers without restarting. + +When starting, `datahub-dev start` automatically detects and stops conflicting DataHub instances +from other worktrees/compose projects that occupy the same ports. + +### Recovery Escalation + +**When to use each:** + +- `stop`: Just shut down DataHub — no restart, no data loss +- `reset`: GMS returns 503 and doesn't recover, frontend shows "Unable to connect", tests fail + with connection errors +- `nuke --keep-data`: Containers in restart loops, port conflicts, `reset` didn't fix it +- `nuke`: ES index corruption, MySQL schema issues after model changes, PDL model changes needing + clean slate, `nuke --keep-data` didn't fix it + +### Structured Test Output + +Set `AGENT_MODE=1` to get machine-readable JSON test reports at `smoke-test/build/test-report.json`: + +```bash +AGENT_MODE=1 scripts/dev/datahub-dev.sh test tests/test_system_info.py +``` + +## Common Operations + +These commands work against **any** DataHub instance — local dev, staging, or production. +Provide connection details via environment variables: + +```bash +export DATAHUB_GMS_URL=http://localhost:8080 # or your instance URL +export DATAHUB_GMS_TOKEN= # omit if auth is not required +``` + +### Init (setup authentication) + +`datahub init` writes `~/.datahubenv` with the GMS URL and an access token. Run it once before +using any other CLI commands that require authentication. + +```bash +# Quickstart: local instance with default credentials +datahub init --username datahub --password datahub + +# Full agent best-practices guide (defaults, env vars, all scenarios) +datahub init --agent-context +``` + +### GraphQL + +`datahub graphql` executes queries and mutations against the DataHub GraphQL API and can +introspect the live schema to discover available operations. + +```bash +# Discover what's available +datahub graphql --list-operations --format json + +# Inspect a specific operation's arguments +datahub graphql --describe dataset --format json + +# Preview a query before executing +datahub graphql --query "{ me { corpUser { urn } } }" --dry-run + +# Execute a query +datahub graphql --query "{ me { corpUser { urn username } } }" --format json +``` + +For full agent best practices (discovery, dry-run, error codes, common recipes): + +```bash +datahub graphql --agent-context +``` + +## Key Documentation + +**Essential reading:** + +- `docs/architecture/architecture.md` - System architecture overview +- `docs/modeling/metadata-model.md` - How metadata is modeled +- `docs/what-is-datahub/datahub-concepts.md` - Core concepts (URNs, entities, etc.) + +**External docs:** + +- https://docs.datahub.com/docs/developers - Official developer guide +- https://demo.datahub.com/ - Live demo environment + +## Python Virtual Environments + +Gradle tasks manage all venvs automatically. Never create, activate, or pip-install into them manually. When running smoke tests outside Gradle: `smoke-test/venv/bin/python -m pytest ...` + +## Important Notes + +- Entity Registry is defined in YAML, not code (`entity-registry.yml`) +- All metadata changes flow through the event streaming system +- GraphQL schema is generated from backend GMS APIs diff --git a/docs-archive/versioned_docs/version-1.5.0/CLAUDE.md b/docs-archive/versioned_docs/version-1.5.0/CLAUDE.md new file mode 100644 index 00000000..7133a226 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/CLAUDE.md @@ -0,0 +1,8 @@ +--- +title: CLAUDE.md +slug: /claude +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/CLAUDE.md' +--- +# CLAUDE.md + +@AGENTS.md diff --git a/docs-archive/versioned_docs/version-1.5.0/README.md b/docs-archive/versioned_docs/version-1.5.0/README.md new file mode 100644 index 00000000..91fed2ab --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/README.md @@ -0,0 +1,891 @@ +--- +description: >- + DataHub is a data discovery application built on an extensible metadata + platform that helps you tame the complexity of diverse data ecosystems. +hide_title: true +title: Introduction +slug: /introduction +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/README.md' +--- + +import useBaseUrl from '@docusaurus/useBaseUrl'; + +export const Logo = (props) => { + return ( +
+ DataHub Logo +
+ ); +}; + + + + + +# The #1 Open Source AI Data Catalog + +_Enterprise-grade metadata platform enabling discovery, governance, and observability across your entire data ecosystem_ + +

+ + Build Status + + + PyPI Version + + + PyPI Downloads + + + Docker Pulls + +
+ + Join Slack + + + YouTube Subscribers + + + DataHub Blog + + + Contributors + + + GitHub Stars + + + License + +

+ +

+ Quick Start • + Live Demo • + Documentation • + Roadmap • + Slack Community • + YouTube +

+ +

+ Built with ❤️ by DataHub and LinkedIn +

+ +--- + +

+ + DataHub Product Tour + +

+ +

+ Search, discover, and understand your data with DataHub's unified metadata platform +

+ +--- + +### 🤖 **NEW: Connect AI Agents to DataHub via Model Context Protocol (MCP)** + +

+ + DataHub MCP Demo - Query metadata with AI agents + +
+ ▶️ Click to watch full demo on YouTube +

+ +Connect your AI coding assistants (Cursor, Claude Desktop, Cline) directly to DataHub. +Query metadata with natural language: _"What datasets contain PII?"_ or _"Show me lineage for this table"_ + +**Quick setup:** + +```bash +npx -y @acryldata/mcp-server-datahub init +``` + +[Learn more →](https://github.com/acryldata/mcp-server-datahub) + +--- + +## What is DataHub? + +> **🔍 Finding the right DataHub?** This is the **open-source metadata platform** at [datahub.com](https://datahub.com) (GitHub: [datahub-project/datahub](https://github.com/datahub-project/datahub)). It was previously hosted at `datahubproject.io`, which now redirects to [datahub.com](https://datahub.com). This project is **not related to** [datahub.io](https://datahub.io), which is a separate public dataset hosting service. See the [FAQ](#-frequently-asked-questions) below. + +**DataHub is the #1 open-source AI data catalog** that enables discovery, governance, and observability across your entire data ecosystem. Originally built at LinkedIn, DataHub now powers data discovery at thousands of organizations worldwide, managing millions of data assets. + +**The Challenge:** Modern data stacks are fragmented across dozens of tools—warehouses, lakes, BI platforms, ML systems, AI agents, orchestration engines. Finding the right data, understanding its lineage, and ensuring governance is like searching through a maze blindfolded. + +**The DataHub Solution:** DataHub acts as the central nervous system for your data stack—connecting all your tools through real-time streaming or batch ingestion to create a unified metadata graph. Unlike static catalogs, DataHub keeps your metadata fresh and actionable—powering both human teams and AI agents. + +![DataHub for Humans and AI](https://raw.githubusercontent.com/datahub-project/static-assets/refs/heads/main/imgs/datahub_for_human_and_ai.png) + +### Why DataHub? + +- **🚀 Battle-Tested at Scale:** Born at LinkedIn to handle hyperscale data, now proven at thousands of organizations worldwide managing millions of data assets +- **⚡ Real-Time Streaming:** Metadata updates in seconds, not hours or days +- **🤖 AI-Ready:** Native support for AI agents via MCP, LLM integrations, and context management +- **🔌 Pioneering Ingestion Architecture:** Flexible push/pull framework (widely adopted by other catalogs) with 80+ production-grade connectors extracting deep metadata—column lineage, usage stats, profiling, and quality metrics +- **👨‍💻 Developer-First:** Rich APIs (GraphQL, OpenAPI), Python + Java SDKs, CLI tools +- **🏢 Enterprise Ready:** Battle-tested security, authentication, authorization, and audit trails +- **🌍 Open Source:** Apache 2.0 licensed, vendor-neutral, community-driven + +--- + +## 🧠 The Context Foundation + +Essential for modern data teams and reliable AI agents: + +- **[Context Management Is the Missing Piece in the Agentic AI Puzzle](https://datahub.com/blog/context-management-is-the-missing-piece-in-the-agentic-ai-puzzle/)** - Why context management is essential for deploying reliable AI agents at scale +- **[Data Lineage: What It Is and Why It Matters](https://datahub.com/blog/data-lineage-what-it-is-and-why-it-matters/)** - Understanding the map of how data flows through your organization +- **[What is Metadata Management?](https://datahub.com/blog/what-is-metadata-management/)** - A comprehensive guide for enterprise data leaders + +--- + +## 📑 Table of Contents + +- [FAQ](#-frequently-asked-questions) +- [See DataHub in Action](#-see-datahub-in-action) +- [Quick Start](#-quick-start-60-seconds) +- [Installation Options](#-installation-options) +- [Architecture](#-architecture-overview) +- [Use Cases & Examples](#-use-cases--examples) +- [Trusted By](#-trusted-by-industry-leaders) +- [Ecosystem](#-datahub-ecosystem) +- [Community](#-community--support) +- [Contributing](#-contributing) +- [Resources](#-resources--learning) +- [License](#-license) + +--- + +## ❓ Frequently Asked Questions + +
+Is this the same project as datahub.io? + +No. [datahub.io](https://datahub.io) is a completely separate project — a public dataset hosting service with no affiliation to this project. DataHub (this project) is an open-source metadata platform for data discovery, governance, and observability, hosted at [datahub.com](https://datahub.com) and developed at [github.com/datahub-project/datahub](https://github.com/datahub-project/datahub). + +
+ +
+What happened to datahubproject.io? + +DataHub was previously hosted at `datahubproject.io`. That domain now redirects to [datahub.com](https://datahub.com). All documentation has moved to [docs.datahub.com](/docs/quickstart). If you find references to `datahubproject.io` in blog posts or tutorials, they refer to this same project — just under its former domain. + +
+ +
+Is DataHub related to LinkedIn's internal DataHub? + +Yes. DataHub was originally built at LinkedIn to manage metadata at scale across their data ecosystem. LinkedIn open-sourced DataHub in 2020. It has since grown into an independent community project under the [datahub-project](https://github.com/datahub-project) GitHub organization, now hosted at [datahub.com](https://datahub.com). + +
+ +
+How do I install the DataHub metadata platform? + +```bash +pip install acryl-datahub +datahub docker quickstart +``` + +See the [Quick Start](#-quick-start-60-seconds) section below for full instructions. The PyPI package is [`acryl-datahub`](https://pypi.org/project/acryl-datahub/). + +
+ +--- + +## 🎨 See DataHub in Action + + + + + + + + + + +
+ Universal Search +

🔍 Universal Search
Find any data asset instantly across your entire stack

+ 		+ Column-Level Lineage +

📊 Column-Level Lineage
Trace data flow from source to consumption

+
+ Rich Dataset Profiles +

📋 Rich Dataset Profiles
Schema, statistics, documentation, and ownership

+ 		+ Governance Dashboard +

🏛️ Governance Dashboard
Manage policies, tags, and compliance

+
+ +**▶️ Watch DataHub in Action:** + +- [5-Minute Product Tour](https://www.youtube.com/channel/UC3qFQC5IiwR5fvWEqi_tJ5w) (YouTube) +- [Try Live Demo](https://demo.datahub.com) (No installation required) + +--- + +## 🚀 Quick Start + +### Option 1: Try the Hosted Demo (Fastest) + +No installation required. Explore a fully-loaded DataHub instance with sample data instantly: + +**🌐 [Launch Live Demo: demo.datahub.com](https://demo.datahub.com)** + +### Option 2: Run Locally with Python (Recommended) + +Get DataHub running on your machine in under 2 minutes: + +```bash +# Prerequisites: Docker Desktop with 8GB+ RAM allocated + +# Upgrade pip and install DataHub CLI +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub + +# Launch DataHub locally via Docker +datahub docker quickstart + +# Access DataHub at http://localhost:9002 +# Default credentials: datahub / datahub +``` + +**Note:** You can also use `uv` or other Python package managers instead of pip. + +**What's included:** + +- ✅ **Full Stack:** GMS backend, React UI, Elasticsearch, MySQL, and Kafka. +- ✅ **Sample Data:** Pre-loaded datasets, lineage, and owners for exploration. +- ✅ **Ingestion Ready:** Fully prepared to connect your own local or cloud data sources. + +### Option 3: Run from Source (For Contributors) + +Best for advanced users who want to modify the core codebase or run directly from the repository: + +```bash +# Clone the repository +git clone https://github.com/datahub-project/datahub.git +cd datahub + +# Start all services with docker-compose +./docker/quickstart.sh + +# Access DataHub at http://localhost:9002 +# Default credentials: datahub / datahub +``` + +### Next Steps + +- **🔌 Connect Your Data:** Explore our [Ingestion Guides](/docs/metadata-ingestion) for Snowflake, BigQuery, dbt, and more. +- **📚 Learn the Basics:** Walk through the [Getting Started Guide](/docs/quickstart) +- **🎓 DataHub Academy:** Deep dive with our [Advanced Tutorials](/docs/quickstart) + +--- + +## 📦 Installation Options + +DataHub supports three deployment models: + +- **[Managed SaaS (DataHub Cloud)](https://datahub.com/get-datahub-cloud/)** — zero infrastructure, SLA-backed, enterprise-ready +- **[Self-hosted via Docker](/docs/quickstart)** — ideal for development and small teams +- **[Kubernetes (Helm)](docs/deploy/kubernetes.md)** — recommended for production self-hosted deployments + +**→ [See all deployment guides (AWS, Azure, GCP, environment variables)](https://github.com/datahub-project/datahub/blob/master/docs/deploy/)** + +--- + +## 🏗️ Architecture Overview + +- ✅ **Streaming-First:** Real-time metadata updates via Kafka +- ✅ **API-First:** All features accessible via APIs +- ✅ **Extensible:** Plugin architecture for custom entity types +- ✅ **Scalable:** Proven to 10M+ assets and O(1B) relationships at LinkedIn and other companies in production +- ✅ **Cloud-Native:** Designed for Kubernetes deployment + +**→ [Full architecture breakdown: components, storage layer, APIs, and design decisions](docs/architecture/architecture.md)** + +--- + +## 💻 Use Cases & Examples + +
+Example 1: Ingest Metadata from Snowflake + +**Use Case:** Extract table metadata, column schemas, and usage statistics from Snowflake data warehouse. + +**Prerequisites:** + +- DataHub instance running (local or remote) +- Snowflake account with read permissions +- DataHub CLI installed (`pip install 'acryl-datahub[snowflake]'`) + +```yaml +# snowflake_recipe.yml +source: + type: snowflake + config: + # Connection details + account_id: "xy12345.us-east-1" + warehouse: "COMPUTE_WH" + username: "${SNOWFLAKE_USER}" + password: "${SNOWFLAKE_PASSWORD}" + + # Optional: Filter specific databases + database_pattern: + allow: + - "ANALYTICS_DB" + - "MARKETING_DB" + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" +``` + +```bash +# Run ingestion +datahub ingest -c snowflake_recipe.yml + +# Expected output: +# ✓ Connecting to Snowflake... +# ✓ Discovered 150 tables in ANALYTICS_DB +# ✓ Discovered 75 tables in MARKETING_DB +# ✓ Ingesting metadata... +# ✓ Successfully ingested 225 datasets to DataHub +``` + +**What gets ingested:** + +- Table and view schemas (columns, data types, descriptions) +- Table statistics (row counts, size, last modified) +- Lineage information (upstream/downstream tables) +- Usage statistics (query frequency, top users) + +
+ +--- + +
+Example 2: Search for Datasets via Python SDK + +**Use Case:** Programmatically search DataHub catalog and retrieve dataset metadata. + +**Prerequisites:** + +- DataHub instance accessible +- Python 3.8+ installed +- DataHub Python package installed (`pip install 'acryl-datahub[datahub-rest]'`) + +```python +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +# Initialize DataHub client +config = DatahubClientConfig(server="http://localhost:8080") +graph = DataHubGraph(config) + +# Search for datasets containing "customer" +# Returns up to 10 most relevant results +results = graph.search( + entity="dataset", + query="customer", + count=10 +) + +# Process and display results +for result in results: + print(f"Found: {result.entity.urn}") + print(f" Name: {result.entity.name}") + print(f" Platform: {result.entity.platform}") + print(f" Description: {result.entity.properties.description}") + print("---") + +# Example output: +# Found: urn:li:dataset:(urn:li:dataPlatform:snowflake,analytics.customer_profiles,PROD) +# Name: customer_profiles +# Platform: snowflake +# Description: Aggregated customer data from CRM and transactions +# --- +``` + +**Response format:** Each result contains: + +- `urn`: Unique resource identifier for the dataset +- `name`: Human-readable dataset name +- `platform`: Source platform (snowflake, bigquery, etc.) +- `properties`: Additional metadata (description, tags, owners, etc.) + +
+ +--- + +
+Example 3: Query Lineage via GraphQL + +**Use Case:** Retrieve upstream and downstream dependencies for a specific dataset. + +**Prerequisites:** + +- DataHub GMS endpoint accessible +- Dataset URN available from search or ingestion + +**GraphQL Query:** + +```graphql +query GetLineage { + dataset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,analytics.customer_profiles,PROD)" + ) { + # Get upstream dependencies (source tables) + upstream: lineage(input: { direction: UPSTREAM }) { + entities { + urn + ... on Dataset { + name + platform { + name + } + } + } + } + + # Get downstream dependencies (consuming tables/dashboards) + downstream: lineage(input: { direction: DOWNSTREAM }) { + entities { + urn + type + ... on Dataset { + name + platform { + name + } + } + ... on Dashboard { + dashboardId + tool + } + } + } + } +} +``` + +**Execute via cURL:** + +```bash +curl -X POST http://localhost:8080/api/graphql \ + -H "Content-Type: application/json" \ + -d '{"query": "query GetLineage { ... }"}' +``` + +**Response structure:** + +- `upstream`: Array of datasets that feed into this dataset +- `downstream`: Array of datasets, dashboards, or ML models that consume this dataset +- Each entity includes URN, type, and basic metadata + +
+ +--- + +
+Example 4: Add Documentation via Python API + +**Use Case:** Programmatically add or update dataset documentation and custom properties. + +**Prerequisites:** + +- DataHub Python SDK installed +- Write permissions to DataHub instance +- Dataset already exists in DataHub (from ingestion) + +```python +from datahub.metadata.schema_classes import DatasetPropertiesClass +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Create emitter to send metadata to DataHub +emitter = DatahubRestEmitter("http://localhost:8080") + +# Create dataset URN (unique identifier) +dataset_urn = make_dataset_urn( + platform="snowflake", + name="analytics.customer_profiles", + env="PROD" +) + +# Define dataset properties +properties = DatasetPropertiesClass( + description=""" + Customer profiles aggregated from CRM and transaction data. + + **Update Schedule:** Updated nightly via Airflow DAG `customer_profile_etl` + **Data Retention:** 7 years for compliance + **Owner:** Data Platform Team + """, + customProperties={ + "owner_team": "data-platform", + "update_frequency": "daily", + "data_sensitivity": "PII", + "upstream_dag": "customer_profile_etl", + "business_domain": "customer_analytics" + } +) + +# Emit metadata to DataHub +emitter.emit_mcp( + entityUrn=dataset_urn, + aspectName="datasetProperties", + aspect=properties +) + +print(f"✓ Successfully updated documentation for {dataset_urn}") +``` + +**What this does:** + +1. Adds rich markdown documentation visible in DataHub UI +2. Sets custom properties for governance and discovery +3. Makes dataset searchable by custom property values +4. Enables filtered searches (e.g., "show me all PII datasets") + +
+ +--- + +
+Example 5: Connect AI Coding Assistants via Model Context Protocol + +**Use Case:** Enable AI agents (Cursor, Claude Desktop, Cline) to query DataHub metadata directly from your IDE or development environment. + +**Prerequisites:** + +- DataHub instance running and accessible +- MCP-compatible AI tool installed (Cursor, Claude Desktop, Cline, etc.) +- Node.js 18+ installed + +**Quick Setup:** + +```bash +# Initialize MCP server for DataHub +npx -y @acryldata/mcp-server-datahub init + +# Follow the interactive prompts to configure: +# - DataHub GMS endpoint (e.g., http://localhost:8080) +# - Authentication token (if required) +# - MCP server settings +``` + +**Configure your AI tool:** + +For **Claude Desktop**, add to `~/Library/Application Support/Claude/claude_desktop_config.json`: + +```json +{ + "mcpServers": { + "datahub": { + "command": "npx", + "args": ["-y", "@acryldata/mcp-server-datahub"] + } + } +} +``` + +For **Cursor**, configure in Settings → Features → MCP Servers + +**What you can ask your AI:** + +- _"What datasets contain customer PII in production?"_ +- _"Show me the lineage for analytics.revenue_table"_ +- _"Who owns the 'Revenue Dashboard' in Looker?"_ +- _"Find all datasets in the marketing domain"_ +- _"What's the schema for user_events table?"_ +- _"List datasets tagged as 'critical' or 'sensitive'"_ + +**Example conversation:** + +``` +You: "What datasets are owned by the data-platform team?" + +AI: Based on DataHub metadata, here are the datasets owned by data-platform: +- urn:li:dataset:(urn:li:dataPlatform:snowflake,analytics.customer_profiles,PROD) + Name: customer_profiles + Platform: Snowflake + Description: Aggregated customer data from CRM and transactions + +- urn:li:dataset:(urn:li:dataPlatform:bigquery,marketing.campaign_performance,PROD) + Name: campaign_performance + Platform: BigQuery + Description: Marketing campaign metrics and ROI tracking + +[... more results] +``` + +**Benefits:** + +- ✅ Query metadata without leaving your IDE +- ✅ Natural language interface (no SQL/GraphQL needed) +- ✅ Real-time access to DataHub's metadata graph +- ✅ Understand data context while coding +- ✅ Discover relevant datasets for your task + +📖 **Full Documentation:** [MCP Server for DataHub](https://github.com/acryldata/mcp-server-datahub) + +
+ +--- + +### Common Use Cases + +| Use Case | Description | Learn More | +| ---------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------- | +| 🔍 **Data Discovery** | Help users find the right data for analytics and ML | [Guide](/docs/features) | +| 📊 **Impact Analysis** | Understand downstream impact before making changes | [Lineage Docs](/docs/features/feature-guides/lineage) | +| 🏛️ **Data Governance** | Enforce policies, classify PII, manage access | [Governance Guide](/docs/authorization/access-policies-guide) | +| 🔔 **Data Quality** | Monitor freshness, volumes, schema changes | [Quality Checks](/docs/api/tutorials/assertions) | +| 📚 **Documentation** | Centralize data documentation and knowledge | [Docs Features](/docs/features) | +| 👥 **Collaboration** | Foster data culture with discussions and ownership | [Collaboration](/docs/features) | + +--- + +## 📝 DataHub in Action + +Learn from teams using DataHub in production and get practical guidance: + + + + + + + +
+

🏆 Best Practices from the Field

+

Real-world metadata strategies from teams at Grab, Slack, and Checkout.com who manage data at scale.

+ Case Studies + 		+

📋 Data Contracts: How to Use Them

+

Practical guide to implementing data contracts between producers and consumers for quality and accountability.

+ Implementation Guide + 		+

🤖 How Block Powers AI Agents with DataHub

+

Real-world case study: scaling data governance and AI operations across 50+ platforms using MCP.

+ AI Case Study +
+ +

+ → Explore all posts on our blog +

+ +--- + +## 🏢 Trusted by Industry Leaders + +**3,000+ organizations** run DataHub in production worldwide — across both open-source deployments and DataHub Cloud — from hyperscale tech companies to regulated financial institutions and healthcare providers. + +### By Industry + +**🛒 E-Commerce & Retail:** Etsy • Experius • Klarna • LinkedIn • MediaMarkt Saturn • Uphold • Wealthsimple • Wolt + +**🏥 Healthcare & Life Sciences:** CVS Health • IOMED • Optum + +**✈️ Travel & Transportation:** Cabify • DFDS • Expedia Group • Hurb • Peloton • Viasat + +**📚 Education & EdTech:** ClassDojo • Coursera • Udemy + +**💰 Financial Services:** Banksalad • Block • Chime • FIS • Funding Circle • GEICO • Inter&Co • N26 • Santander • Shanghai HuaRui Bank • Stash • Visa + +**🎮 Gaming, Entertainment & Streaming:** Netflix • Razer • Showroomprive • TypeForm • UKEN Games • Zynga + +**🚀 Technology & SaaS:** Adevinta • Apple • Digital Turbine • DPG Media • Foursquare • Geotab • HashiCorp • hipages • inovex • KPN • Miro • MYOB • Notion • Okta • Rippling • Saxo Bank • Slack • ThoughtWorks • Twilio • Wikimedia • WP Engine + +**📊 Data & Analytics:** ABLY • DefinedCrowd • Grofers • Haibo Technology • Moloco • PITS Global Data Recovery Services • SpotHero + +_And thousands more across DataHub Core and DataHub Cloud._ + +### Featured Case Studies + +- 📰 **Optum:** [Data Mesh via DataHub](https://datahub.com/customer-stories/optum/) +- 🏦 **Saxo Bank:** [Enabling Data Discovery in Data Mesh](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f) +- 🚗 **SpotHero:** [Data Discoverability at Scale](https://www.slideshare.net/MaggieHays/data-discoverability-at-spothero) + +**Using DataHub?** Please feel free to add your organization to the list if we missed it — open a [PR](https://github.com/datahub-project/datahub/pulls) or let us know on [Slack](https://datahub.com/slack). + +--- + +## 🌐 DataHub Ecosystem + +DataHub is part of a rich ecosystem of tools and integrations. + +### Official Repositories + +| Repository | Description | Links | +| --------------------------------------------------------------------- | --------------------------------------------------------------- | ------------------------------------------------ | +| **[datahub](https://github.com/datahub-project/datahub)** | Core platform: metadata model, services, connectors, and web UI | [Docs](/docs/quickstart) | +| **[datahub-actions](https://github.com/acryldata/datahub-actions)** | Framework for responding to metadata changes in real-time | [Guide](/docs/actions) | +| **[datahub-helm](https://github.com/acryldata/datahub-helm)** | Production-ready Helm charts for Kubernetes deployment | [Charts](https://helm.datahubproject.io/) | +| **[static-assets](https://github.com/datahub-project/static-assets)** | Logos, images, and brand assets for DataHub | - | + +### Community Plugins & Integrations + +| Project | Description | Maintainer | +| ----------------------------------------------------------------------------------------------- | ------------------------------------------------ | ---------- | +| **[datahub-tools](https://github.com/makenotion/datahub-tools)** | Python tools for GraphQL endpoint interaction | Notion | +| **[dbt-impact-action](https://github.com/acryldata/dbt-impact-action)** | GitHub Action for dbt change impact analysis | Acryl Data | +| **[business-glossary-sync-action](https://github.com/acryldata/business-glossary-sync-action)** | Sync business glossary via GitHub PRs | Acryl Data | +| **[mcp-server-datahub](https://github.com/acryldata/mcp-server-datahub)** | Model Context Protocol server for AI integration | Acryl Data | +| **[meta-world](https://github.com/acryldata/meta-world)** | Recipes, custom sources, and transformations | Community | + +### Integrations by Category + +**📊 BI & Analytics:** Tableau • Looker • Power BI • Superset • Metabase • Mode • Redash + +**🗄️ Data Warehouses:** Snowflake • BigQuery • Redshift • Databricks • Synapse • ClickHouse + +**🔄 Data Orchestration:** Airflow • dbt • Dagster • Prefect • Luigi + +**🤖 ML Platforms:** SageMaker • MLflow • Feast • Kubeflow • Weights & Biases + +**🔗 Data Integration:** Fivetran • Airbyte • Stitch • Matillion + +[View all 80+ integrations →](/integrations) + +--- + +## 💬 Community & Support + +Join thousands of data practitioners building with DataHub! + +### 🗓️ Town Halls + +**Next Town Hall:** + +- 🎟️ [Register for the next Town Hall](https://luma.com/zp3h4ex8) + +**Last Town Hall:** + +- 📺 [Powering AI Agents with DataHub Context](https://youtu.be/dqZNV09yvA0?si=IWUKhLm0Xa_PoYsy) (January 2026) + +[→ View all past recordings](https://www.youtube.com/playlist?list=PLdCtLs64vZvHTXGqybmOfyxXbGDn2Reb9) + +### 💬 Get Help & Connect + +| Channel | Purpose | Link | +| ---------------------- | ---------------------------------------- | ---------------------------------------------------------------------------- | +| **Slack Community** | Real-time chat, questions, announcements | [Join 14,000+ members](https://datahub.com/slack) | +| **GitHub Discussions** | Technical discussions, feature requests | [Start a Discussion](https://github.com/datahub-project/datahub/discussions) | +| **GitHub Issues** | Bug reports, feature requests | [Open an Issue](https://github.com/datahub-project/datahub/issues) | +| **Stack Overflow** | Technical Q&A (tag: `datahub`) | [Ask a Question](https://stackoverflow.com/questions/tagged/datahub) | +| **YouTube** | Tutorials, demos, talks | [Subscribe](https://www.youtube.com/@datahubproject) | +| **LinkedIn** | Company updates, blogs | [Follow Us](https://linkedin.com/company/datahubproject) | +| **Twitter/X** | Quick updates, community highlights | [Follow @datahubproject](https://twitter.com/datahubproject) | + +### 📧 Stay Updated + +- 📝 [Read the Blog](https://datahub.com/blog/) - Deep dives and case studies +- 📖 [Monthly Release Notes](/docs/releases) - What's new + +### 🎓 Learning Resources + +- **[DataHub Quickstart](/docs/quickstart)** - Get started in 15 minutes +- **[API Documentation](/docs/api/datahub-apis)** - GraphQL & REST API reference +- **[Architecture Guide](/docs/architecture/architecture)** - Deep dive into internals +- **[Video Tutorials](https://www.youtube.com/@datahubproject)** - Step-by-step guides + +--- + +## 🤝 Contributing + +We ❤️ contributions from the community! See **[CONTRIBUTING.md](docs/CONTRIBUTING.md)** for setup, guidelines, and ways to get involved. + +Browse [Good First Issues](https://github.com/datahub-project/datahub/labels/good-first-issue) to get started! + +--- + +## 📚 Resources & Learning + +### 📰 Featured Content + +**Blog Posts & Articles:** + +- [DataHub: Popular Metadata Architectures Explained](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) - LinkedIn Engineering +- [Open Sourcing DataHub](https://engineering.linkedin.com/blog/2020/open-sourcing-datahub--linkedins-metadata-search-and-discovery-p) - LinkedIn Engineering +- [Enabling Data Discovery in Data Mesh](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f) - Saxo Bank +- [Data Discoverability at SpotHero](https://www.slideshare.net/MaggieHays/data-discoverability-at-spothero) - SpotHero +- [Emerging Architectures for Modern Data Infrastructure](https://future.com/emerging-architectures-for-modern-data-infrastructure-2020/) - a16z + +**Conference Talks:** + +- [The Evolution of Metadata: LinkedIn's Journey](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) - Strata 2019 +- [Driving DataOps Culture with DataHub](https://www.youtube.com/watch?v=ccsIKK9nVxk) - DataOps Unleashed 2021 +- [Journey of Metadata at LinkedIn](https://www.youtube.com/watch?v=OB-O0Y6OYDE) - Crunch Conference 2019 +- [DataHub Journey with Expedia Group](https://www.youtube.com/watch?v=ajcRdB22s5o) - Expedia + +**Podcasts:** + +- [Bringing The Power Of The Real-Time Metadata Graph To Everyone](https://www.dataengineeringpodcast.com/acryl-data-datahub-metadata-graph-episode-230/) - Data Engineering Podcast + +### 🔗 Important Links + +| Resource | URL | +| ------------------------- | -------------------------------------------------- | +| 📖 Official Documentation | https://docs.datahub.com | +| 🏠 Project Website | https://datahub.com | +| 🌐 Live Demo | https://demo.datahub.com | +| 📊 Roadmap | https://feature-requests.datahubproject.io/roadmap | +| 🗓️ Town Hall Schedule | https://docs.datahub.com/docs/townhalls | +| 💬 Slack Community | https://datahub.com/slack | +| 📺 YouTube Channel | https://youtube.com/@datahubproject | +| 📝 Blog | https://datahub.com/blog/ | +| 🔗 LinkedIn | https://www.linkedin.com/company/72009941 | +| 🐦 Twitter/X | https://twitter.com/datahubproject | +| 🔒 Security | https://docs.datahub.com/docs/security | + +--- + +## 📄 License + +DataHub is open source software released under the **[Apache License 2.0](https://github.com/datahub-project/datahub/blob/master/LICENSE)**. + +``` +Copyright 2015-2025 LinkedIn Corporation +Copyright 2025-Present DataHub Project Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +``` + +**What this means:** + +- ✅ Commercial use allowed +- ✅ Modification allowed +- ✅ Distribution allowed +- ✅ Patent use allowed +- ✅ Private use allowed + +**Learn more:** [Choose a License - Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) + +--- + +

+ ⭐ If you find DataHub useful, please star the repository! ⭐ +

+ +

+ Made with ❤️ by the DataHub community +

diff --git a/docs-archive/versioned_docs/version-1.5.0/SECURITY.md b/docs-archive/versioned_docs/version-1.5.0/SECURITY.md new file mode 100644 index 00000000..ffa26d3d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/SECURITY.md @@ -0,0 +1,23 @@ +--- +title: Reporting Security Issues +slug: /security +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/SECURITY.md' +--- +# Reporting Security Issues + +If you think you have found a security vulnerability, please send a report to security@datahub.com. This address can be used for all of DataHub’s open source and commercial products (including but not limited to DataHub Core and DataHub Cloud). We can accept only vulnerability reports at this address. + +It's not mandatory, but if you'd like to encrypt your message to us; please use our PGP key. The key fingerprint is: + +61C8 635F 856A 97AA 90D2 E1D9 EBD4 FB71 D997 8918 + +The key is available from [keys.openpgp.org](https://keys.openpgp.org/search?q=61C8635F856A97AA90D2E1D9EBD4FB71D9978918). + +DataHub will send you a response indicating the next steps in handling your report. After the initial reply to your report, the security team will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance. + +**Important:** We ask you to not disclose the vulnerability before it have been fixed and announced, unless you received a response from the DataHub security team that you can do so. + +## Security announcements + +We maintain [Security Advisories](https://github.com/datahub-project/datahub/security/advisories) on the DataHub project GitHub repository, +where we will post a summary, remediation, and mitigation details for any patch containing security fixes. diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-actions/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-actions/README.md new file mode 100644 index 00000000..8772888b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-actions/README.md @@ -0,0 +1,248 @@ +--- +title: ⚡ DataHub Actions Framework +slug: /datahub-actions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-actions/README.md +--- +# ⚡ DataHub Actions Framework + +Welcome to DataHub Actions! The Actions framework makes responding to realtime changes in your Metadata Graph easy, enabling you to seamlessly integrate [DataHub](https://github.com/datahub-project/datahub) into a broader events-based architecture. + +For a detailed introduction, check out the [original announcement](https://www.youtube.com/watch?v=7iwNxHgqxtg&t=2189s) of the DataHub Actions Framework at the DataHub April 2022 Town Hall. For a more in-depth look at use cases and concepts, check out [DataHub Actions Concepts](../docs/actions/concepts.md). + +## Quickstart + +To get started right away, check out the [DataHub Actions Quickstart](../docs/actions/quickstart.md) Guide. + +## Prerequisites + +The DataHub Actions CLI commands are an extension of the base `datahub` CLI commands. We recommend +first installing the `datahub` CLI: + +```shell +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub +datahub --version +``` + +> Note that the Actions Framework requires a version of `acryl-datahub` >= v0.8.34 + +## Installation + +Next, simply install the `acryl-datahub-actions` package from PyPi: + +```shell +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub-actions +datahub actions version +``` + +## Configuring an Action + +Actions are configured using a YAML file, much in the same way DataHub ingestion sources are. An action configuration file consists of the following + +1. Action Pipeline Name (Should be unique and static) +2. Source Configurations +3. Transform + Filter Configurations +4. Action Configuration +5. Pipeline Options (Optional) +6. DataHub API configs (Optional - required for select actions) + +With each component being independently pluggable and configurable. + +```yml +# 1. Required: Action Pipeline Name +name: + +# 2. Required: Event Source - Where to source event from. +source: + type: + config: + # Event Source specific configs (map) + +# 3a. Optional: Filter to run on events (map) +filter: + event_type: + event: + # Filter event fields by exact-match + + +# 3b. Optional: Custom Transformers to run on events (array) +transform: + - type: + config: + # Transformer-specific configs (map) + +# 4. Required: Action - What action to take on events. +action: + type: + config: + # Action-specific configs (map) + +# 5. Optional: Additional pipeline options (error handling, etc) +options: + retry_count: 0 # The number of times to retry an Action with the same event. (If an exception is thrown). 0 by default. + failure_mode: "CONTINUE" # What to do when an event fails to be processed. Either 'CONTINUE' to make progress or 'THROW' to stop the pipeline. Either way, the failed event will be logged to a failed_events.log file. + failed_events_dir: "/tmp/datahub/actions" # The directory in which to write a failed_events.log file that tracks events which fail to be processed. Defaults to "/tmp/logs/datahub/actions". + +# 6. Optional: DataHub API configuration +datahub: + server: "http://localhost:8080" # Location of DataHub API + # token: # Required if Metadata Service Auth enabled +``` + +### Example: Hello World + +An simple configuration file for a "Hello World" action, which simply prints all events it receives, is + +```yml +# 1. Action Pipeline Name +name: "hello_world" +# 2. Event Source: Where to source event from. +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +# 3. Action: What action to take on events. +action: + type: "hello_world" +``` + +We can modify this configuration further to filter for specific events, by adding a "filter" block. + +```yml +# 1. Action Pipeline Name +name: "hello_world" + +# 2. Event Source - Where to source event from. +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} + +# 3. Filter - Filter events that reach the Action +filter: + event_type: "EntityChangeEvent_v1" + event: + category: "TAG" + operation: "ADD" + modifier: "urn:li:tag:pii" + +# 4. Action - What action to take on events. +action: + type: "hello_world" +``` + +## Running an Action + +To run a new Action, just use the `actions` CLI command + +``` +datahub actions -c +``` + +Once the Action is running, you will see + +``` +Action Pipeline with name '' is now running. +``` + +### Running multiple Actions + +You can run multiple actions pipeline within the same command. Simply provide multiple +config files by restating the "-c" command line argument. + +For example, + +``` +datahub actions -c -c +``` + +### Running in debug mode + +Simply append the `--debug` flag to the CLI to run your action in debug mode. + +``` +datahub actions -c --debug +``` + +### Stopping an Action + +Just issue a Control-C as usual. You should see the Actions Pipeline shut down gracefully, with a small +summary of processing results. + +``` +Actions Pipeline with name '- + https://github.com/datahub-project/datahub/blob/master/datahub-actions/src/datahub_actions/plugin/action/propagation/docs/README.md +--- +# Documentation Propagation Action + +The Documentation Propagation Action allows you to automatically propagate documentation from schema fields to related schema fields. For example, when you add or update documentation for a column in a dataset, this action can automatically propagate that documentation to upstream, downstream, or sibling columns. + +## Functionality + +This action listens for documentation changes on schema fields and propagates those changes to related fields based on your configuration. It supports: + +- **Downstream Propagation**: Propagate documentation to columns in downstream datasets +- **Upstream Propagation**: Propagate documentation to columns in upstream datasets +- **Sibling Propagation**: Propagate documentation to columns in sibling datasets (e.g., views of the same table) + +## Configuration Options + +The Documentation Propagation Action provides several configuration options: + +- `enabled`: Controls whether documentation propagation is enabled (default: `true`) +- `columns_enabled`: Controls whether column-level documentation propagation is enabled (default: `true`) +- `datasets_enabled`: Controls whether dataset-level documentation propagation is enabled (default: `false`) - Note: Currently not implemented +- `column_propagation_relationships`: Specifies which relationships to use for propagation. Valid values are: + - `UPSTREAM`: Propagate to upstream columns + - `DOWNSTREAM`: Propagate to downstream columns + - `SIBLING`: Propagate to sibling columns +- `max_propagation_depth`: Maximum depth for propagation chains (default: `5`) +- `max_propagation_fanout`: Maximum number of entities to propagate to in a single hop (default: `1000`) +- `max_propagation_time_millis`: Maximum time in milliseconds for a propagation chain (default: `3600000` - 1 hour) + +## Example Configuration + +```yaml +name: "documentation_propagation" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} + # Topic Routing - which topics to read from. + topic_routes: + #mcl: ${METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME:-MetadataChangeLog_Versioned_v1} # Topic name for MetadataChangeLogEvent_v1 events. + pe: ${PLATFORM_EVENT_TOPIC_NAME:PlatformEvent_v1} # Topic name for PlatformEvent_v1 events. +filter: + event_type: "EntityChangeEvent_v1" + event: + entityType: "schemaField" + category: "DOCUMENTATION" +action: + type: "doc_propagation" + config: + enabled: true + columns_enabled: true + max_propagation_depth: 3 # Optional: Limit propagation depth + +datahub: + server: ${DATAHUB_GMS_HOST:-http://localhost:8080} + token: ${DATAHUB_GMS_TOKEN} +``` + +## Behavior + +When a documentation change is detected on a schema field: + +1. The action checks if propagation is enabled and if the change should be propagated +2. It determines the appropriate propagation relationships based on your configuration +3. It finds related schema fields based on those relationships +4. It propagates the documentation to those fields, preserving attribution information +5. It respects propagation limits (depth, fanout, time) to prevent excessive propagation + +## Propagation Attribution + +When documentation is propagated, the action adds attribution metadata to track: + +- The original source of the documentation +- The propagation path +- The time of propagation +- The depth of propagation + +This attribution information is stored with the propagated documentation and can be viewed in the DataHub UI. + +## Caveats and Limitations + +- **Upstream Propagation**: When propagating upstream, the action only propagates if there is exactly one upstream field. This prevents ambiguous propagation when multiple upstream fields exist. +- **Dataset Documentation**: Dataset-level documentation propagation is not currently supported, only schema field (column) documentation. +- **Single Upstream Field**: For downstream propagation, the action only propagates to a downstream field if it has exactly one upstream field (the field being propagated from). This ensures that documentation is only propagated in clear one-to-one relationships. diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/tag/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/tag/README.md new file mode 100644 index 00000000..bffbdc02 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/tag/README.md @@ -0,0 +1,43 @@ +--- +title: Tag Sync Action +slug: /datahub-actions/src/datahub_actions/plugin/action/tag +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-actions/src/datahub_actions/plugin/action/tag/README.md +--- +# Tag Sync Action + +The Tag Sync (or Tag Propagation) Action allows you to propagate tags from your assets into downstream entities. e.g. You can apply a tag (like `critical`) on a dataset and have it propagate down to all the downstream datasets. + +## Configurability + +You can control which tags should be propagated downstream using a prefix system. E.g. You can specify that only tags that start with `tier:` should be propagated downstream. + +## Additions and Removals + +The action supports both additions and removals of tags. + +### Example Config + +```yaml +name: "tag_propagation" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +filter: + event_type: "EntityChangeEvent_v1" +action: + type: "tag_propagation" + config: + tag_prefixes: + - classification + +datahub: + server: "http://localhost:8080" +``` + +## Caveats + +- Tag Propagation is currently only supported for downstream datasets. Tags will not propagate to downstream dashboards or charts. Let us know if this is an important feature for you. diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/term/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/term/README.md new file mode 100644 index 00000000..0de7df2d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-actions/src/datahub_actions/plugin/action/term/README.md @@ -0,0 +1,62 @@ +--- +title: Glossary Term Propagation Action +slug: /datahub-actions/src/datahub_actions/plugin/action/term +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-actions/src/datahub_actions/plugin/action/term/README.md +--- +# Glossary Term Propagation Action + +The Glossary Term Propagation Action allows you to propagate glossary terms from your assets into downstream entities. + +## Use Cases + +Enable classification of datasets or field of datasets to propagate metadata to downstream datasets with minimum manual work. + +## Functionality + +Propagation can be controlled via a specified list of terms or a specified list of term groups. + +### Target Terms + +- Given a list of "target terms", the propagation action will detect application of the target term to any field or dataset and propagate it down (as a dataset-level tag) on all downstream datasets. For example, given a target term of `Classification.Confidential` (the default), if you apply `Classification.Confidential` term to a dataset (at the dataset level or a field-level), this action will find all the downstream datasets and apply the `Classification.Confidential` tag to them at the dataset level. Note that downstream application is only at the dataset level, regardless of whether the primary application was at the field level or the dataset level. +- This action also supports term linkage. If you apply a term that is linked to the target term via inheritance, then this action will detect that application and propagate it downstream as well. For example, if the term `PersonalInformation.Email` inherits `Classification.Confidential` (the target term), and if you apply the `PersonalInformation.Email` term to a dataset (or a field in the dataset), it will be picked up by the action, and the `PersonalInformation.Email` term will be applied at the dataset level to all the downstream entities. + +### Term Groups + +- Given a list of "term groups", the propagation action will only propagate terms that belong to these term groups. + +### Addition and Removals + +The action supports propagation of term additions and removals. + +## Configurability + +You can control what the target term should be. Linkage to the target term is controlled through your business glossary which is completely under your control. + +### Example Config + +```yaml +name: "term_propagation" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +filter: + event_type: "EntityChangeEvent_v1" +action: + type: "term_propagation" + config: + target_terms: + - Classification + term_groups: + - "Personal Information" + +datahub: + server: "http://localhost:8080" +``` + +## Caveats + +- Term Propagation is currently only supported for downstream datasets. Terms will not propagate to downstream dashboards or charts. Let us know if this is an important feature for you. diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/README.md new file mode 100644 index 00000000..4bdfd615 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/README.md @@ -0,0 +1,176 @@ +--- +title: DataHub Agent Context +sidebar_label: Agent Context +slug: /datahub-agent-context +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/README.md +--- +# DataHub Agent Context + +**DataHub Agent Context** provides a collection of tools and utilities for building AI agents that interact with DataHub metadata. This package contains MCP (Model Context Protocol) tools that enable AI agents to search, retrieve, and manipulate metadata in DataHub. These can be used directly to create an agent, or be included in an MCP server such as Datahub's open source MCP server. + +## Features + +## Installation + +### Base Installation + +```shell +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade datahub-agent-context +``` + +### With LangChain Support + +For building LangChain agents with pre-built tools: + +```shell +python3 -m pip install --upgrade "datahub-agent-context[langchain]" +``` + +## Prerequisites + +This package requires: + +- Python 3.9 or higher +- `acryl-datahub` package + +## Quick Start + +### Basic Example + +These tools are designed to be used with an AI agent and have the responses passed directly to an LLM, so the return schema is a simple dict, but they can be used independently if desired. + +```python +from datahub.ingestion.graph.client import DataHubGraph +from datahub_agent_context.mcp_tools.search import search +from datahub_agent_context.mcp_tools.entities import get_entities + +# Initialize DataHub graph client +client = DataHubClient.from_env() + +# Search for datasets +with client as client: + results = search( + query="user_data", + filters={"entity_type": ["dataset"]}, + num_results=10 + ) + +# Get detailed entity information +with client as client: + entities = get_entities( + urns=[result["entity"]["urn"] for result in results["searchResults"]] + ) +``` + +### LangChain Integration + +Build AI agents with pre-built LangChain tools: + +```python +from datahub.sdk.main_client import DataHubClient +from datahub_agent_context.langchain_tools import build_langchain_tools +from langchain.agents import create_agent + +# Initialize DataHub client +client = DataHubClient.from_env() + +# Build all tools (read-only by default) +tools = build_langchain_tools(client, include_mutations=False) + +# Or include mutation tools for tagging, descriptions, etc. +tools = build_langchain_tools(client, include_mutations=True) + +# Create agent +agent = create_agent(model, tools=tools, system_prompt="...") +``` + +**See [examples/langchain/](https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/examples/langchain/)** for complete LangChain agent examples including: + +- [simple_search.py](https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/examples/langchain/simple_search.py) - Minimal example with AWS Bedrock + +### Available Tools + +#### Search Tools + +- `search()` - Search across all entity types with filters and sorting +- `search_documents()` - Search specifically for Document entities +- `grep_documents()` - Grep for patterns in document content + +#### Entity Tools + +- `get_entities()` - Get detailed information about entities by URN +- `list_schema_fields()` - List and filter schema fields for datasets + +#### Lineage Tools + +- `get_lineage()` - Get upstream or downstream lineage +- `get_lineage_paths_between()` - Get detailed paths between two entities + +#### Query Tools + +- `get_dataset_queries()` - Get SQL queries for datasets or columns + +#### Mutation Tools + +- `add_tags()`, `remove_tags()` - Manage tags +- `update_description()` - Update entity descriptions +- `set_domains()`, `remove_domains()` - Manage domains +- `add_owners()`, `remove_owners()` - Manage owners +- `add_glossary_terms()`, `remove_glossary_terms()` - Manage glossary terms +- `add_structured_properties()`, `remove_structured_properties()` - Manage structured properties +- `save_document()` - Save or update a Document. + +#### User Tools + +- `get_me()` - Get information about the authenticated user + +## Architecture + +The package is organized into the following modules: + +- `mcp_tools/` - Core MCP tool implementations + - `base.py` - Base GraphQL execution and response cleaning + - `search.py` - Search functionality + - `documents.py` - Document search and grep + - `entities.py` - Entity retrieval + - `lineage.py` - Lineage querying + - `queries.py` - Query retrieval + - `tags.py`, `descriptions.py`, `domains.py`, etc. - Mutation tools + - `helpers.py` - Shared utility functions + - `gql/` - GraphQL query definitions + +## Development + +### Setup + +```shell +# Clone the repository +git clone https://github.com/datahub-project/datahub.git +cd datahub/datahub-agent-context + +# Set up development environment +./gradlew :datahub-agent-context:installDev + +# Run tests +./gradlew :datahub-agent-context:testFull + +# Run linting +./gradlew :datahub-agent-context:lintFix +``` + +### Testing + +The package includes comprehensive unit tests for all tools: + +```shell +# Run full test suite +./gradlew :datahub-agent-context:testFull +``` + +## Support + +- [Documentation](https://datahubproject.io/docs/) +- [Slack Community](https://datahub.com/slack) +- [GitHub Issues](https://github.com/datahub-project/datahub/issues) diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/src/datahub_agent_context/snowflake/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/src/datahub_agent_context/snowflake/README.md new file mode 100644 index 00000000..80930788 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-agent-context/src/datahub_agent_context/snowflake/README.md @@ -0,0 +1,182 @@ +--- +title: Snowflake Agent Setup for DataHub +slug: /datahub-agent-context/src/datahub_agent_context/snowflake +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/src/datahub_agent_context/snowflake/README.md +--- +# Snowflake Agent Setup for DataHub + +This module provides tools to generate and deploy Snowflake UDFs (User-Defined Functions) that enable Snowflake Cortex Intelligence to query DataHub metadata. + +## Overview + +The module generates SQL scripts to create: + +1. Network rules and external access integrations for DataHub API calls +2. Python UDFs for searching and retrieving metadata from DataHub +3. Stored procedures for dynamic SQL execution +4. Cortex Agent configuration with DataHub tools + +## Implementation + +UDFs use the `datahub-agent-context` package wrapper methods that abstract away GraphQL/REST API complexity. + +## Usage + +### Basic Usage + +```bash +python -m datahub.ai.snowflake.snowflake \ + --sf-account YOUR_ACCOUNT \ + --sf-user YOUR_USER \ + --sf-role YOUR_ROLE \ + --sf-warehouse YOUR_WAREHOUSE \ + --sf-database YOUR_DATABASE \ + --sf-schema YOUR_SCHEMA \ + --datahub-url https://your-datahub.acryl.io \ + --datahub-token YOUR_TOKEN \ + --enable-mutations +``` + +### Direct Execution + +Add `--execute` and `--sf-password` to automatically run the generated scripts: + +```bash +python -m datahub.ai.snowflake.snowflake \ + --sf-account YOUR_ACCOUNT \ + --sf-user YOUR_USER \ + --sf-password YOUR_PASSWORD \ + --sf-role YOUR_ROLE \ + --sf-warehouse YOUR_WAREHOUSE \ + --sf-database YOUR_DATABASE \ + --sf-schema YOUR_SCHEMA \ + --datahub-url https://your-datahub.acryl.io \ + --datahub-token YOUR_TOKEN \ + --enable-mutations \ + --execute +``` + +### Direct Execution with SSO + +Use `--sf-authenticator=externalbrowser` for SSO authentication (no password required): + +```bash +python -m datahub.ai.snowflake.snowflake \ + --sf-account YOUR_ACCOUNT \ + --sf-user YOUR_USER \ + --sf-authenticator externalbrowser \ + --sf-role YOUR_ROLE \ + --sf-warehouse YOUR_WAREHOUSE \ + --sf-database YOUR_DATABASE \ + --sf-schema YOUR_SCHEMA \ + --datahub-url https://your-datahub.acryl.io \ + --datahub-token YOUR_TOKEN \ + --enable-mutations \ + --execute +``` + +This will open your browser for SSO authentication. Ideal for organizations using SAML, Okta, or other identity providers configured with Snowflake. + +## Options + +| Option | Description | Default | +| ----------------------- | ------------------------------------------------------------------------------------------------ | ----------------------- | +| `--sf-account` | Snowflake account identifier | Required | +| `--sf-user` | Snowflake user name | Required | +| `--sf-role` | Snowflake role | Required | +| `--sf-warehouse` | Snowflake warehouse name | Required | +| `--sf-database` | Snowflake database name | Required | +| `--sf-schema` | Snowflake schema name | Required | +| `--datahub-url` | DataHub instance URL | Required | +| `--datahub-token` | DataHub Personal Access Token | Required | +| `--agent-name` | Agent name in Snowflake | `DATAHUB_SQL_AGENT` | +| `--agent-display-name` | Agent display name in UI | `DataHub SQL Assistant` | +| `--agent-color` | Agent color in UI | `blue` | +| `--output-dir` | Output directory for SQL files | `./snowflake_setup` | +| `--enable-mutations` | Include mutation/write tools (tags, descriptions, owners, etc.) | `True` (enabled) | +| `--no-enable-mutations` | Disable mutation tools (read-only mode with 9 UDFs instead of 20) | N/A | +| `--execute` | Execute scripts directly | `False` | +| `--sf-password` | Snowflake password (required if --execute is used with `snowflake` authenticator) | None | +| `--sf-authenticator` | Authentication method: `snowflake` (password), `externalbrowser` (SSO), or `oauth` (token-based) | `snowflake` | + +## Authentication Methods + +Three authentication methods are supported when using `--execute`: + +### 1. Password Authentication (Default) + +Standard username/password authentication: + +```bash +--execute --sf-password YOUR_PASSWORD +``` + +### 2. SSO / External Browser Authentication (Recommended) + +Browser-based SSO authentication (SAML, Okta, Azure AD, etc.): + +```bash +--execute --sf-authenticator externalbrowser +``` + +When using this method: + +- A browser window will automatically open for authentication +- No password is required on the command line +- Ideal for organizations with federated identity providers +- Your Snowflake account must be configured for SSO + +### 3. OAuth Authentication + +Token-based OAuth authentication: + +```bash +--execute --sf-authenticator oauth +``` + +Note: OAuth authentication requires additional token configuration in your environment. + +## Read-Only vs Read+Write Modes + +By default, the generator creates 20 UDFs including both read and write operations: + +- **Read-only tools (9 UDFs)**: Search, retrieve metadata, lineage, queries, documents +- **Mutation tools (11 UDFs)**: Add/remove tags, update descriptions, manage owners, domains, glossary terms, and structured properties + +Use `--no-enable-mutations` to generate only the 9 read-only UDFs for environments where metadata modifications should be restricted. + +## Generated Files + +1. **00_configuration.sql** - Configuration variables and secrets +2. **01_network_rules.sql** - Network rules and external access integration +3. **02_datahub_udfs.sql** - DataHub API UDFs (9 read-only or 20 read+write depending on `--enable-mutations`) +4. **03_stored_procedure.sql** - Dynamic SQL execution procedure +5. **04_cortex_agent.sql** - Cortex Agent definition + +## Manual Execution + +If not using `--execute`, run the generated SQL files in order: + +```sql +-- 1. Set up configuration and secrets +@00_configuration.sql; + +-- 2. Create network rules +@01_network_rules.sql; + +-- 3. Create DataHub UDFs +@02_datahub_udfs.sql; + +-- 4. Create stored procedure +@03_stored_procedure.sql; + +-- 5. Create Cortex Agent +@04_cortex_agent.sql; +``` + +## Related Documentation + +- [DataHub Agent Context Documentation](../../../README.md) +- [Snowflake Python UDFs](https://docs.snowflake.com/en/developer-guide/udf/python/udf-python) +- [Snowflake Cortex Intelligence](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-intelligence) diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-frontend/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-frontend/README.md new file mode 100644 index 00000000..7f181861 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-frontend/README.md @@ -0,0 +1,97 @@ +--- +title: datahub-frontend +sidebar_label: datahub-frontend +slug: /datahub-frontend +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-frontend/README.md +--- + +# DataHub Frontend Proxy + +DataHub frontend is a [Play](https://www.playframework.com/) service written in Java. It is served as a mid-tier +between [DataHub GMS](https://github.com/datahub-project/datahub/blob/master/metadata-service) which is the backend service and [DataHub Web](../datahub-web-react/README.md). + +## Pre-requisites + +- You need to have [JDK11](https://openjdk.org/projects/jdk/11/) + installed on your machine to be able to build `DataHub Frontend`. +- You need to have [Chrome](https://www.google.com/chrome/) web browser + installed to be able to build because UI tests have a dependency on `Google Chrome`. + +## Build + +`DataHub Frontend` is already built as part of top level build: + +``` +./gradlew build +``` + +However, if you only want to build `DataHub Frontend` specifically: + +``` +./gradlew :datahub-frontend:dist +``` + +## Dependencies + +Before starting `DataHub Frontend`, you need to make sure that [DataHub GMS](https://github.com/datahub-project/datahub/blob/master/metadata-service) and +all its dependencies have already started and running. + +## Start via Docker image + +Quickest way to try out `DataHub Frontend` is running the [Docker image](https://github.com/datahub-project/datahub/blob/master/docker/datahub-frontend). + +## Start via command line + +If you do modify things and want to try it out quickly without building the Docker image, you can also run +the application directly from command line after a successful [build](#build): + +``` +cd datahub-frontend/run && ./run-local-frontend +``` + +## Checking out DataHub UI + +After starting your application in one of the two ways mentioned above, you can connect to it by typing below +into your favorite web browser: + +``` +http://localhost:9002 +``` + +To be able to sign in, you need to provide your user name. The default account is `datahub`, password `datahub`. + +## Authentication + +DataHub frontend leverages [Java Authentication and Authorization Service (JAAS)](https://docs.oracle.com/javase/7/docs/technotes/guides/security/jaas/JAASRefGuide.html) to perform the authentication. By default we provided a [DummyLoginModule](https://github.com/datahub-project/datahub/blob/master/datahub-frontend/app/security/DummyLoginModule.java) which will accept any username/password combination. You can update [jaas.conf](https://github.com/datahub-project/datahub/blob/master/datahub-frontend/conf/jaas.conf) to match your authentication requirement. For example, use the following config for LDAP-based authentication, + +``` +WHZ-Authentication { +  com.sun.security.auth.module.LdapLoginModule sufficient +  userProvider="ldaps://:636/dc=" +  authIdentity="{USERNAME}" +  userFilter="(&(objectClass=person)(uid={USERNAME}))" +  java.naming.security.authentication="simple" +  debug="false" +  useSSL="true"; +}; +``` + +### Authentication in React + +The React app supports both JAAS as described above and separately OIDC authentication. To learn about configuring OIDC for React, +see the [OIDC in React](../docs/authentication/guides/sso/configure-oidc-react.md) document. + +### API Debugging + +Most DataHub frontend API endpoints are protected using [Play Authentication](https://www.playframework.com/documentation/2.1.0/JavaGuide4), which means it requires authentication information stored in the cookie for the request to go through. This makes debugging using curl difficult. One option is to first make a curl call against the `/authenticate` endpoint and stores the authentication info in a cookie file like this + +``` +curl -c cookie.txt -d '{"username":"datahub", "password":"datahub"}' -H 'Content-Type: application/json' http://localhost:9002/authenticate +``` + +You can then make all subsequent calls using the same cookie file to pass the authentication check. + +``` +curl -b cookie.txt "http://localhost:9001/api/v2/search?type=dataset&input=page" +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-graphql-core/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-graphql-core/README.md new file mode 100644 index 00000000..26ebc507 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-graphql-core/README.md @@ -0,0 +1,93 @@ +--- +title: datahub-graphql-core +sidebar_label: datahub-graphql-core +slug: /datahub-graphql-core +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/README.md +--- + +# DataHub GraphQL Core + +DataHub GraphQL API is a shared lib module containing a GraphQL API on top of the GMS service layer. It exposes a graph-based representation +permitting reads and writes against the entities and aspects on the Metadata Graph, including Datasets, CorpUsers, & more. + +Contained within this module are + +1. **GMS Schema**: A GQL schema based on GMS models, located under [resources](https://github.com/datahub-project/datahub/tree/master/datahub-graphql-core/src/main/resources) folder. +2. **GMS Data Fetchers** (Resolvers): Components used by the GraphQL engine to resolve individual fields in the GQL schema. +3. **GMS Data Loaders**: Components used by the GraphQL engine to fetch data from downstream sources efficiently (by batching). +4. **GraphQLEngine**: A wrapper on top of the default `GraphQL` object provided by `graphql-java`. Provides a way to configure all of the important stuff using a simple `Builder API`. +5. **GMSGraphQLEngine**: An engine capable of resolving the GMS schema using the data fetchers + loaders mentioned above (with no additional configuration required). + +We've chosen to place these components in a library module so that GraphQL servers can be deployed in multiple "modes": + +1. **Standalone**: GraphQL facade, mainly used for programmatic access to the GMS graph from a non-Java environment +2. **Embedded**: Leverageable within another Java server to surface an extended GraphQL schema. For example, we use this to extend the GMS GraphQL schema in `datahub-frontend` + +## Extending the Graph + +### Adding an Entity + +When adding an entity to the GMS graph, the following steps should be followed: + +1. Extend [entity.graphql](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/resources/entity.graphql) schema with new `types` (Queries) or `inputs` (Mutations) required for fetching & updating your Entity. + +These models should generally mirror the GMS models exactly, with notable exceptions: + +- **Maps**: the GQL model must instead contain a list of { key, value } objects (e.g. Dataset.pdl 'properties' field) +- **Foreign-Keys**: Foreign-key references embedded in GMS models should be resolved if the referenced entity exists in the GQL schema, + replacing the key with the actual entity model. (Example: replacing the 'owner' urn field in 'Ownership' with an actual `CorpUser` type) + +In GraphQL, the new Entity should extend the `Entity` interface. Additionally, you will need to add a new symbol to the standard +`EntityType` enum. + +The convention we follow is to have a top-level Query for each entity that takes a single "urn" parameter. This is for primary key lookups. +See all the existing entity Query types [here](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/resources/entity.graphql#L19). + +On rebuilding the module (`./gradlew datahub-graphql-core:build`) you'll find newly generated classes corresponding to +the types you've defined inside the GraphQL schema inside the `mainGeneratedGraphQL` folder. These classes will be used in the next step. + +2. Implement `EntityType` classes for any new entities + +- These 'type' classes define how to load entities from GMS, and map them to the GraphQL data model. See [DatasetType.java](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java) as an example. + +3. Implement `Mappers` to transform Pegasus model returned by GMS to an auto-generated GQL POJO. (under `/mainGeneratedGraphQL`, generated on `./gradlew datahub-graphql-core:build`) These mappers + will be used inside the type class defined in step 2. + +- If you've followed the guidance above, these mappers should be simple, mainly + providing identity mappings for fields that exist in both the GQL + Pegasus POJOs. +- In some cases, you'll need to perform small lambdas (unions, maps) to materialize the GQL object. + +4. Wire up your `EntityType` to the GraphQL schema. + +We use [GmsGraphQLEngine.java](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java) to +configure the wiring for the GraphQL schema. This means associating "resolvers" to specific fields present in the GraphQL schema file. + +Inside of this file, you need to register your new `Type` object to be used in resolving primary-key entity queries. +To do so, simply follow the examples for other entities. + +5. Implement `EntityType` test for the new type defined in Step 2. See [ContainerTypeTest](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/container/ContainerTypeTest.java) as an example. + +6. Implement `Resolver` tests for any new `DataFetchers` that you needed to add. See [SetDomainResolverTest](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/domain/SetDomainResolverTest.java) as an example. + +7. [Optional] Sometimes, your new entity will have relationships to other entities, or fields that require specific business logic + as opposed to basic mapping from the GMS model. In such cases, we tend to create an entity-specific configuration method in [GmsGraphQLEngine.java](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java) + which allows you to wire custom resolvers (DataFetchers) to the fields in your Entity type. You also may need to do this, depending + on the complexity of the new entity. See [here](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java#L438) for reference. + +> Note: If you want your new Entity to be "browsable" (folder navigation) via the UI, make sure you implement the `BrowsableEntityType` interface. + +#### Enabling Search for a new Entity + +In order to enable searching an Entity, you'll need to modify the [SearchAcrossEntities.java](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java) resolver, which enables unified search +across all DataHub entities. + +Steps: + +1. Add your new Entity type to [this list](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java#L32). +2. Add a new statement to [UrnToEntityMapper.java](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java#L35). This maps + an URN to a "placeholder" GraphQL entity which is subsequently resolved by the GraphQL engine. + +That should be it! + +Now, you can try to issue a search for the new entities you've ingested diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/BASE_PATH.md b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/BASE_PATH.md new file mode 100644 index 00000000..de4c28d8 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/BASE_PATH.md @@ -0,0 +1,160 @@ +--- +title: Base Path Support in DataHub Frontend +slug: /datahub-web-react/base_path +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-web-react/BASE_PATH.md +--- +# Base Path Support in DataHub Frontend + +DataHub's React frontend supports runtime base path detection, allowing the same build to be deployed at different URL paths (e.g., `/` or `/datahub/`) without requiring rebuild or reconfiguration. + +## How It Works + +The base path is determined through a hierarchical detection system: + +1. **Server Template** (Primary): The Play Framework backend injects the base path into the HTML template +2. **Config Endpoint** (Fallback): Attempts to fetch base path from `/config` or `config` endpoints +3. **Default** (Last Resort): Falls back to root path `/` + +## Using Base Paths in Code + +### For Asset URLs + +Use the `resolveRuntimePath()` function to resolve any asset path: + +```typescript +import { resolveRuntimePath } from '../utils/runtimeBasePath'; + +// Resolve asset paths +const logoUrl = resolveRuntimePath('/assets/logo.png'); +const apiUrl = resolveRuntimePath('/api/graphql'); + +// Use in components +DataHub +``` + +### For Navigation Links + +Use React Router's relative paths or resolve absolute paths: + +```typescript +import { resolveRuntimePath } from '../utils/runtimeBasePath'; + +// For React Router navigation (preferred - automatic) +Datasets + +// For absolute URLs when needed +Browse +``` + +### For API Endpoints + +```typescript +import { resolveRuntimePath } from '../utils/runtimeBasePath'; + +// Resolve API endpoints +const endpoint = resolveRuntimePath('/api/v2/graphql'); +fetch(endpoint, { ... }); +``` + +## Configuration + +### Server Configuration + +The base path is configured in the backend Play application: + +```hocon +# datahub-frontend/conf/application.conf +datahub.basePath = "/datahub" +``` + +### Environment Variables + +For containerized deployments: + +```bash +# Docker/Kubernetes +DATAHUB_BASE_PATH=/datahub +``` + +## Examples + +### Development + +```bash +# Serve at root +yarn start # Available at http://localhost:3000/ + +# Serve at subpath +yarn preview --base /datahub # Available at http://localhost:3000/datahub/ +``` + +### Production Deployment + +**At Root Path:** + +```bash +DATAHUB_BASE_PATH="" docker run datahub-frontend +# Accessible at: https://datahub.company.com/ +``` + +or unset + +``` +unset DATAHUB_BASE_PATH +docker run datahub-frontend +``` + +**At Subpath:** + +```bash +DATAHUB_BASE_PATH=/datahub docker run datahub-frontend +# Accessible at: https://company.com/datahub/ +``` + +## Browser Support + +The implementation uses standard HTML `` tags and modern JavaScript features: + +- All modern browsers (Chrome 60+, Firefox 60+, Safari 12+, Edge 79+) +- Progressive enhancement with fallback detection + +## Troubleshooting + +### Assets Not Loading + +1. Check browser console for 404 errors +2. Verify `window.__DATAHUB_BASE_PATH__` is set correctly +3. Ensure all asset references use `resolveRuntimePath()` + +### Incorrect Redirects + +1. Check that authentication endpoints use resolved paths +2. Verify React Router basename configuration +3. Test config endpoint accessibility + +### Development Issues + +```bash +# Clear cache and rebuild +yarn clean +yarn build + +# Check base path detection +console.log(window.__DATAHUB_BASE_PATH__); +``` + +## Implementation Details + +- **HTML Template**: `datahub-frontend/app/views/index.scala.html` +- **Runtime Utility**: `src/utils/runtimeBasePath.ts` +- **Asset Resolution**: All `src/` files using `resolveRuntimePath()` +- **Server Handler**: `datahub-frontend/app/controllers/Application.java` + +The system automatically handles: + +- Favicon and manifest paths +- Main application JS/CSS assets +- Platform logos and theme assets +- API endpoint resolution +- Authentication redirects diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/CLAUDE.md b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/CLAUDE.md new file mode 100644 index 00000000..5f66afe7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/CLAUDE.md @@ -0,0 +1,216 @@ +--- +title: CLAUDE.md +slug: /datahub-web-react/claude +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-web-react/CLAUDE.md +--- +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this directory. +It also contains our style guide, which can be consumed by engineers. + +## Style Guide + +### File Structure + +- What should our top-level folders be? + - `app`: components that implement the entire application + - Based on nav page + - Each new high-level page gets a new top-level folder + - Nested pages are within the same top-level folder + - `graphql`: graphql files and generated types + - `images`: all custom images + - `conf` / `fonts` / `providers` / `utils`: do we need these? + - When do we create a new one? +- What to put in `index.ts(x)`? + - Do not create these files +- Where to put tests? + - In `__tests__/` in the same directory as the file + - Named `OriginalFile.test.ts(x)` +- Where to put utils? + - For utils used in a single file: `SourceFile.utils.ts` + - For utils used in a folder: `folderName/folderName.utils.ts` +- Where to put hooks? + - For helper hooks used in a single file: `SourceFile.hooks.ts(x)` + - For standalone, reusable hooks: `hookName.ts(x)` + - Try not to write hooks that generate JSX, but sometimes it may make sense +- Where to put types? + - Types only used once should be in the same file where they’re used + - Shared types go in `folderName/folderName.types.ts` +- Where to put helper components? + - If the component is only used once and is small: `parent/SourceFile.components.tsx` + - e.g. `search/SearchPage.components.tsx` + - If a component is shared / large and doesn’t have its own children: `parent/NewComponent.tsx` + - e.g. `search/SearchPageHeader.tsx` + - If the component has its own child components, put it in a new folder:`parent/NewComponent/NewComponent.tsx` + - e.g. `search/SearchCard/SearchCard.tsx` + +### Component Library + +- What is an DataHub component? (previously: alchemy component) + - Can be reused anywhere in the app + - Generalized + - Doesn’t depend on our aspect / data structure + - Doesn’t depend on graphql types +- Where do reusable non-DataHub components go? + - e.g. `EntityHealth` icon, `HoverEntityTooltip` + - Can be reused anywhere within the web app + - Not general: can take in very DataHub/gms specific inputs, e.g. graphql types + - shared/ + - components/ + - entity/ + - `entity.types.ts` + - health/ + - `EntityHealth` + - tooltip + - `HoverEntityTooltip` + - button/ + - `DownloadSearchResultsButton` + - hooks/ + - utils/ + +### Code Structure + +- When to break into a new component? + - _We should err on the side of more and smaller components (files)_ + - When JSX is getting too large or complex + - Any logical or reusable chunk +- When to break into a hook? + - When a component’s logic is getting too large or complex + - Any logical or reusable chunk of logic +- When to use a React context vs passing props? + - _Err on the side of not creating contexts — only add when necessary_ + - Globally: use a context when we want values available globally, e.g. `AppContext` or `UserContext` + - Low-level, small-scope: use a tightly-defined context for a small, self-contained set of files + - `folder/Context.ts` + - Contains type, default value, helper hooks + - `folder/ContextProvider.tsx` + - Contains just the provider + - TBD: How to handle contexts for entity components? + - `shared/entity/EntityContext.tsx`? +- How to pass props? + - For small amount of props, can use individual values + - When props gets larger, try to break into logical pieces + - e.g. `entityData` rather than individual props for each aspect of the entity + - For components reused between OSS and SaaS: + - SaaS-only props should go in a single field: `acrylProps: { ... }` +- Component file sections: + - Imports + - Constants (in SNAKE_CASE) + - Styled components + - Props + - Main component + +### Code Conventions + +- How to pass styles to a component? + - Use styled components for any simple components + - For custom components (e.g. our DataHub components), take `className` in as a prop and pass it to the top-level element — whatever it makes sense for custom styles to be applied to + - This allows us to pass styles by styled component inheritance + - e.g. if there’s a custom component `Button`, this allows us to define `const CustomButton = styled(Button)` + - If there are multiple elements for which custom styles can be applied, non-top-level ones should be passed as a CSSProperties prop, with a specific name, e.g. `textStyle` or `buttonStyle` +- How to handle images and icons + - For component library icons, use `` DataHub component and specify color and size via props + - If size is not known, use `size="inherit"` and set `font-size` in parent div + - Always use phosphor icons — ant and material ui icons are deprecated + - For custom images, use `` (once it’s built) and specify color and size via props + - For svgs, in svg definition, set `fill="currentColor"` and then … TODO + - Alt text should be a required field +- How to do theming? + - Use `styled-components` theming, with stringed css rather than object css +- **React Components**: Use TypeScript interfaces for props instead of PropTypes (which are deprecated) + +```jsx +// YES +styled.div` + border-radius: 2px; +`; + +// NO +styled.div({ borderRadius: '2px' }); +``` + +### Code Style + +_Mostly handled by linter and formatter (prettier). Mostly unimportant changes that we should be consistent on to improve readability._ + +- Functions vs lambdas + - Top-level, prefer named functions (`export default function f() { }`) + - Nested, use lambdas (`const onClick = () => { }`) +- Prefer `type` over `interface` (except when using classes) +- When to omit types + - For variables, do not need to specify + - For function signatures, try to specify, unless it’s too difficult. In that case, prefer inferred typing over `any` + - Prefer a mapper function over unsafe type casting + - If you must cast types, do so as early (high) as possible +- Prefer destructuring over dot notation, but use your own intuition + - When accessing arrays, always option chain (i.e. `x?.[0]`) +- When to optional chain + - If it type checks, it should be fine, except for the array access case above +- Prefer direct imports, e.g. `import React, { useState } from 'react'` over `React.useState` + +## Development Commands + +### Setup and Dependencies + +```bash +# Install dependencies and set up dev environment +# Update dependencies after changes to package.json +../gradlew yarnInstall +``` + +### Running the Service + +```bash +# Local development on localhost:3000 (with hot reload) +../gradlew yarnServe + +# Development with remote GMS (e.g., dev01) +../gradlew yarnPreview -Pproxy="" +# e.g. ../gradlew yarnPreview -Pproxy="https://dev01.acryl.io/" +``` + +### Testing and Code Quality + +Run formatting / linting / type checking / relevant tests after all changes + +```bash +# Generate ts files based on graphql +yarn generate + +# Run formatting +yarn format + +# Run linting on all files +../gradlew yarnLint + +# Run lint-fix on a single file +../gradlew -x yarnInstall -x yarnGenerate yarnLintFix -Pfile=src/path/to/file.tsx + +# Run linting on a single file +# This does not run full type-check when we run for a single file +# that should be run at the end of all changes before commit +../gradlew -x yarnInstall -x yarnGenerate yarnLint -Pfile=src/path/to/file.tsx + +# Run lint-fix on all files +../gradlew yarnLintFix + +# Run type checking +yarn type-check + +# Run tests +yarn test + +# Run specific test file +yarn test path/to/file.test.tsx --run +``` + +## Writing Tests - Best Practices & Common Pitfalls + +### Test Setup Essentials + +**Always use the existing test infrastructure:** + +- Use `TestPageContainer` from `@utils/test-utils/TestPageContainer` - it provides all necessary providers +- Use `MockedProvider` from `@apollo/client/testing` for GraphQL components +- Use Vitest with React Testing Library diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/README.md new file mode 100644 index 00000000..ba4c8795 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/README.md @@ -0,0 +1,187 @@ +--- +title: datahub-web-react +sidebar_label: datahub-web-react +slug: /datahub-web-react +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-web-react/README.md +--- + +# DataHub React App + +## About + +This module contains a React application that serves as the DataHub UI. + +Feel free to take a look around, deploy, and contribute. + +## Functional Goals + +The initial milestone for the app was to achieve functional parity with the previous Ember app. This meant supporting + +- Dataset Profiles, Search, Browse Experience +- User Profiles, Search +- LDAP Authentication Flow + +This has since been achieved. The new set of functional goals are reflected in the latest version of the [DataHub Roadmap](../docs/roadmap.md). + +## Design Goals + +In building out the client experience, we intend to leverage learnings from the previous Ember-based app and incorporate feedback gathered +from organizations operating DataHub. Two themes have emerged to serve as guideposts: + +1. **Configurability**: The client experience should be configurable, such that deploying organizations can tailor certain + aspects to their needs. This includes theme / styling configurability, showing and hiding specific functionality, + customizing copy & logos, etc. +2. **Extensibility**: Extending the _functionality_ of DataHub should be as simple as possible. Making changes like + extending an existing entity & adding a new entity should require minimal effort and should be well covered in detailed + documentation. + +## Starting the Application + +### Quick Start + +Follow the instructions [here](/docs/developers#building-the-project) to build and deploy your project locally. The initial build might take a while. You will be able to navigate to the application at `http://localhost:9002`. + +If you want to make changes to the UI see them live without having to rebuild the `datahub-frontend-react` docker image, you +can run the following in this directory: + +`yarn install && yarn run start` + +which will start a forwarding server at `localhost:3000`. Note that to fetch real data, `datahub-frontend` server will also +need to be deployed, still at `http://localhost:9002`, to service GraphQL API requests. + +Optionally you could also start the app with the mock server without running the docker containers by executing `yarn start:mock`. See [here](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/graphql-mock/fixtures/searchResult/userSearchResult.ts#L6) for available login users. + +### Testing your customizations + +There is two options to test your customizations: + +- **Option 1**: Initialize the docker containers with the `quickstart.sh` script (or if any custom docker-compose file) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at `http://localhost:9002` to fetch real data. +- **Option 2**: Change the environment variable `REACT_APP_PROXY_TARGET` in the `.env` file to point to your `datahub-frontend` server (ex: https://my_datahub_host.com) and then run `yarn start` in this directory. This will start a forwarding server at `localhost:3000` that will use the `datahub-frontend` server at some domain to fetch real data. To test in Safari, you need to run over https because Safari drops the Secure cookies over http that manage user session. You can do this by running `yarn start:https` and ignoring Safari's security warning. + +The option 2 is useful if you want to test your React customizations without having to run the whole DataHub stack locally. However, if you changed other components of the DataHub stack, you will need to run the hole stack locally (building the docker images) and use the option 1. + +### Functional testing + +In order to start a server and run frontend unit tests using react-testing-framework, run: + +`yarn test :e2e` + +There are also more automated tests using Cypress in the `smoke-test` folder of the repository root. + +#### Troubleshooting + +`Error: error:0308010C:digital envelope routines::unsupported`: This error message shows up when using Node 17, due to an OpenSSL update related to md5. +The best workaround is to revert to the Active LTS version of Node, 16.13.0 with the command `nvm install 16.13.0` and if necessary reinstall yarn `npm install --global yarn`. + +### Theming + +#### Customizing your App without rebuilding assets + +To see the results of any change to a theme, you will need to rebuild your datahub-frontend-react container. While this may work for some users, if you don't want to rebuild your container +you can change two things without rebuilding. + +1. You customize the logo on the homepage & the search bar header by setting the `REACT_APP_LOGO_URL` env variable when deploying GMS. +2. You can customize the favicon (the icon on your browser tab) by setting the `REACT_APP_FAVICON_URL` env var when deploying GMS. + +#### Selecting a theme + +Theme configurations are defined in `./src/conf/theme/themes.ts`. By default, the theme is chosen based on the `REACT_APP_CUSTOM_THEME_ID` env variable in GMS. If no theme is specified, the default themes `themeV2` or `themeV1` are used based on whether the V2 UI is enabled, which is controlled by environment variables `THEME_V2_ENABLED`, `THEME_V2_DEFAULT`, and `THEME_V2_TOGGLEABLE` in GMS. See `metadata-service/configuration/src/main/resources/application.yaml` for more details. + +For quick local development, you can set env variable `REACT_APP_THEME` in `.env` to any of the themes defined in `themes.ts`. + +We are transitioning away from Ant theming, but still depend on it for some styling. The Ant theme is stored in json files, in `./src/conf/theme`. To select the Ant theme, choose a json file and set env variable `ANT_THEME_CONFIG` in `.env` to the theme's filename, including `.json`, then re-run `yarn start` from `datahub/datahub-web-react`. + +#### Editing a theme + +To edit an existing theme, the recommendation is to clone one of the existing themes into a new file with the name `.ts`, then update `themes.ts` by importing your theme and adding it to the `themes` object. You can also create a json theme by creating a new file in `./src/conf/theme` with the name `.config.json`. The theme interface is defined in `./src/conf/theme/types.ts` and has four sections: + +`colors` configures semantic color tokens. +These are not yet widely used but will be the primary way to configure colors in the app going forward. + +`styles` configures overrides for the app's deprecated theming variables and for Ant components. + +`assets` configures the logo url. + +`content` specifies customizable text fields. + +While developing on your theme, all changes to assets and content are seen immediately in your local app. However, changes to styles require +you to terminate and re-run `yarn start` to see updated styles. + +## Design Details + +### Package Organization + +The `src` dir of the app is broken down into the following modules + +**conf** - Stores global configuration flags that can be referenced across the app. For example, the number of +search results shown per page, or the placeholder text in the search bar box. It serves as a location where levels +for functional configurability should reside. + +**app** - Contains all important components of the app. It has a few sub-modules: + +- `auth`: Components used to render the user authentication experience. +- `browse`: Shared components used to render the 'browse-by-path' experience. The experience is akin to navigating a filesystem hierarchy. +- `preview`: Shared components used to render Entity 'preview' views. These can appear in search results, browse results, + and within entity profile pages. +- `search`: Shared components used to render the full-text search experience. +- `shared`: Misc. shared components +- `entity`: Contains Entity definitions, where entity-specific functionality resides. + Configuration is provided by implementing the 'Entity' interface. (See DatasetEntity.tsx for example) + There are 2 visual components each entity should supply: + + - `profiles`: display relevant details about an individual entity. This serves as the entity's 'profile'. + - `previews`: provide a 'preview', or a smaller details card, containing the most important information about an entity instance. + + When rendering a preview, the entity's data and the type of preview (SEARCH, BROWSE, PREVIEW) are provided. This + allows you to optionally customize the way an entities preview is rendered in different views. + + - `entity registry`: There's another very important piece of code living within this module: the **EntityRegistry**. This is a layer + of abstraction over the intimate details of rendering a particular entity. It is used + to render a view associated with a particular entity type (user, dataset, etc.). + +

+ +

+ +**graphql** - The React App talks to the `dathub-frontend` server using GraphQL. This module is where the _queries_ issued +against the server are defined. Once defined, running `yarn run generate` will code-gen TypeScript objects to make invoking +these queries extremely easy. An example can be found at the top of `SearchPage.tsx.` + +**images** - Images to be displayed within the app. This is where one would place a custom logo image. + +## Adding an Entity + +The following outlines a series of steps required to introduce a new entity into the React app: + +1. Declare the GraphQL Queries required to display the new entity + + - If search functionality should be supported, extend the "search" query within `search.graphql` to fetch the new + entity data. + - If browse functionality should be supported, extend the "browse" query within `browse.graphql` to fetch the new + entity data. + - If display a 'profile' should be supported (most often), introduce a new `.graphql` file that contains a + `get` query to fetch the entity by primary key (urn). + + Note that your new entity _must_ implement the `Entity` GraphQL type interface, and thus must have a corresponding + `EntityType`. + +2. Implement the `Entity` interface + + - Create a new folder under `src/components/entity` corresponding to your entity + - Create a class that implements the `Entity` interface (example: `DatasetEntity.tsx`) + - Provide an implementation each method defined on the interface. + - This class specifies whether your new entity should be searchable & browsable, defines the names used to + identify your entity when instances are rendered in collection / when entity appears + in the URL path, and provides the ability to render your entity given data returned by the GQL API. + +3. Register the new entity in the `EntityRegistry` + - Update `App.tsx` to register an instance of your new entity. Now your entity will be accessible via the registry + and appear in the UI. To manually retrieve the info about your entity or others, simply use an instance + of the `EntityRegistry`, which is provided via `ReactContext` to _all_ components in the hierarchy. + For example + ``` + entityRegistry.getCollectionName(EntityType.YOUR_NEW_ENTITY) + ``` + +That's it! For any questions, do not hesitate to reach out on the DataHub Slack community in #datahub-react. diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/analytics/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/analytics/README.md new file mode 100644 index 00000000..eba5eb0a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/analytics/README.md @@ -0,0 +1,161 @@ +--- +title: DataHub React Analytics +sidebar_label: React Analytics +slug: /datahub-web-react/src/app/analytics +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/analytics/README.md +--- +# DataHub React Analytics + +## About + +The DataHub React application can be configured to emit a set of standardized product analytics events to multiple backend providers including + +- Mixpanel +- Amplitude +- Google Analytics + +This provides operators of DataHub with visibility into how their users are engaging with the platform, allowing them to answer questions around weekly active users, the most used features, the least used features, and more. + +To accomplish this, we have built a small extension on top of the popular [Analytics](https://www.npmjs.com/package/analytics) npm package. This package was chosen because it offers a clear pathway to extending support to many other providers, all of which you can find listed [here](https://github.com/DavidWells/analytics#analytic-plugins). + +## Configuring an Analytics Provider + +Currently, configuring an analytics provider requires that you fork DataHub & modify code. As described in 'Coming Soon', we intend to improve this process by implementing no-code configuration. + +### Mixpanel + +1. Open `datahub-web-react/src/conf/analytics.ts` +2. Uncomment the `mixpanel` field within the `config` object. +3. Replace the sample `token` with the API token provided by Mixpanel. +4. Rebuild & redeploy `datahub-frontend-react` to start tracking. + +```typescript +const config: any = { + mixpanel: { + token: 'fad1285da4e618b618973cacf6565e61', + }, +}; +``` + +### Amplitude + +1. Open `datahub-web-react/src/conf/analytics.ts` +2. Uncomment the `amplitude` field within the `config` object. +3. Replace the sample `apiKey` with the key provided by Amplitude. +4. Rebuild & redeploy `datahub-frontend-react` to start tracking. + +```typescript +const config: any = { + amplitude: { + apiKey: 'c5c212632315d19c752ab083bc7c92ff', + }, +}; +``` + +### Google Analytics + +1. Open `datahub-web-react/src/conf/analytics.ts` +2. Uncomment the `googleAnalytics` field within the `config`. +3. Replace the sample `measurementIds` with the one provided by Google Analytics. +4. Rebuild & redeploy `datahub-frontend-react` to start tracking. + +Example: + +```typescript +const config: any = { + googleAnalytics: { + measurementIds: ['G-ATV123'], + }, +}; +``` + +## Verifying your Analytics Setup + +To verify that analytics are being sent to your provider, you can inspect the networking tab of a Google Chrome inspector window: + +With DataHub open on Google Chrome + +1. Right click, then Inspect +2. Click 'Network' +3. Issue a search in DataHub +4. Inspect the outbound traffic for requests routed to your analytics provider. + +## Development + +### Adding a plugin + +To add a new plugin from the [Analytics](https://www.npmjs.com/package/analytics) library: + +1. Add a new file under `src/app/analytics/plugin` named based on the plugin +2. Extract configs from the analytics config object required to instantiate the plugin +3. Instantiate the plugin +4. Export a default object with 'isEnabled' and 'plugin' fields +5. Import / Export the new plugin module from `src/app/analytics/plugin/index.js` + +If you're unsure, check out the existing plugin implements as examples. Before contributing a plugin, please be sure to verify the integration by viewing the product metrics in the new analytics provider. + +### Adding an event + +To add a new DataHub analytics event, make the following changes to `src/app/analytics/event.ts`: + +1. Add a new value to the `EventType` enum + +```typescript + export enum EventType { + LogInEvent, + LogOutEvent, + ..., + MyNewEvent +} +``` + +2. Create a new interface extending `BaseEvent` + +```typescript +export interface MyNewEvent extends BaseEvent { + type: EventType.MyNewEvent; // must be the type you just added + ... your event's custom fields +} +``` + +3. Add the interface to the exported `Event` type. + +```typescript +export type Event = + | LogInEvent + | LogOutEvent + .... + | MyNewEvent +``` + +### Emitting an event + +Emitting a tracking DataHub analytics event is a 2-step process: + +1. Import relevant items from `analytics` module + +```typescript +import analytics, { EventType } from '../analytics'; +``` + +2. Call the `event` method, passing in an event object of the appropriate type + +```typescript +analytics.event({ type: EventType.MyNewEvent, ...my event fields }); +``` + +### Debugging: Enabling Event Logging + +To log events to the console for debugging / verification purposes + +1. Open `datahub-web-react/src/conf/analytics.ts` +2. Uncomment `logging: true` within the `config` object. +3. Rebuild & redeploy `datahub-frontend-react` to start logging all events to your browser's console. + +## Coming Soon + +In the near future, we intend to + +1. Send product analytics events back to DataHub itself, using them as feedback to improve the product experience. +2. No-code configuration of Analytics plugins. This will be achieved using server driven configuration for the React app. diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/ARCHITECTURE.md b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/ARCHITECTURE.md new file mode 100644 index 00000000..3e96dc85 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/ARCHITECTURE.md @@ -0,0 +1,296 @@ +--- +title: Document Change History Architecture +slug: /datahub-web-react/src/app/entityv2/document/changehistory/architecture +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/entityV2/document/changeHistory/ARCHITECTURE.md +--- +# Document Change History Architecture + +## Overview + +The Document Change History feature provides a timeline view of all changes made to a document, including creation, title changes, content modifications, moves, state changes, and deletions. Users can view previous versions and restore them if needed. + +## Architecture Principles + +This implementation follows best practices for: + +- ✅ **Testability**: Complex logic extracted into pure utility functions and custom hooks +- ✅ **Extensibility**: Easy to add new change types without touching existing code +- ✅ **Type Safety**: Full TypeScript support with proper type definitions +- ✅ **Error Handling**: Graceful degradation with loading and error states +- ✅ **User Experience**: Relative timestamps with full date on hover, smooth interactions + +## Directory Structure + +``` +changeHistory/ +├── ARCHITECTURE.md # This file +├── hooks/ +│ └── useParentDocumentTitle.ts # Custom hook for fetching parent titles +├── utils/ +│ └── changeUtils.ts # Pure utility functions for data extraction +├── changeMessages/ +│ ├── ChangeMessageComponents.tsx # Individual message components + router +│ └── README.md # Guide for adding new change types +├── DocumentChangeHistoryDrawer.tsx # Main drawer container +├── DocumentHistoryTimeline.tsx # Timeline list component +├── DocumentChangeTimelineContent.tsx # Individual timeline entry +├── DocumentChangeTimelineDot.tsx # User avatar for each change +└── PreviousVersionModal.tsx # View/restore previous content +``` + +## Component Hierarchy + +``` +DocumentChangeHistoryDrawer +└── DocumentHistoryTimeline + └── Timeline (from alchemy-components) + ├── DocumentChangeTimelineDot (for each item) + └── DocumentChangeTimelineContent (for each item) + ├── ChangeMessage (router component) + │ ├── CreatedMessage + │ ├── TitleChangedMessage + │ ├── TextChangedMessage + │ ├── StateChangedMessage + │ ├── ParentChangedMessage (uses useParentDocumentTitle hook) + │ ├── DeletedMessage + │ └── DefaultMessage + └── PreviousVersionModal (conditional, for text changes) +``` + +## Key Features + +### 1. Testable Utilities + +#### `extractChangeDetails(details)` + +Converts GraphQL StringMapEntry[] to Record + +- **Input**: Array of {key, value} objects +- **Output**: Simple key-value object +- **Testable**: Pure function, no dependencies +- **Location**: `utils/changeUtils.ts` + +```typescript +// Easy to unit test +const details = [ + { key: 'oldTitle', value: 'Old' }, + { key: 'newTitle', value: 'New' }, +]; +const result = extractChangeDetails(details); +// { oldTitle: 'Old', newTitle: 'New' } +``` + +#### `getActorDisplayName(actor, getDisplayName)` + +Extracts actor display name with fallback + +- **Input**: Actor object + display name function +- **Output**: String name or 'System' +- **Testable**: Can mock the getDisplayName function +- **Location**: `utils/changeUtils.ts` + +### 2. Custom Hooks + +#### `useParentDocumentTitle(parentUrn)` + +Fetches parent document title with proper state handling + +- **Returns**: `{ title, loading, error }` +- **States**: + - Loading: returns `'...'` + - Error: returns `'Unknown Document'` + logs error + - Success: returns actual title + - No URN: returns `'...'` +- **Testable**: Can be tested with mocked GraphQL queries +- **Location**: `hooks/useParentDocumentTitle.ts` + +### 3. Error & Loading States + +| Component | Loading Handled | Error Handled | Notes | +| ------------------------- | --------------- | ------------- | ------------------------------------------------------ | +| `useParentDocumentTitle` | ✅ | ✅ | Shows '...' while loading, 'Unknown Document' on error | +| `PreviousVersionModal` | ✅ | ✅ | Disables buttons during restore, shows error message | +| `DocumentHistoryTimeline` | ✅ | ✅ | Shows loading skeleton, empty state message | + +### 4. User Experience + +#### Timestamps + +- **Relative**: "2 minutes ago", "3 days ago" (using dayjs.fromNow()) +- **Absolute**: Hover shows full timestamp "March 15, 2024 2:30:45 PM" +- **Implementation**: Popover with formatted timestamp + +#### Change Messages + +All messages follow the pattern: `{ActorName} {action} {target}` + +- ✅ Actor names in bold +- ✅ Important values (titles, parent names) in bold +- ✅ Interactive elements (links) styled appropriately + +#### Restore Flow + +1. Click "See previous version" → Opens modal +2. Review old content +3. Click "Restore" → Shows confirmation +4. Confirm → Restores content + refetches + closes modals +5. Error handling → Shows error message, keeps modals open + +## Adding a New Change Type + +See `changeMessages/README.md` for detailed instructions. Quick summary: + +1. **Update GraphQL schema** (backend) +2. **Update event generator** (backend, if needed) +3. **Create message component**: + ```typescript + export const MyNewMessage: React.FC = ({ actorName, details }) => ( + + {actorName} did something + + ); + ``` +4. **Add to router**: + ```typescript + case DocumentChangeType.MyNewType: + return ; + ``` +5. **Run `yarn generate`** to update types + +## GraphQL Integration + +### Query + +```graphql +query getDocumentChangeHistory($urn: String!, $limit: Int) { + document(urn: $urn) { + changeHistory(limit: $limit) { + changeType + description + actor { + urn + type + username + info + editableProperties + } + timestamp + details { + key + value + } + } + } +} +``` + +### Mutation (for restore) + +```graphql +mutation updateDocumentContents($input: UpdateDocumentContentsInput!) { + updateDocumentContents(input: $input) +} +``` + +## Testing Strategy + +### Unit Tests (Recommended) + +- `changeUtils.ts` functions (pure functions, easy to test) +- Individual message components (can pass mock data) +- `useParentDocumentTitle` hook (mock GraphQL responses) + +### Integration Tests + +- Full timeline rendering with mock data +- Restore flow (mock mutations) +- Error handling scenarios + +### E2E Tests + +- Create document → view history → see creation event +- Edit title → view history → see title change with new title +- Edit content → view history → see content change → restore previous version +- Move document → view history → see move with parent name + +## Performance Considerations + +### Optimizations + +1. **Conditional Query**: Parent title only fetched when needed +2. **Skip Flag**: GraphQL queries skipped when URN is empty +3. **Memoization**: `details` object memoized in component +4. **Code Splitting**: Large modal only loaded when needed + +### Data Loading + +- Timeline loads up to 100 most recent changes +- Each change with a parent triggers a separate query (could be optimized with batching if needed) +- Modal content is part of change details (no additional fetch needed) + +## Future Enhancements + +### Potential Improvements + +1. **Diff View**: Show inline diffs for content changes (not just full previous version) +2. **Batch Parent Queries**: Load all parent titles in one query +3. **Infinite Scroll**: Load more changes on demand +4. **Filtering**: Filter by change type, date range, or actor +5. **Comparison**: Compare any two versions side-by-side +6. **Annotations**: Add comments to specific changes + +### Extensibility Points + +- Add new change types in `ChangeMessageComponents.tsx` +- Add new data fetching hooks in `hooks/` +- Add new utilities in `utils/` +- Customize message rendering per change type + +## Code Quality Metrics + +- **Lines of Code**: ~600 total +- **Number of Components**: 11 +- **Number of Hooks**: 1 custom +- **Number of Utilities**: 2 +- **TypeScript Coverage**: 100% +- **Linting Errors**: 0 +- **Type Errors**: 0 + +## Dependencies + +### Internal + +- `@app/entityV2/document` - Document queries and mutations +- `@app/useEntityRegistry` - Entity display names +- `@app/sharedV2/modals/ConfirmationModal` - Confirmation dialogs +- `@src/alchemy-components` - UI components (Timeline, Popover, Button, etc.) + +### External + +- `dayjs` - Date formatting and relative time +- `antd` - Modal component +- `react` - Component framework +- `styled-components` - Styling + +## Maintenance Guide + +### Common Tasks + +**Update message text**: Edit the appropriate component in `changeMessages/ChangeMessageComponents.tsx` + +**Change timestamp format**: Modify `dayjs.format()` calls in `DocumentChangeTimelineContent.tsx` + +**Add new detail field**: Update backend event generator → regenerate types → use in message component + +**Customize styling**: Update styled-components in respective files + +### Debugging + +**Timeline not showing**: Check GraphQL query response in DevTools Network tab + +**Parent name shows '...'**: Check if parent URN is valid and document exists + +**Restore not working**: Check mutation response and refetch behavior + +**Wrong actor name**: Verify actor data in change history response diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/changeMessages/README.md b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/changeMessages/README.md new file mode 100644 index 00000000..144eabf8 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/entityV2/document/changeHistory/changeMessages/README.md @@ -0,0 +1,178 @@ +--- +title: Document Change Messages +slug: /datahub-web-react/src/app/entityv2/document/changehistory/changemessages +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/entityV2/document/changeHistory/changeMessages/README.md +--- +# Document Change Messages + +This directory contains the individual components for rendering different types of document changes in the change history timeline. + +## Architecture + +- **`ChangeMessageComponents.tsx`**: Contains all individual change message components and the main router +- Each change type has its own component for clean separation and maintainability + +## Adding a New Change Type + +Follow these steps to add support for a new document change type: + +### 1. Update Backend GraphQL Schema + +First, add your new change type to the enum in `datahub-graphql-core/src/main/resources/knowledge.graphql`: + +```graphql +enum DocumentChangeType { + CREATED + TITLE_CHANGED + TEXT_CHANGED + # ... existing types ... + MY_NEW_CHANGE_TYPE # Add your new type here +} +``` + +### 2. Update Backend Event Generator (if needed) + +If you need to capture additional data, update `DocumentInfoChangeEventGenerator.java` to include relevant parameters in the change event. + +### 3. Create a New Message Component + +In `ChangeMessageComponents.tsx`, create a new component: + +```typescript +export const MyNewChangeMessage: React.FC = ({ actorName, details }) => ( + + {actorName} performed a new action on {details.someField} + +); +``` + +**Tips:** + +- Use `` for bold text (actor names, titles, etc.) +- Access change details via the `details` prop +- If you need to fetch additional data, use GraphQL hooks (see `ParentChangedMessage` for example) +- If you need user interaction (like "See previous version"), accept an `onAction` prop + +### 4. Add to the Router + +In the `ChangeMessage` component's switch statement, add your new case: + +```typescript +export const ChangeMessage: React.FC = ({ + changeType, + actorName, + details, + description, + onSeeVersion, +}) => { + switch (changeType) { + // ... existing cases ... + + case DocumentChangeType.MyNewChangeType: + return ; + + // ... rest of cases ... + } +}; +``` + +### 5. Update Frontend Types + +Run `yarn generate` to regenerate GraphQL types from the schema. + +### 6. Test! + +1. Create a test document +2. Perform the action that triggers your new change type +3. Open the change history drawer +4. Verify your message displays correctly + +## Component Guidelines + +### Styling + +- Use the provided `ActionText` for the message wrapper +- Use `ActorName` for bold text (names, titles, important values) +- Use `SeeVersionLink` for clickable links + +### Message Format + +Follow this pattern for consistency: + +``` +{ActorName} {action verb} {optional object} +``` + +Examples: + +- `John Doe created document` +- `Jane Smith changed title to New Title` +- `Bob Johnson moved document to Marketing Folder` + +### Data Fetching + +If you need to fetch additional data: + +```typescript +const { data } = useGetSomeQuery({ + variables: { urn: details.someUrn }, + skip: !details.someUrn, // Only fetch when needed +}); +``` + +## Examples + +### Simple Message (no additional data needed) + +```typescript +export const CreatedMessage: React.FC = ({ actorName }) => ( + + {actorName} created document + +); +``` + +### Message with Details + +```typescript +export const TitleChangedMessage: React.FC = ({ actorName, details }) => ( + + {actorName} changed title to {details.newTitle} + +); +``` + +### Message with Data Fetching + +```typescript +export const ParentChangedMessage: React.FC = ({ actorName, details }) => { + const { data } = useGetDocumentQuery({ + variables: { urn: details.newParent || '' }, + skip: !details.newParent, + }); + + const parentTitle = data?.document?.info?.title || 'Unknown'; + + return ( + + {actorName} moved document to {parentTitle} + + ); +}; +``` + +### Message with User Interaction + +```typescript +interface MyMessageProps extends BaseChangeMessageProps { + onAction: () => void; +} + +export const MyMessage: React.FC = ({ actorName, onAction }) => ( + + {actorName} did something.{' '} + Click here + +); +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/mfeframework/README-MFE.md b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/mfeframework/README-MFE.md new file mode 100644 index 00000000..2fedfd60 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/datahub-web-react/src/app/mfeframework/README-MFE.md @@ -0,0 +1,122 @@ +--- +title: Micro-Frontends in DataHub +slug: /datahub-web-react/src/app/mfeframework/readme-mfe +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/mfeframework/README-MFE.md +--- +# Micro-Frontends in DataHub + +DataHub now supports hosting micro-frontends (MFEs), which can be easily configured via YAML files. Each MFE must expose a `remoteEntry.js` file using [Module Federation](https://webpack.js.org/concepts/module-federation/). + +> **Note:** Exporting your `` component is not sufficient. +> You must export a `mount` function that accepts a DOM element and renders your app inside it. +> This approach allows DataHub to support MFEs built with any framework (React, Angular, Vue, Svelte, etc.). + +## Getting Started Locally + +To get started, refer to the [Module Federation documentation](https://webpack.js.org/concepts/module-federation/), online tutorials (such as [this example](https://medium.com/paloit/a-beginners-guide-to-micro-frontends-with-webpack-module-federation-712f3855f813)), or use your preferred AI tool to write and expose your app's `mount()` function via a remote entry. + +A variety of Module Federation examples are available [here](https://github.com/module-federation/module-federation-examples/). +Most examples include both a "host app" and a "remote app." For DataHub, you only need to implement the "remote app," as DataHub acts as the host. + +### Edit the Configuration File + +Edit [`mfe.config.local.yaml`](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/mfeframework/datahub-frontend/conf/mfe.config.local.yaml) to resemble the following: + +```yaml +subNavigationMode: false +microFrontends: + - id: HelloWorld + label: HelloWorld DEV + path: /helloworld-mfe + remoteEntry: http://localhost:3002/remoteEntry.js + module: helloWorldMFE/mount + flags: + enabled: true + showInNav: true + navIcon: HandWaving +``` + +To ensure compatibility between the DataHub MFE configuration above and your actual MFE, verify the following: + +- The HelloWorld app is running on `localhost:3002`. +- The HelloWorld Webpack configuration includes: + +``` + plugins: [ + // ...other plugins... + new ModuleFederationPlugin({ + name: 'helloWorldMFE', + filename: 'remoteEntry.js', + exposes: { + './mount': './src/whatever/sub/path/mount.tsx', + }, + // ...other options... + }), + // ...other plugins... + ] +``` + +### Build the `datahub-frontend` Binary + +```shell +cd datahub-frontend +../gradlew build +``` + +### Run the Binary + +```shell +cd run +./run-local-frontend +``` + +By default, the above script ([run-local-frontend](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/mfeframework/datahub-frontend/run/run-local-frontend)) uses the [`frontend.env`](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/mfeframework/datahub-frontend/run/frontend.env) file, which sets the `MFE_CONFIG_FILE_PATH` environment variable to point to your edited [`mfe.config.local.yaml`](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/mfeframework/datahub-frontend/conf/mfe.config.local.yaml). + +Additionally, this section in [`frontend.env`](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/mfeframework/datahub-frontend/run/frontend.env): + +``` +PORT=9002 +``` + +ensures the app is available at [http://localhost:9002](http://localhost:9002). + +### Start Supporting Services + +As described in the [DataHub Quickstart Guide](/docs/quickstart), you will need to start several supporting services to use the DataHub GUI. + +### Test Your MFE + +Navigate to [http://localhost:9002](http://localhost:9002). +You should see a waving hand menu item in the left navigation bar. + +## Deploying to Kubernetes + +Suppose HelloWorld is deployed at `https://mydomain-dev.com/helloworld/remoteEntry.js`. + +Edit [`mfe.config.dev.yaml`](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/mfeframework/datahub-frontend/conf/mfe.config.dev.yaml). This file will be similar to your local configuration, but update the `remoteEntry` field: + +``` +remoteEntry: https://mydomain-dev.com/helloworld/remoteEntry.js +``` + +**Note:** The above file name and location is just an example. You may create any file and place it in a separate configuration repository, depending on your organization's practices. + +In your Kubernetes YAML, ensure the environment variable `MFE_CONFIG_FILE_PATH` points to your configuration via volumes and volume mounts: + +```yaml + env: + - name: MFE_CONFIG_FILE_PATH + value: /mfeconfig/mfe.config.dev.yaml + volumeMounts: + - name: mfe-config + mountPath: /mfeconfig + readOnly: true + +volumes: + - name: mfe-config + configMap: + name: datahub-mfe-config +``` + +Then include the file in the ConfigMap following Kubernetes best practices. diff --git a/docs-archive/versioned_docs/version-1.5.0/docker/README.md b/docs-archive/versioned_docs/version-1.5.0/docker/README.md new file mode 100644 index 00000000..28be578d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docker/README.md @@ -0,0 +1,302 @@ +--- +title: Deploying with Docker +hide_title: true +sidebar_label: Deploying with Docker +slug: /docker +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docker/README.md' +--- + +# Docker Images + +## Prerequisites + +You need to install [docker](https://docs.docker.com/install/) and +[docker-compose](https://docs.docker.com/compose/install/) (if using Linux; on Windows and Mac compose is included with +Docker Desktop). + +Make sure to allocate enough hardware resources for Docker engine. Tested & confirmed config: 2 CPUs, 8GB RAM, 2GB Swap +area. + +_If you prefer not to use Docker Desktop (which requires a license for commercial use), you can opt for free and open-source alternatives such as [Podman Desktop](https://podman-desktop.io/) or [Rancher Desktop](https://rancherdesktop.io/). To configure them, you can add the following aliases to your `~/.bashrc` file:_ + +```bash +# podman +alias docker=podman +alias docker-compose="podman compose" + +# Rancher (or nerdctl) +alias docker=nerdctl +alias docker-compose="nerdctl compose" +``` + +## Quickstart + +The easiest way to bring up and test DataHub is using DataHub [Docker](https://www.docker.com) images +which are continuously deployed to [Docker Hub](https://hub.docker.com/u/acryldata) with every commit to repository. + +You can easily download and run all these images and their dependencies with our +[quick start guide](../docs/quickstart.md). + +DataHub Docker Images: + +Do not use `latest` or `debug` tags for any of the image as those are not supported and present only due to legacy reasons. Please use `head` or tags specific for versions like `v0.8.40`. For production we recommend using version specific tags not `head`. + +- [acryldata/datahub-ingestion](https://hub.docker.com/r/acryldata/datahub-ingestion/) +- [acryldata/datahub-gms](https://hub.docker.com/repository/docker/acryldata/datahub-gms/) +- [acryldata/datahub-frontend-react](https://hub.docker.com/repository/docker/acryldata/datahub-frontend-react/) +- [acryldata/datahub-mae-consumer](https://hub.docker.com/repository/docker/acryldata/datahub-mae-consumer/) +- [acryldata/datahub-mce-consumer](https://hub.docker.com/repository/docker/acryldata/datahub-mce-consumer/) +- [acryldata/datahub-upgrade](https://hub.docker.com/r/acryldata/datahub-upgrade/) (runs SystemUpdate; performs SQL and search index setup when `DATAHUB_SQL_SETUP_ENABLED=true`) +- [acryldata/datahub-actions](https://hub.docker.com/r/acryldata/datahub-actions). Do not use `acryldata/acryl-datahub-actions` as that is deprecated and no longer used. + +## Image Variants + +Both `datahub-ingestion` and `datahub-actions` are available only as **full**, **slim**, and **locked** (Ubuntu-based; no Alpine variants). + +| Variant | Base OS | Image Size | Use Case | +| ---------------- | ------------ | ---------- | --------------------------------------- | +| `full` (default) | Ubuntu 24.04 | Largest | All connectors, maximum compatibility | +| `slim` | Ubuntu 24.04 | Medium | Common connectors, good balance | +| `locked` | Ubuntu 24.04 | Medium | Air-gapped environments (pypi disabled) | + +### Variant Tag Format + +``` +acryldata/datahub-ingestion:v0.x.y # full (default) +acryldata/datahub-ingestion:v0.x.y-slim # slim +acryldata/datahub-ingestion:v0.x.y-locked # locked +``` + +### datahub-ingestion Feature Matrix + +| Feature | full | slim | locked | +| --------------------- | :--: | :--: | :----: | +| Core CLI & REST/Kafka | Yes | Yes | Yes | +| S3 / GCS / Azure Blob | Yes | Yes | Yes | +| Snowflake | Yes | Yes | - | +| BigQuery | Yes | Yes | - | +| Redshift | Yes | Yes | - | +| MySQL / PostgreSQL | Yes | Yes | - | +| ClickHouse | Yes | Yes | - | +| dbt | Yes | Yes | - | +| Looker / LookML | Yes | Yes | - | +| Tableau / PowerBI | Yes | Yes | - | +| Superset | Yes | Yes | - | +| Glue | Yes | Yes | - | +| Spark lineage (JRE) | Yes | - | - | +| Oracle client | Yes | - | - | +| Runtime `pip install` | Yes | Yes | - | + +### datahub-actions Feature Matrix + +| Feature | full | slim | locked | +| ---------------------------- | :--: | :--: | :----: | +| Core actions | Yes | Yes | Yes | +| Kafka / Executor | Yes | Yes | Yes | +| Slack / Teams | Yes | Yes | Yes | +| Tag / Term / Doc propagation | Yes | Yes | Yes | +| Snowflake tag propagation | Yes | Yes | Yes | +| Bundled CLI venvs | Yes | Yes | Yes | +| Runtime `pip install` | Yes | Yes | - | + +### CI Testing Coverage + +| Image Variant | Tested in CI | Notes | +| ------------- | :----------: | --------------------------- | +| `full` | Yes | Smoke tests run on every PR | +| `slim` | Yes | Smoke tests run on every PR | +| `locked` | Build only | | + +### Choosing the Right Variant + +- **`full`**: Use when you need maximum connector coverage or aren't sure what you'll need +- **`slim`**: Recommended for most production deployments with standard cloud data stacks +- **`locked`**: Required for air-gapped environments where runtime package installation is prohibited + +Dependencies: + +- [Elasticsearch](https://github.com/datahub-project/datahub/blob/master/docker/elasticsearch) (or OpenSearch) +- [MySQL](https://github.com/datahub-project/datahub/blob/master/docker/mysql) (or PostgreSQL) +- [(Optional) Neo4j](https://github.com/datahub-project/datahub/blob/master/docker/neo4j) + +SQL and search index setup are performed by the system update job (datahub-upgrade with `-u SystemUpdate`) when backing services are healthy; no separate setup containers are required. + +### Ingesting demo data. + +If you want to test ingesting some data once DataHub is up, use the `./docker/ingestion/ingestion.sh` script or `datahub docker ingest-sample-data`. See the [quickstart guide](../docs/quickstart.md) for more details. + +## Using Docker Images During Development + +See [Using Docker Images During Development](../docs/docker/development.md). + +## Building And Deploying Docker Images + +We use GitHub Actions to build and continuously deploy our images. There should be no need to do this manually; a +successful release on Github will automatically publish the images. + +### Building images + +> This is **not** our recommended development flow and most developers should be following the +> [Using Docker Images During Development](../docs/docker/development.md) guide. + +To build the full images (that we are going to publish), you need to run the following: + +``` +COMPOSE_DOCKER_CLI_BUILD=1 DOCKER_BUILDKIT=1 docker compose -p datahub build +``` + +This is because we're relying on builtkit for multistage builds. It does not hurt also set `DATAHUB_VERSION` to +something unique. + +### Community Built Images + +As the open source project grows, community members would like to contribute additions to the docker images. Not all contributions to the images can be accepted because those changes are not useful for all community members, it will increase build times, add dependencies and possible security vulns. In those cases this section can be used to point to `Dockerfiles` hosted by the community which build on top of the images published by the DataHub core team along with any container registry links where the result of those images are maintained. + +# DataHub Docker Nuke Tasks + +This document describes the nuke task system for cleaning up DataHub Docker containers and volumes. + +## Overview + +The nuke tasks provide a way to completely remove DataHub containers and volumes for different project namespaces. This is useful for: + +- Cleaning up test environments +- Resetting development setups +- Isolating different project instances +- Troubleshooting container issues + +## Available Tasks + +### All Configurations Have Nuke Tasks + +Every quickstart configuration automatically gets a nuke task for targeted cleanup: + +- **`quickstartNuke`** - Removes containers and volumes for the default project namespace (`datahub`) +- **`quickstartDebugNuke`** - Removes containers and volumes for the debug configuration (`datahub`) +- **`quickstartCypressNuke`** - Removes containers and volumes for the cypress configuration (`dh-cypress`) +- **`quickstartDebugMinNuke`** - Removes containers and volumes for the debug-min configuration (`datahub`) +- **`quickstartDebugConsumersNuke`** - Removes containers and volumes for the debug-consumers configuration (`datahub`) +- **`quickstartPgNuke`** - Removes containers and volumes for the postgres configuration (`datahub`) +- **`quickstartPgDebugNuke`** - Removes containers and volumes for the debug-postgres configuration (`datahub`) +- **`quickstartSlimNuke`** - Removes containers and volumes for the backend configuration (`datahub`) +- **`quickstartSparkNuke`** - Removes containers and volumes for the spark configuration (`datahub`) +- **`quickstartStorageNuke`** - Removes containers and volumes for the storage configuration (`datahub`) +- **`quickstartBackendDebugNuke`** - Removes containers and volumes for the backend-debug configuration (`datahub`) + +### Project Namespace Behavior + +- **Default project namespace (`datahub`)**: Most configurations use this, so their nuke tasks will clean up containers in the same namespace +- **Custom project namespace (`dh-cypress`)**: The cypress configuration uses its own namespace for isolation + +## Usage + +### Basic Usage + +```bash +# Remove containers and volumes for specific configurations +./gradlew quickstartDebugNuke # For debug configuration +./gradlew quickstartCypressNuke # For cypress configuration +./gradlew quickstartDebugMinNuke # For debug-min configuration +./gradlew quickstartPgNuke # For postgres configuration + +# For general cleanup of all containers +./gradlew quickstartDown +``` + +### When to Use Each Task + +- **Use specific nuke tasks when:** + + - You want to clean up a specific configuration environment + - You need targeted cleanup without affecting other configurations + - You're working with a particular development setup (debug, postgres, cypress, etc.) + +- **Use `quickstartDown` when:** + - You want to stop all running containers + - You need a general cleanup option + - You want to ensure all environments are stopped + +## How It Works + +1. **Volume Management**: Each nuke task sets `removeVolumes = true` for relevant configurations +2. **Container Cleanup**: Tasks are finalized by appropriate `ComposeDownForced` operations +3. **Project Isolation**: Each task operates within its own Docker Compose project namespace +4. **Configuration Respect**: Tasks respect the `projectName` settings in `quickstart_configs` + +## Configuration + +The nuke tasks are automatically generated based on the `quickstart_configs` in `docker/build.gradle`. To add a new nuke task: + +1. Add a configuration to `quickstart_configs`: + + ```gradle + 'quickstartCustom': [ + profile: 'debug', + modules: [...], + // Optional: custom project name for isolation + additionalConfig: [ + projectName: 'dh-custom' + ] + ] + ``` + +2. The task `quickstartCustomNuke` will be automatically created + +## Troubleshooting + +### Task Not Found + +- Ensure the configuration exists in `quickstart_configs` +- Check that the task name follows the pattern `{configName}Nuke` + +### Containers Not Removed + +- Verify the project namespace is correct +- Check that the configuration has the right `projectName` setting +- Ensure the task is targeting the correct `ComposeDownForced` operations + +### Volume Persistence + +- Check if `preserveVolumes` is set to `true` in the configuration +- Verify the `removeVolumes` setting is properly applied + +## Related Commands + +- **Start services**: `./gradlew quickstartDebug`, `./gradlew quickstartCypress` +- **Stop services**: `./gradlew quickstartDown` +- **Reload services**: `./gradlew debugReload`, `./gradlew cypressReload` + +## Examples + +### Complete Development Environment Reset + +```bash +# Clean up specific debug environment +./gradlew quickstartDebugNuke + +# Start fresh +./gradlew quickstartDebug +``` + +### Cypress Environment Isolation + +```bash +# Clean up cypress environment +./gradlew quickstartCypressNuke + +# Start fresh cypress environment +./gradlew quickstartCypress +``` + +### Mixed Environment Management + +```bash +# Clean up only cypress (leaving main environment intact) +./gradlew quickstartCypressNuke + +# Clean up only debug environment (leaving cypress intact) +./gradlew quickstartDebugNuke + +# Clean up only postgres environment (leaving others intact) +./gradlew quickstartPgNuke +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docker/airflow/local_airflow.md b/docs-archive/versioned_docs/version-1.5.0/docker/airflow/local_airflow.md new file mode 100644 index 00000000..47e29fa2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docker/airflow/local_airflow.md @@ -0,0 +1,208 @@ +--- +title: Running Airflow locally with DataHub +slug: /docker/airflow/local_airflow +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docker/airflow/local_airflow.md +--- +:::caution + +This guide is currently unmaintained. As of 0.10.0 the container described is not published alongside the DataHub CLI. If you'd like to use it, please reach out to us on the [community slack.](docs/slack.md) + +::: + +# Running Airflow locally with DataHub + +## Introduction + +This document describes how you can run Airflow side-by-side with DataHub's quickstart docker images to test out Airflow lineage with DataHub. +This offers a much easier way to try out Airflow with DataHub, compared to configuring containers by hand, setting up configurations and networking connectivity between the two systems. + +## Prerequisites + +- Docker: ensure that you have a working Docker installation and you have at least 8GB of memory to allocate to both Airflow and DataHub combined. + +``` +docker info | grep Memory + +> Total Memory: 7.775GiB +``` + +- Quickstart: Ensure that you followed [quickstart](../../docs/quickstart.md) to get DataHub up and running. + +## Step 1: Set up your Airflow area + +- Create an area to host your airflow installation +- Download the docker-compose file hosted in DataHub's repo in that directory +- Download a sample dag to use for testing Airflow lineage + +``` +mkdir -p airflow_install +cd airflow_install +# Download docker-compose file +curl -L 'https://raw.githubusercontent.com/datahub-project/datahub/master/docker/airflow/docker-compose.yaml' -o docker-compose.yaml +# Create dags directory +mkdir -p dags +# Download a sample DAG +curl -L 'https://raw.githubusercontent.com/datahub-project/datahub/master/metadata-ingestion/src/datahub_provider/example_dags/lineage_backend_demo.py' -o dags/lineage_backend_demo.py +``` + +### What is different between this docker-compose file and the official Apache Airflow docker compose file? + +- This docker-compose file is derived from the [official Airflow docker-compose file](https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html#docker-compose-yaml) but makes a few critical changes to make interoperability with DataHub seamless. +- The Airflow image in this docker compose file extends the [base Apache Airflow docker image](https://airflow.apache.org/docs/docker-stack/index.html) and is published [here](https://hub.docker.com/r/acryldata/airflow-datahub). It includes the latest `acryl-datahub` pip package installed by default so you don't need to install it yourself. +- This docker-compose file sets up the networking so that + - the Airflow containers can talk to the DataHub containers through the `datahub_network` bridge interface. + - Modifies the port-forwarding to map the Airflow Webserver port `8080` to port `58080` on the localhost (to avoid conflicts with DataHub metadata-service, which is mapped to 8080 by default) +- This docker-compose file also sets up the ENV variables to configure Airflow's Lineage Backend to talk to DataHub. (Look for the `AIRFLOW__LINEAGE__BACKEND` and `AIRFLOW__LINEAGE__DATAHUB_KWARGS` variables) + +## Step 2: Bring up Airflow + +First you need to initialize airflow in order to create initial database tables and the initial airflow user. + +``` +docker compose up airflow-init +``` + +You should see the following final initialization message + +``` +airflow-init_1 | Admin user airflow created +airflow-init_1 | 2.1.3 +airflow_install_airflow-init_1 exited with code 0 + +``` + +Afterwards you need to start the airflow docker compose + +``` +docker compose up +``` + +You should see a host of messages as Airflow starts up. + +``` +Container airflow_deploy_airflow-scheduler_1 Started 15.7s +Attaching to airflow-init_1, airflow-scheduler_1, airflow-webserver_1, airflow-worker_1, flower_1, postgres_1, redis_1 +airflow-worker_1 | BACKEND=redis +airflow-worker_1 | DB_HOST=redis +airflow-worker_1 | DB_PORT=6379 +airflow-worker_1 | +airflow-webserver_1 | +airflow-init_1 | DB: postgresql+psycopg2://airflow:***@postgres/airflow +airflow-init_1 | [2021-08-31 20:02:07,534] {db.py:702} INFO - Creating tables +airflow-init_1 | INFO [alembic.runtime.migration] Context impl PostgresqlImpl. +airflow-init_1 | INFO [alembic.runtime.migration] Will assume transactional DDL. +airflow-scheduler_1 | ____________ _____________ +airflow-scheduler_1 | ____ |__( )_________ __/__ /________ __ +airflow-scheduler_1 | ____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / / +airflow-scheduler_1 | ___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ / +airflow-scheduler_1 | _/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/ +airflow-scheduler_1 | [2021-08-31 20:02:07,736] {scheduler_job.py:661} INFO - Starting the scheduler +airflow-scheduler_1 | [2021-08-31 20:02:07,736] {scheduler_job.py:666} INFO - Processing each file at most -1 times +airflow-scheduler_1 | [2021-08-31 20:02:07,915] {manager.py:254} INFO - Launched DagFileProcessorManager with pid: 25 +airflow-scheduler_1 | [2021-08-31 20:02:07,918] {scheduler_job.py:1197} INFO - Resetting orphaned tasks for active dag runs +airflow-scheduler_1 | [2021-08-31 20:02:07,923] {settings.py:51} INFO - Configured default timezone Timezone('UTC') +flower_1 | +airflow-worker_1 | * Serving Flask app "airflow.utils.serve_logs" (lazy loading) +airflow-worker_1 | * Environment: production +airflow-worker_1 | WARNING: This is a development server. Do not use it in a production deployment. +airflow-worker_1 | Use a production WSGI server instead. +airflow-worker_1 | * Debug mode: off +airflow-worker_1 | [2021-08-31 20:02:09,283] {_internal.py:113} INFO - * Running on http://0.0.0.0:8793/ (Press CTRL+C to quit) +flower_1 | BACKEND=redis +flower_1 | DB_HOST=redis +flower_1 | DB_PORT=6379 +flower_1 | +``` + +Finally, Airflow should be healthy and up on port 58080. Navigate to [http://localhost:58080](http://localhost:58080) to confirm and find your Airflow webserver. +The default username and password is: + +``` +airflow:airflow +``` + +## Step 3: Register DataHub connection (hook) to Airflow + +``` +docker exec -it `docker ps | grep webserver | cut -d " " -f 1` airflow connections add --conn-type 'datahub_rest' 'datahub_rest_default' --conn-host 'http://datahub-gms:8080' +``` + +### Result + +``` +Successfully added `conn_id`=datahub_rest_default : datahub_rest://:@http://datahub-gms:8080: +``` + +### What is the above command doing? + +- Find the container running airflow webserver: `docker ps | grep webserver | cut -d " " -f 1` +- Running the `airflow connections add ...` command inside that container to register the `datahub_rest` connection type and connect it to the `datahub-gms` host on port 8080. +- Note: This is what requires Airflow to be able to connect to `datahub-gms` the host (this is the container running datahub-gms image) and this is why we needed to connect the Airflow containers to the `datahub_network` using our custom docker-compose file. + +## Step 4: Find the DAGs and run it + +Navigate the Airflow UI to find the sample Airflow dag we just brought in + +

+ +

+ +By default, Airflow loads all DAG-s in paused status. Unpause the sample DAG to use it. + +

+ +

+ +

+ +

+ +Then trigger the DAG to run. + +

+ +

+ +After the DAG runs successfully, go over to your DataHub instance to see the Pipeline and navigate its lineage. + +

+ +

+ +

+ +

+ +

+ +

+ +

+ +

+ +## TroubleShooting + +Most issues are related to connectivity between Airflow and DataHub. + +Here is how you can debug them. + +

+ +

+ +

+ +

+ +In this case, clearly the connection `datahub-rest` has not been registered. Looks like we forgot to register the connection with Airflow! +Let's execute Step 4 to register the datahub connection with Airflow. + +In case the connection was registered successfully but you are still seeing `Failed to establish a new connection`, check if the connection is `http://datahub-gms:8080` and not `http://localhost:8080`. + +After re-running the DAG, we see success! + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docker/datahub-upgrade/README.md b/docs-archive/versioned_docs/version-1.5.0/docker/datahub-upgrade/README.md new file mode 100644 index 00000000..90d5153c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docker/datahub-upgrade/README.md @@ -0,0 +1,168 @@ +--- +title: DataHub Upgrade Docker Image +sidebar_label: Upgrade Docker Image +slug: /docker/datahub-upgrade +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docker/datahub-upgrade/README.md +--- +# DataHub Upgrade Docker Image + +This container is used to automatically apply upgrades from one version of DataHub to another. It contains +a set of executable jobs, which can be used to perform various types of system maintenance on-demand. + +It also supports regularly upgrade tasks that need to occur between versions of DataHub, and should be run +each time DataHub is deployed. More on this below. (TODO) + +## Supported Upgrades + +The following jobs are supported: + +1. **SystemUpdate**: Performs any tasks required to update to a new version of DataHub. For example, applying new configurations to the search & graph indexes, ingesting default settings, and more. Once completed, emits a message to the DataHub Upgrade History Kafka (`DataHubUpgradeHistory_v1`) topic, which signals to other pods that DataHub is ready to start. + Note that this _must_ be executed any time the DataHub version is incremented before starting or restarting other system containers. Dependent services will wait until the Kafka message is emitted corresponding to the code they are running. + A unique "version id" is generated based on a combination of the a) embedded git tag corresponding to the version of DataHub running and b) an optional revision number, provided via the `DATAHUB_REVISION` environment variable. Helm uses + the latter to ensure that the system upgrade job is executed every single time a deployment of DataHub is performed, even if the container version has not changed. + Important: This job runs as a pre-install hook via the DataHub Helm Charts, i.e. before deploying new version tags for each container. + +2. **SystemUpdateBlocking**: Performs any _blocking_ tasks required to update to a new version of DataHub, as a subset of **SystemUpdate**. + +3. **SystemUpdateNonBlocking**: Performs any _nonblocking_ tasks required to update to a new version of DataHub, as a subset of **SystemUpdate**. + +4. **RestoreIndices**: Restores indices by fetching the latest version of each aspect and restating MetadataChangeLog events for each latest aspect. Arguments include: + + - _batchSize_ (Optional): The number of rows to migrate at a time. Defaults to 1000. + - _batchDelayMs_ (Optional): The number of milliseconds of delay between migrated batches. Used for rate limiting. Defaults to 250. + - _numThreads_ (Optional): The number of threads to use, defaults to 1. Note that this is not used if `urnBasedPagination` is true. + - _aspectName_ (Optional): The aspect name for producing events. + - _urn_ (Optional): The urn for producing events. + - _urnLike_ (Optional): The urn pattern for producing events, using `%` as a wild card + - _urnBasedPagination_ (Optional): Paginate the SQL results using the urn + aspect string instead of `OFFSET`. Defaults to false, + though should improve performance for large amounts of data. + +5. **RestoreBackup**: Restores the primary storage - the SQL document DB - from an available backup of the local database. Requires that the backup reader and backup are provided. Note that this does not also restore the secondary indexes, the graph or search storage. To do so, you should run the **RestoreIndices** upgrade job. + Arguments include: + + - _BACKUP_READER_ (Required): The backup reader to use to read and restore the db. The only backup reader currently supported is `LOCAL_PARQUET`, which requires a parquet-formatted backup file path to be specified via the `BACKUP_FILE_PATH` argument. + - _BACKUP_FILE_PATH_ (Required): The path of the backup file. If you are running in a container, this needs to the location where the backup file has been mounted into the container. + +6. **EvaluateTests**: Executes all Metadata Tests in batches. Running this job can slow down DataHub, and it in some cases requires full scans of the document db. Generally, it's recommended to configure this to run one time per day (which is the helm CronJob default). + Arguments include: + + - _batchSize_ (Optional): The number of assets to test at a time. Defaults to 1000. + - _batchDelayMs_ (Optional): The number of milliseconds of delay between evaluated asset batches. Used for rate limiting. Defaults to 250. + +7. (Legacy) **NoCodeDataMigration**: Performs a series of pre-flight qualification checks and then migrates metadata\*aspect table data + to metadata_aspect_v2 table. Arguments include: + + - _batchSize_ (Optional): The number of rows to migrate at a time. Defaults to 1000. + - _batchDelayMs_ (Optional): The number of milliseconds of delay between migrated batches. Used for rate limiting. Defaults to 250. + - _dbType_ (Optional): The target DB type. Valid values are `MYSQL`, `MARIA`, `POSTGRES`. Defaults to `MYSQL`. + + If you are using newer versions of DataHub (v1.0.0 or above), this upgrade job will not be relevant. + +8. (Legacy) **NoCodeDataMigrationCleanup**: Cleanses graph index, search index, and key-value store of legacy DataHub data (metadata_aspect table) once + the No Code Data Migration has completed successfully. No arguments. + + If you are using newer versions of DataHub (v1.0.0 or above), this upgrade job will not be relevant. + +## Environment Variables + +To run the `datahub-upgrade` container, some environment variables must be provided in order to tell the upgrade CLI +where the running DataHub containers reside. + +Below details the required configurations. By default, these configs are provided for local docker-compose deployments of +DataHub within `docker/datahub-upgrade/env/docker.env`. They assume that there is a Docker network called datahub_network +where the DataHub containers can be found. + +These are also the variables used when the provided `datahub-upgrade.sh` script is executed. To run the upgrade CLI for non-local deployments, +follow these steps: + +1. Define new ".env" variable to hold your environment variables. + +The following variables may be provided: + +```aidl +# Required Environment Variables +EBEAN_DATASOURCE_USERNAME=datahub +EBEAN_DATASOURCE_PASSWORD=datahub +EBEAN_DATASOURCE_HOST=:3306 +EBEAN_DATASOURCE_URL=jdbc:mysql://:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 +EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + +KAFKA_BOOTSTRAP_SERVER=:29092 +KAFKA_SCHEMAREGISTRY_URL=http://:8081 + +ELASTICSEARCH_HOST= +ELASTICSEARCH_PORT=9200 + +NEO4J_HOST=http://:7474 +NEO4J_URI=bolt:// +NEO4J_USERNAME=neo4j +NEO4J_PASSWORD=datahub + +DATAHUB_GMS_HOST=> +DATAHUB_GMS_PORT=8080 + +# Datahub protocol (default http) +# DATAHUB_GMS_PROTOCOL=http + +DATAHUB_MAE_CONSUMER_HOST= +DATAHUB_MAE_CONSUMER_PORT=9091 + +# Optional Arguments + +# Uncomment and set these to support SSL connection to Elasticsearch +# ELASTICSEARCH_USE_SSL= +# ELASTICSEARCH_SSL_PROTOCOL= +# ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL= +# ELASTICSEARCH_SSL_TRUSTSTORE_FILE= +# ELASTICSEARCH_SSL_TRUSTSTORE_TYPE= +# ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD= +# ELASTICSEARCH_SSL_KEYSTORE_FILE= +# ELASTICSEARCH_SSL_KEYSTORE_TYPE= +# ELASTICSEARCH_SSL_KEYSTORE_PASSWORD= +``` + +These variables tell the upgrade job how to connect to critical storage systems like Kafka, MySQL / Postgres, and Elasticsearch or OpenSearch. + +2. Pull (or build) & execute the `datahub-upgrade` container: + +```aidl +docker pull acryldata/datahub-upgrade:head && docker run --env-file *path-to-custom-env-file.env* acryldata/datahub-upgrade:head -u -a +``` + +## Command-Line Arguments + +### Selecting the Upgrade to Run + +The primary argument required by the datahub-upgrade container is the name of the upgrade to perform. This argument +can be specified using the `-u` flag when running the `datahub-upgrade` container. + +For example, to run the migration named "NoCodeDataMigration", you would do execute the following: + +```aidl +./datahub-upgrade.sh -u NoCodeDataMigration +``` + +OR + +```aidl +docker pull acryldata/datahub-upgrade:head && docker run --env-file env/docker.env acryldata/datahub-upgrade:head -u NoCodeDataMigration +``` + +### Provided Arguments for a Given Upgrade Job + +In addition to the required `-u` argument, each upgrade may require specific arguments. You can provide arguments to individual +upgrades using multiple `-a` arguments. + +For example, the NoCodeDataMigration upgrade provides 2 optional arguments detailed above: _batchSize_ and _batchDelayMs_. +To specify these, you can use a combination of `-a` arguments and of the form _argumentName=argumentValue_ as follows: + +```aidl +./datahub-upgrade.sh -u NoCodeDataMigration -a batchSize=500 -a batchDelayMs=1000 // Small batches with 1 second delay. +``` + +OR + +```aidl +docker pull acryldata/datahub-upgrade:head && docker run --env-file env/docker.env acryldata/datahub-upgrade:head -u NoCodeDataMigration -a batchSize=500 -a batchDelayMs=1000 +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/CODE_OF_CONDUCT.md b/docs-archive/versioned_docs/version-1.5.0/docs/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..a637a798 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,81 @@ +--- +title: Code of Conduct +slug: /code_of_conduct +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/CODE_OF_CONDUCT.md' +--- +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by direct messaging the project team on [Slack]. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[Slack]: https://datahub.com/slack +[homepage]: https://www.contributor-covenant.org +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/CONTRIBUTING.md b/docs-archive/versioned_docs/version-1.5.0/docs/CONTRIBUTING.md new file mode 100644 index 00000000..4801bdcf --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/CONTRIBUTING.md @@ -0,0 +1,67 @@ +--- +title: Contributing +slug: /contributing +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/CONTRIBUTING.md' +--- +# Contributing + +We always welcome contributions to help make DataHub better. Take a moment to read this document if you would like to contribute. + +## Provide Feedback + +Have ideas about how to make DataHub better? Head over to [DataHub Feature Requests](https://feature-requests.datahubproject.io/) and tell us all about it! + +Show your support for other requests by upvoting; stay up to date on progress by subscribing for updates via email. + +## Reporting Issues + +We use GitHub issues to track bug reports and submit pull requests. + +If you find a bug: + +1. Use the GitHub issue search to check whether the bug has already been reported. + +1. If the issue has been fixed, try to reproduce the issue using the latest master branch of the repository. + +1. If the issue still reproduces or has not yet been reported, try to isolate the problem before opening an issue. + +## Submitting a Request For Comment (RFC) + +If you have a substantial feature or a design discussion that you'd like to have with the community, follow the RFC process outlined [here](./rfc.md). + +## Submitting a Pull Request (PR) + +Before you submit your Pull Request (PR), consider the following guidelines: + +- Search GitHub for an open or closed PR that relates to your submission. You don't want to duplicate effort. +- Open a pull request (PR) following [GitHub’s standard workflow](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork). +- Please make sure to follow our [PR Title Format](#pr-title-format) for clarity and consistency. +- PRs are squashed and merged, resulting in a single commit with the PR title as the commit message. +- If there are any breaking changes, potential downtime, deprecations, or big features, please add an update in [Updating DataHub under Next](how/updating-datahub.md). +- That's it! Thank you for your contribution! + +### PR Title Format + +``` +[optional scope]: +``` + +Example: + +``` +feat(parser): add ability to parse arrays +``` + +#### Type + +Must be one of the following: + +- _feat_: A new feature +- _fix_: A bug fix +- _refactor_: A code change that neither fixes a bug nor adds a feature +- _docs_: Documentation only changes +- _test_: Adding missing tests or correcting existing tests +- _perf_: A code change that improves performance +- _style_: Changes that do not affect the meaning of the code (whitespace, formatting, missing semicolons, etc.) +- _build_: Changes that affect the build system or external dependencies +- _ci_: Changes to our CI configuration files and scripts diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/PYSPARK.md b/docs-archive/versioned_docs/version-1.5.0/docs/PYSPARK.md new file mode 100644 index 00000000..f5953088 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/PYSPARK.md @@ -0,0 +1,325 @@ +--- +title: Optional PySpark Support for S3 Source +slug: /pyspark +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/PYSPARK.md' +--- +# Optional PySpark Support for S3 Source + +DataHub's S3 source now supports optional PySpark installation through the `s3-slim` variant. This allows users to choose a lightweight installation when data lake profiling is not needed. + +## Overview + +The S3 source includes PySpark by default for backward compatibility and profiling support. For users who only need metadata extraction without profiling, the `s3-slim` variant provides a ~500MB smaller installation. + +**Current implementation status:** + +- ✅ **S3**: SparkProfiler pattern fully implemented (optional PySpark) +- **ABS**: Not yet implemented (still requires PySpark for profiling) +- **Unity Catalog**: Not affected by this change (uses separate profiling mechanisms) +- **GCS**: Does not support profiling + +> **Note:** This change implements the SparkProfiler pattern for S3 only. The same pattern can be applied to other sources (ABS, etc.) in future PRs. + +## PySpark Version + +> **Current Version:** PySpark 3.5.x (3.5.6) +> +> PySpark 4.0 support is planned for a future release. Until then, all DataHub components use PySpark 3.5.x for compatibility and stability. + +## Installation Options + +### Standard Installation (includes PySpark) + +```bash +pip install 'acryl-datahub[s3]' # S3 with PySpark/profiling support +``` + +### Lightweight Installation (without PySpark) + +For installations where you don't need profiling capabilities and want to save ~500MB: + +```bash +pip install 'acryl-datahub[s3-slim]' # S3 without profiling (~500MB smaller) +``` + +**Recommendation:** Use `s3-slim` when profiling is not needed. + +The `data-lake-profiling` dependencies (included in standard `s3` by default): + +- `pyspark~=3.5.6` +- `pydeequ>=1.1.0` +- Profiling dependencies (cachetools) + +> **Note:** In a future major release (e.g., DataHub 2.0), the `s3-slim` variant may become the default, and PySpark will be truly optional. This current approach provides backward compatibility while giving users time to adapt. + +### What's Included + +**S3 source:** + +Standard `s3` extra: + +- ✅ Metadata extraction (schemas, tables, file listing) +- ✅ Data format detection (Parquet, Avro, CSV, JSON, etc.) +- ✅ Schema inference from files +- ✅ Table and column-level metadata +- ✅ Tags and properties extraction +- ✅ Data profiling (min/max, nulls, distinct counts) +- ✅ Data quality checks (PyDeequ-based) +- Includes: PySpark 3.5.6 + PyDeequ + +`s3-slim` variant: + +- ✅ All metadata features (same as above) +- ❌ Data profiling disabled +- No PySpark dependencies (~500MB smaller) + +## Feature Comparison + +| Feature | `s3-slim` | Standard `s3` | +| ------------------------ | ---------------- | -------------------------- | +| **Metadata extraction** | ✅ Full support | ✅ Full support | +| **Schema inference** | ✅ Full support | ✅ Full support | +| **Tags & properties** | ✅ Full support | ✅ Full support | +| **Data profiling** | ❌ Not available | ✅ Full profiling | +| **Installation size** | ~200MB | ~700MB | +| **Install time** | Fast | Slower (PySpark build) | +| **PySpark dependencies** | ❌ None | ✅ PySpark 3.5.6 + PyDeequ | + +## Configuration + +### With Standard Installation (PySpark included) + +When you install `acryl-datahub[s3]`, profiling works out of the box: + +```yaml +source: + type: s3 + config: + path_specs: + - include: s3://my-bucket/data/**/*.parquet + profiling: + enabled: true # Works seamlessly with standard installation + profile_table_level_only: false +``` + +### With Slim Installation (no PySpark) + +When you install `s3-slim`, disable profiling in your config: + +```yaml +source: + type: s3 + config: + path_specs: + - include: s3://my-bucket/data/**/*.parquet + profiling: + enabled: false # Required for s3-slim installation +``` + +**If you enable profiling with s3-slim installation**, you'll see a clear error message at runtime: + +``` +RuntimeError: PySpark is not installed, but is required for S3 profiling. +Please install with: pip install 'acryl-datahub[s3]' +``` + +## Developer Guide + +### Implementation Pattern + +The S3 source demonstrates the recommended pattern for isolating PySpark-dependent code. This pattern can be applied to ABS and other sources in future PRs. + +**Architecture (currently implemented for S3 only):** + +1. **Main source class** (`source.py`) - Contains no PySpark imports at module level +2. **Profiler class** (`profiling.py`) - Encapsulates all PySpark/PyDeequ logic in `SparkProfiler` class +3. **Conditional instantiation** - `SparkProfiler` created only when profiling is enabled +4. **TYPE_CHECKING imports** - Type annotations use TYPE_CHECKING block for optional dependencies + +**Key Benefits:** + +- ✅ Type safety preserved (mypy passes without issues) +- ✅ Proper code layer separation +- ✅ Works with both standard and `-slim` installations +- ✅ Clear error messages when dependencies missing +- ✅ Pattern can be reused for ABS and other sources + +**Example structure:** + +```python +# source.py +if TYPE_CHECKING: + from datahub.ingestion.source.s3.profiling import SparkProfiler + +class S3Source: + profiler: Optional["SparkProfiler"] + + def __init__(self, config, ctx): + if config.is_profiling_enabled(): + from datahub.ingestion.source.s3.profiling import SparkProfiler + self.profiler = SparkProfiler(...) + else: + self.profiler = None +``` + +```python +# profiling.py +class SparkProfiler: + """Encapsulates all PySpark/PyDeequ profiling logic.""" + + def init_spark(self) -> Any: + # Spark session initialization + + def read_file_spark(self, file: str, ext: str): + # File reading with Spark + + def get_table_profile(self, table_data, dataset_urn): + # Table profiling coordination +``` + +For more details, see the [Adding a Metadata Ingestion Source](../metadata-ingestion/adding-source.md#31-using-optional-dependencies-eg-pyspark) guide. + +## Troubleshooting + +### Error: "PySpark is not installed, but is required for profiling" + +**Problem:** You installed a `-slim` variant but have profiling enabled in your config. + +**Solutions:** + +1. **Recommended:** Use standard installation with PySpark: + + ```bash + pip uninstall acryl-datahub + pip install 'acryl-datahub[s3]' # For S3 profiling + ``` + +2. **Alternative:** Disable profiling in your recipe: + ```yaml + profiling: + enabled: false + ``` + +### Verifying Installation + +Check if PySpark is installed: + +```bash +# Check installed packages +pip list | grep pyspark + +# Test import in Python +python -c "import pyspark; print(pyspark.__version__)" +``` + +Expected output: + +- Standard installation (`s3`): Shows `pyspark 3.5.x` +- Slim installation (`s3-slim`): Import fails or package not found + +## Migration Guide + +### Upgrading from Previous Versions + +**No action required!** This change is fully backward compatible: + +```bash +# Existing installations continue to work exactly as before +pip install 'acryl-datahub[s3]' # Still includes PySpark by default (profiling supported) +``` + +**Recommended: Optimize installations** + +- **S3 with profiling:** Keep using `acryl-datahub[s3]` (includes PySpark) +- **S3 without profiling:** Switch to `acryl-datahub[s3-slim]` to save ~500MB + +```bash +# Recommended installations +pip install 'acryl-datahub[s3]' # S3 with profiling support +pip install 'acryl-datahub[s3-slim]' # S3 metadata only (no profiling) +``` + +### No Breaking Changes + +This implementation maintains full backward compatibility: + +- Standard `s3` extra includes PySpark (unchanged behavior) +- All existing recipes and configs continue to work +- New `s3-slim` variant available for users who want smaller installations +- Future DataHub 2.0 may flip defaults, but provides migration path + +## Benefits for DataHub Actions + +[DataHub Actions](https://github.com/datahub-project/datahub/tree/master/datahub-actions) depends on `acryl-datahub` and can benefit from `s3-slim` when profiling is not needed: + +### Reduced Installation Size + +DataHub Actions typically doesn't need data lake profiling capabilities since it focuses on reacting to metadata events, not extracting metadata from data lakes. Use `s3-slim` to reduce footprint: + +```bash +# If Actions needs S3 metadata access but not profiling +pip install acryl-datahub-actions +pip install 'acryl-datahub[s3-slim]' +# Result: ~500MB smaller than standard s3 extra + +# If Actions needs full S3 with profiling +pip install acryl-datahub-actions +pip install 'acryl-datahub[s3]' +# Result: Includes PySpark for profiling capabilities +``` + +### Faster Deployment + +Actions services using `s3-slim` deploy faster in containerized environments: + +- **Faster pip install**: No PySpark compilation required +- **Smaller Docker images**: Reduced base image size +- **Quicker cold starts**: Less code to load and initialize + +### Fewer Dependency Conflicts + +Actions workflows often integrate with other tools (Slack, Teams, email services). Using `s3-slim` reduces: + +- Python version constraint conflicts +- Java/Spark runtime conflicts in restricted environments +- Transitive dependency version mismatches + +### When Actions Needs Profiling + +If your Actions workflow needs to trigger data lake profiling jobs, use the standard extra: + +```bash +# Actions with data lake profiling capability +pip install 'acryl-datahub-actions' +pip install 'acryl-datahub[s3]' # Includes PySpark by default +``` + +**Common Actions use cases that DON'T need PySpark:** + +- Slack notifications on schema changes +- Propagating tags and terms to downstream systems +- Triggering dbt runs on metadata updates +- Sending emails on data quality failures +- Creating Jira tickets for governance issues +- Updating external catalogs (e.g., Alation, Collibra) + +**Rare Actions use cases that MIGHT need PySpark:** + +- Custom actions that programmatically trigger S3 profiling +- Actions that directly process data lake files (not typical) + +## Benefits Summary + +✅ **Backward compatible**: Standard `s3` extra unchanged, existing users unaffected +✅ **Smaller installations**: Save ~500MB with `s3-slim` +✅ **Faster setup**: No PySpark compilation with `s3-slim` +✅ **Flexible deployment**: Choose based on profiling needs +✅ **Type safety maintained**: Refactored with proper code layer separation (mypy passes) +✅ **Clear error messages**: Runtime errors guide users to correct installation +✅ **Actions-friendly**: DataHub Actions benefits from reduced footprint with `s3-slim` + +**Key Takeaways:** + +- Use `s3` if you need S3 profiling, `s3-slim` if you don't +- Pattern can be applied to other sources (ABS, etc.) in future PRs +- Existing installations continue working without changes diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/SECURITY_STANCE.md b/docs-archive/versioned_docs/version-1.5.0/docs/SECURITY_STANCE.md new file mode 100644 index 00000000..0aec2633 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/SECURITY_STANCE.md @@ -0,0 +1,86 @@ +--- +title: DataHub's Commitment to Security +slug: /security_stance +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/SECURITY_STANCE.md' +--- +# DataHub's Commitment to Security + +## Introduction + +The open-source DataHub project takes security seriously. As part of our commitment to maintaining a secure environment +for our users and contributors, we have established a comprehensive security policy. This document outlines the key +aspects of our approach to handling security vulnerabilities and keeping our community informed. + +## Our Track Record + +We have a proactive approach to security. To date we've successfully resolved many security related issues reported by +community members or flagged by automated scanners (which includes upstream dependencies and what known risks the +dependencies contain), demonstrating our commitment to maintaining a secure platform. This is a testament to the +collaborative efforts of our community in identifying and helping us address potential vulnerabilities. It truly takes +a village. + +## Reporting Security Issues + +If you believe you've discovered a security vulnerability in DataHub, we encourage you to report it immediately. We have +a dedicated process for handling security-related issues to ensure they're addressed promptly and discreetly. + +For detailed instructions on how to report a security vulnerability, including our PGP key for encrypted communications, +please visit our official security policy page: + +[DataHub Security Policy](https://github.com/datahub-project/datahub/security/policy) + +We kindly ask that you do not disclose the vulnerability publicly until the committers have had the chance to address it +and make an announcement. + +## Our Response Process + +Once a security issue is reported, the project follows a structured process to ensure that each report is handled with +the attention and urgency it deserves. This includes: + +1. Verifying the reported vulnerability +2. Assessing its potential impact +3. Developing and testing a fix +4. Releasing security patches +5. Coordinating the public disclosure of the vulnerability + +All reported vulnerabilities are carefully assessed and triaged internally to ensure appropriate action is taken. + +## How we prioritize (and the dangers of blindly following automated scanners) + +While we appreciate the value of automated vulnerability detection systems like Dependabot, we want to emphasize the +importance of critical thinking when addressing flagged issues. These systems are excellent at providing signals of +potential vulnerabilities, but they shouldn't be followed blindly. + +Here's why: + +1. Context matters: An issue flagged might only affect a non-serving component of the stack (such as our docs-website + code or our CI smoke tests), which may not pose a significant risk to the overall system. + +2. False positives: Sometimes, these systems may flag vulnerabilities in libraries that are linked but not actively + used. For example, a vulnerability in an email library might be flagged even if the software never sends emails. + +3. Exploit feasibility: Some vulnerabilities may be technically present but extremely difficult or impractical to + exploit in real-world scenarios. Automated scanners often don't consider the actual implementation details or + security controls that might mitigate the risk. For example, a reported SQL injection vulnerability might exist in + theory, but if the application uses parameterized queries or has proper input validation in place, the actual risk + could be significantly lower than the scanner suggests. + +We carefully review all automated alerts in the context of our specific implementation to determine the actual risk and +appropriate action. + +## Keeping the Community Informed + +Transparency is key in maintaining trust within our open-source community. To keep everyone informed about +security-related matters: + +- We maintain Security Advisories on the DataHub project GitHub repository +- These advisories include summaries of security issues, details on the fixes implemented, and any necessary mitigation + steps for users + +## Conclusion + +Security is an ongoing process, and we're committed to continuously improving our practices. By working together with +our community of users and contributors, we aim to maintain DataHub as a secure and reliable metadata platform. + +We encourage all users to stay updated with our security announcements and to promptly apply any security patches +released. Together, we can ensure a safer environment for everyone in the DataHub community. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/_api-guide-template.md b/docs-archive/versioned_docs/version-1.5.0/docs/_api-guide-template.md new file mode 100644 index 00000000..f5bd3d2d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/_api-guide-template.md @@ -0,0 +1,76 @@ +--- +title: '[Feature Name]' +slug: /_api-guide-template +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/_api-guide-template.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# [Feature Name] + + + +### Goal of This Guide + +This guide will show you how to... + + + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. For detailed steps, please refer to [DataHub Quickstart Guide] + + + +## [Action] [Feature Name] + + + + + + + + + + + + + + + + + + + + + + + + + +### Expected Outcome of [Action] [Feature Name] + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/_feature-guide-template.md b/docs-archive/versioned_docs/version-1.5.0/docs/_feature-guide-template.md new file mode 100644 index 00000000..228608dd --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/_feature-guide-template.md @@ -0,0 +1,87 @@ +--- +title: '[Feature Name]' +slug: /_feature-guide-template +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/_feature-guide-template.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# [Feature Name] + + + + + + + + + +## [Feature Name] Setup, Prerequisites, and Permissions + + + +## Using [Feature Name] + + + +## Additional Resources + + + +### Videos + + + + + +### GraphQL + + + +### DataHub Blog + + + +## FAQ and Troubleshooting + + + +### Related Features + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata.md b/docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata.md new file mode 100644 index 00000000..3e5b615e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata.md @@ -0,0 +1,20 @@ +--- +title: Act on Metadata Overview +slug: /act-on-metadata +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/act-on-metadata.md' +--- +# Act on Metadata Overview + +DataHub's metadata infrastructure is stream-oriented, meaning that all changes in metadata are communicated and reflected within the platform within seconds. + +This unlocks endless opportunities to automate data governance and data management workflows, such as: + +- Automatically enrich or annotate existing data entities within DataHub, i.e., apply Tags, Terms, Owners, etc. +- Leverage the [Actions Framework](actions/README.md) to trigger external workflows or send alerts to external systems, i.e., send a message to a team channel when there's a schema change +- Proactively identify what business-critical data resources will be impacted by a breaking schema change + +This section contains resources to help you take real-time action on your rapidly evolving data stack. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata/impact-analysis.md b/docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata/impact-analysis.md new file mode 100644 index 00000000..77bf33af --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/act-on-metadata/impact-analysis.md @@ -0,0 +1,106 @@ +--- +title: Lineage Impact Analysis +slug: /act-on-metadata/impact-analysis +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/act-on-metadata/impact-analysis.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Lineage Impact Analysis + + + +Lineage Impact Analysis is a powerful workflow for understanding the complete set of upstream and downstream dependencies of a Dataset, Dashboard, Chart, and many other DataHub Entities. + +This allows Data Practitioners to proactively identify the impact of breaking schema changes or failed data pipelines on downstream dependencies, rapidly discover which upstream dependencies may have caused unexpected data quality issues, and more. + +Lineage Impact Analysis is available via the DataHub UI and GraphQL endpoints, supporting manual and automated workflows. + +## Lineage Impact Analysis Setup, Prerequisites, and Permissions + +Lineage Impact Analysis is enabled for any Entity that has associated Lineage relationships with other Entities and does not require any additional configuration. + +Any DataHub user with “View Entity Page” permissions is able to view the full set of upstream or downstream Entities and export results to CSV from the DataHub UI. + +## Using Lineage Impact Analysis + +Follow these simple steps to understand the full dependency chain of your data entities. + +1. On a given Entity Page, select the **Lineage** tab + +

+ +

+ +2. Easily toggle between **Upstream** and **Downstream** dependencies + +

+ +

+ +3. Choose the **Degree of Dependencies** you are interested in. The default filter is “1 Degree of Dependency” to minimize processor-intensive queries. + +

+ +

+ +4. Slice and dice the result list by Entity Type, Platform, Owner, and more to isolate the relevant dependencies + +

+ +

+ +5. Export the full list of dependencies to CSV + +

+ +

+ +6. View the filtered set of dependencies via CSV, with details about assigned ownership, domain, tags, terms, and quick links back to those entities within DataHub + +

+ +

+ +### Known Issues + +Impact Analysis is a powerful feature that can place significant demands on the system. To maintain high performance when handling large result sets, we've implemented "Lightning Cache" - an alternate processing path that delivers results more quickly. By default, this cache activates with simple queries when there are more than 300 assets in the result set. You can customize this threshold by setting the environment variable `CACHE_SEARCH_LINEAGE_LIGHTNING_THRESHOLD` in your GMS pod. + +However, the Lightning Cache has a limitation: it may include assets that are soft-deleted or no longer exist in the DataHub database. This occurs because lineage references may contain "ghost entities" (URNs without associated data). + +Note that when you download Impact Analysis results, our system properly filters out these soft-deleted and non-existent assets. As a result, you might notice differences between what appears in the UI and what appears in your downloaded results. + +## Additional Resources + +### Videos + +**DataHub 201: Impact Analysis** + +

+ +

+ +### GraphQL + +- [searchAcrossLineage](../../graphql/queries.md#searchacrosslineage) +- [searchAcrossLineageInput](../../graphql/inputObjects.md#searchacrosslineageinput) + +Looking for an example of how to use `searchAcrossLineage` to read data lineage? Look [here](../api/tutorials/lineage.md#read-lineage) + +### DataHub Blog + +- [Dependency Impact Analysis, Data Validation Outcomes, and MORE! - Highlights from DataHub v0.8.27 & v.0.8.28](https://medium.com/datahub-project/dependency-impact-analysis-data-validation-outcomes-and-more-1302604da233) + +### FAQ and Troubleshooting + +**The Lineage Tab is greyed out - why can’t I click on it?** + +This means you have not yet ingested Lineage metadata for that entity. Please see the Lineage Guide to get started. + +**Why is my list of exported dependencies incomplete?** + +We currently limit the list of dependencies to 10,000 records; we suggest applying filters to narrow the result set if you hit that limit. + +### Related Features + +- [DataHub Lineage](../features/feature-guides/lineage.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/README.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/README.md new file mode 100644 index 00000000..f35f5cd7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/README.md @@ -0,0 +1,249 @@ +--- +title: Introduction +slug: /actions +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/actions/README.md' +--- +# ⚡ DataHub Actions Framework + +Welcome to DataHub Actions! The Actions framework makes responding to realtime changes in your Metadata Graph easy, enabling you to seamlessly integrate [DataHub](https://github.com/datahub-project/datahub) into a broader events-based architecture. + +For a detailed introduction, check out the [original announcement](https://www.youtube.com/watch?v=7iwNxHgqxtg&t=2189s) of the DataHub Actions Framework at the DataHub April 2022 Town Hall. For a more in-depth look at use cases and concepts, check out [DataHub Actions Concepts](concepts.md). + +## Quickstart + +To get started right away, check out the [DataHub Actions Quickstart](quickstart.md) Guide. + +## Prerequisites + +The DataHub Actions CLI commands are an extension of the base `datahub` CLI commands. We recommend +first installing the `datahub` CLI: + +```shell +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub +datahub --version +``` + +> Note that the Actions Framework requires a version of `acryl-datahub` >= v0.8.34 + +## Installation + +Next, simply install the `acryl-datahub-actions` package from PyPi: + +```shell +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub-actions +datahub actions version +``` + +## Configuring an Action + +Actions are configured using a YAML file, much in the same way DataHub ingestion sources are. An action configuration file consists of the following + +1. Action Pipeline Name (Should be unique and static) +2. Source Configurations +3. Transform + Filter Configurations +4. Action Configuration +5. Pipeline Options (Optional) +6. DataHub API configs (Optional - required for select actions) + +With each component being independently pluggable and configurable. + +```yml +# 1. Required: Action Pipeline Name +name: + +# 2. Required: Event Source - Where to source event from. +source: + type: + config: + # Event Source specific configs (map) + +# 3a. Optional: Filter to run on events (map) +filter: + event_type: + event: + # Filter event fields by exact-match + + +# 3b. Optional: Custom Transformers to run on events (array) +transform: + - type: + config: + # Transformer-specific configs (map) + +# 4. Required: Action - What action to take on events. +action: + type: + config: + # Action-specific configs (map) + +# 5. Optional: Additional pipeline options (error handling, etc) +options: + retry_count: 0 # The number of times to retry an Action with the same event. (If an exception is thrown). 0 by default. + failure_mode: "CONTINUE" # What to do when an event fails to be processed. Either 'CONTINUE' to make progress or 'THROW' to stop the pipeline. Either way, the failed event will be logged to a failed_events.log file. + failed_events_dir: "/tmp/datahub/actions" # The directory in which to write a failed_events.log file that tracks events which fail to be processed. Defaults to "/tmp/logs/datahub/actions". + +# 6. Optional: DataHub API configuration +datahub: + server: "http://localhost:8080" # Location of DataHub API + # token: # Required if Metadata Service Auth enabled +``` + +### Example: Hello World + +An simple configuration file for a "Hello World" action, which simply prints all events it receives, is + +```yml +# 1. Action Pipeline Name +name: "hello_world" +# 2. Event Source: Where to source event from. +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +# 3. Action: What action to take on events. +action: + type: "hello_world" +``` + +We can modify this configuration further to filter for specific events, by adding a "filter" block. + +```yml +# 1. Action Pipeline Name +name: "hello_world" + +# 2. Event Source - Where to source event from. +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} + +# 3. Filter - Filter events that reach the Action +filter: + event_type: "EntityChangeEvent_v1" + event: + category: "TAG" + operation: "ADD" + modifier: "urn:li:tag:pii" + +# 4. Action - What action to take on events. +action: + type: "hello_world" +``` + +## Running an Action + +To run a new Action, just use the `actions` CLI command + +``` +datahub actions -c +``` + +Once the Action is running, you will see + +``` +Action Pipeline with name '' is now running. +``` + +### Running multiple Actions + +You can run multiple actions pipeline within the same command. Simply provide multiple +config files by restating the "-c" command line argument. + +For example, + +``` +datahub actions -c -c +``` + +### Running in debug mode + +Simply append the `--debug` flag to the CLI to run your action in debug mode. NOTE: This will reveal sensitive information in the logs, do not share the logs with outside resources and ensure untrusted +users will not have access to logs through UI ingestions before enabling on instances. + +``` +datahub actions -c --debug +``` + +### Stopping an Action + +Just issue a Control-C as usual. You should see the Actions Pipeline shut down gracefully, with a small +summary of processing results. + +``` +Actions Pipeline with name '- + https://github.com/datahub-project/datahub/blob/master/docs/actions/actions/executor.md +--- +# Ingestion Executor + + + +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + +## Overview + +This Action executes ingestion recipes that are configured via the UI. + +### Capabilities + +- Executing `datahub ingest` command in a sub-process when an Execution Request command is received from DataHub. (Scheduled or manual ingestion run) +- Resolving secrets within an ingestion recipe from DataHub +- Reporting ingestion execution status to DataHub + +### Supported Events + +- `MetadataChangeLogEvent_v1` + +Specifically, changes to the `dataHubExecutionRequestInput` and `dataHubExecutionRequestSignal` aspects of the `dataHubExecutionRequest` entity are required. + +## Action Quickstart + +### Prerequisites + +#### DataHub Privileges + +This action must be executed as a privileged DataHub user (e.g. using Personal Access Tokens). Specifically, the user must have the `Manage Secrets` Platform Privilege, which allows for retrieval +of decrypted secrets for injection into an ingestion recipe. + +An access token generated from a privileged account must be configured in the `datahub` configuration +block of the YAML configuration, as shown in the example below. + +#### Connecting to Ingestion Sources + +In order for ingestion to run successfully, the process running the Actions must have +network connectivity to any source systems that are required for ingestion. + +For example, if the ingestion recipe is pulling from an internal DBMS, the actions container +must be able to resolve & connect to that DBMS system for the ingestion command to run successfully. + +### Install the Plugin(s) + +Run the following commands to install the relevant action plugin(s): + +`pip install 'acryl-datahub-actions[executor]'` + +### Configure the Action Config + +Use the following config(s) to get started with this Action. + +```yml +name: "pipeline-name" +source: + # source configs +action: + type: "executor" +# Requires DataHub API configurations to report to DataHub +datahub: + server: "http://${DATAHUB_GMS_HOST:-localhost}:${DATAHUB_GMS_PORT:-8080}" + # token: # Must have "Manage Secrets" privilege +``` + +
+ View All Configuration Options + + | Field | Required | Default | Description | + | --- | :-: | :-: | --- | + | `executor_id` | ❌ | `default` | An executor ID assigned to the executor. This can be used to manage multiple distinct executors. | +
+ +## Troubleshooting + +### Quitting the Actions Framework + +Currently, when you quit the Actions framework, any in-flight ingestion processing will continue to execute as a subprocess on your system. This means that there may be "orphaned" processes which +are never marked as "Succeeded" or "Failed" in the UI, even though they may have completed. + +To address this, simply "Cancel" the ingestion source on the UI once you've restarted the Ingestion Executor action. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/hello_world.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/hello_world.md new file mode 100644 index 00000000..a92e46a6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/hello_world.md @@ -0,0 +1,62 @@ +--- +title: Hello World +slug: /actions/actions/hello_world +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/actions/hello_world.md +--- +# Hello World + + + +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + +## Overview + +This Action is an example action which simply prints all Events it receives as JSON. + +### Capabilities + +- Printing events that are received by the Action to the console. + +### Supported Events + +All event types, including + +- `EntityChangeEvent_v1` +- `MetadataChangeLogEvent_v1` + +## Action Quickstart + +### Prerequisites + +No prerequisites. This action comes pre-loaded with `acryl-datahub-actions`. + +### Install the Plugin(s) + +This action comes with the Actions Framework by default: + +`pip install 'acryl-datahub-actions'` + +### Configure the Action Config + +Use the following config(s) to get started with this Action. + +```yml +name: "pipeline-name" +source: + # source configs +action: + type: "hello_world" +``` + +
+ View All Configuration Options + + | Field | Required | Default | Description | + | --- | :-: | :-: | --- | + | `to_upper` | ❌| `False` | Whether to print events in upper case. | +
+ +## Troubleshooting + +N/A diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/slack.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/slack.md new file mode 100644 index 00000000..8027d71b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/actions/slack.md @@ -0,0 +1,280 @@ +--- +title: Slack +slug: /actions/actions/slack +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/actions/slack.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Slack + + + +# Slack + +| | | +| ------------------------ | ----------------------------------------------------------------------------------------------------- | +| **Status** | ![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) | +| **Version Requirements** | ![Minimum Version Requirements](https://img.shields.io/badge/acryl_datahub_actions-v0.0.9+-green.svg) | + +## Overview + +This Action integrates DataHub with Slack to send notifications to a configured Slack channel in your workspace. + +### Capabilities + +- Sending notifications of important events to a Slack channel + - Adding or Removing a tag from an entity (dataset, dashboard etc.) + - Updating documentation at the entity or field (column) level. + - Adding or Removing ownership from an entity (dataset, dashboard, etc.) + - Creating a Domain + - and many more. + +### User Experience + +On startup, the action will produce a welcome message that looks like the one below. +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_welcome_message.png) + +On each event, the action will produce a notification message that looks like the one below. +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_notification_message.png) + +Watch the townhall demo to see this in action: +[![Slack Action Demo](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_demo_image.png)](https://www.youtube.com/watch?v=BlCLhG8lGoY&t=2998s) + +### Supported Events + +- `EntityChangeEvent_v1` +- Currently, the `MetadataChangeLogEvent_v1` event is **not** processed by the Action. + +## Action Quickstart + +### Prerequisites + +Ensure that you have configured the Slack App in your Slack workspace. + +#### Install the DataHub Slack App into your Slack workspace + +The following steps should be performed by a Slack Workspace Admin. + +- Navigate to https://api.slack.com/apps/ +- Click Create New App +- Use “From an app manifest” option +- Select your workspace +- Paste this Manifest in YAML. We suggest changing the name and `display_name` to be `DataHub App YOUR_TEAM_NAME` but this is not required. This name will show up in your Slack workspace. + +```yml +display_information: + name: DataHub App + description: An app to integrate DataHub with Slack + background_color: "#000000" +features: + bot_user: + display_name: DataHub App + always_online: false +oauth_config: + scopes: + bot: + - channels:history + - channels:read + - chat:write + - commands + - groups:read + - im:read + - mpim:read + - team:read + - users:read + - users:read.email +settings: + org_deploy_enabled: false + socket_mode_enabled: false + token_rotation_enabled: false +``` + +- Confirm you see the Basic Information Tab + +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_basic_info.png) + +- Click **Install to Workspace** +- It will show you permissions the Slack App is asking for, what they mean and a default channel in which you want to add the slack app + - Note that the Slack App will only be able to post in channels that the app has been added to. This is made clear by slack’s Authentication screen also. +- Select the channel you'd like notifications to go to and click **Allow** +- Go to the DataHub App page + - You can find your workspace's list of apps at https://api.slack.com/apps/ + +#### Getting Credentials and Configuration + +Now that you've created your app and installed it in your workspace, you need a few pieces of information before you can activate your Slack action. + +#### 1. The Signing Secret + +On your app's Basic Information page, you will see a App Credentials area. Take note of the Signing Secret information, you will need it later. + +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_app_credentials.png) + +#### 2. The Bot Token + +Navigate to the **OAuth & Permissions** Tab + +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_oauth_and_permissions.png) + +Here you'll find a “Bot User OAuth Token” which DataHub will need to communicate with your Slack workspace through the bot. + +#### 3. The Slack Channel + +Finally, you need to figure out which Slack channel you will send notifications to. Perhaps it should be called #datahub-notifications or maybe, #data-notifications or maybe you already have a channel where important notifications about datasets and pipelines are already being routed to. Once you have decided what channel to send notifications to, make sure to add the app to the channel. + +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_channel_add_app.png) + +Next, figure out the channel id for this Slack channel. You can find it in the About section for the channel if you scroll to the very bottom of the app. +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_channel_id.png) + +Alternately, if you are on the browser, you can figure it out from the URL. e.g. for the troubleshoot channel in OSS DataHub slack + +![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_channel_url.png) + +- Notice `TUMKD5EGJ/C029A3M079U` in the URL + - Channel ID = `C029A3M079U` from above + +In the next steps, we'll show you how to configure the Slack Action based on the credentials and configuration values that you have collected. + +### Installation Instructions (Deployment specific) + +#### DataHub Cloud + +Head over to the [Configuring Notifications](../../managed-datahub/slack/saas-slack-setup.md#configuring-notifications) section in the DataHub Cloud guide to configure Slack notifications for your DataHub Cloud instance. + +#### Quickstart + +If you are running DataHub using the docker quickstart option, there are no additional software installation steps. The `datahub-actions` container comes pre-installed with the Slack action. + +All you need to do is export a few environment variables to activate and configure the integration. See below for the list of environment variables to export. + +| Env Variable | Required for Integration | Purpose | +| -------------------------------------- | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DATAHUB_ACTIONS_SLACK_ENABLED | ✅ | Set to "true" to enable the Slack action | +| DATAHUB_ACTIONS_SLACK_SIGNING_SECRET | ✅ | Set to the [Slack Signing Secret](#1-the-signing-secret) that you configured in the pre-requisites step above | +| DATAHUB_ACTIONS_SLACK_BOT_TOKEN | ✅ | Set to the [Bot User OAuth Token](#2-the-bot-token) that you configured in the pre-requisites step above | +| DATAHUB_ACTIONS_SLACK_CHANNEL | ✅ | Set to the [Slack Channel ID](#3-the-slack-channel) that you want the action to send messages to | +| DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL | ❌ | Defaults to "http://localhost:9002". Set to the location where your DataHub UI is running. On a local quickstart this is usually "http://localhost:9002", so you shouldn't need to modify this | + +:::note + +You will have to restart the `datahub-actions` docker container after you have exported these environment variables if this is the first time. The simplest way to do it is via the Docker Desktop UI, or by just issuing a `datahub docker quickstart --stop && datahub docker quickstart` command to restart the whole instance. + +::: + +For example: + +```shell +export DATAHUB_ACTIONS_SLACK_ENABLED=true +export DATAHUB_ACTIONS_SLACK_SIGNING_SECRET= +.... +export DATAHUB_ACTIONS_SLACK_CHANNEL= + +datahub docker quickstart --stop && datahub docker quickstart +``` + +#### k8s / helm + +Similar to the quickstart scenario, there are no specific software installation steps. The `datahub-actions` container comes pre-installed with the Slack action. You just need to export a few environment variables and make them available to the `datahub-actions` container to activate and configure the integration. See below for the list of environment variables to export. + +| Env Variable | Required for Integration | Purpose | +| -------------------------------------- | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DATAHUB_ACTIONS_SLACK_ENABLED | ✅ | Set to "true" to enable the Slack action | +| DATAHUB_ACTIONS_SLACK_SIGNING_SECRET | ✅ | Set to the [Slack Signing Secret](#1-the-signing-secret) that you configured in the pre-requisites step above | +| DATAHUB_ACTIONS_SLACK_BOT_TOKEN | ✅ | Set to the [Bot User OAuth Token](#2-the-bot-token) that you configured in the pre-requisites step above | +| DATAHUB_ACTIONS_SLACK_CHANNEL | ✅ | Set to the [Slack Channel ID](#3-the-slack-channel) that you want the action to send messages to | +| DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL | ✅ | Set to the location where your DataHub UI is running. For example, if your DataHub UI is hosted at "https://datahub.my-company.biz", set this to "https://datahub.my-company.biz" | + +#### Bare Metal - CLI or Python-based + +If you are using the `datahub-actions` library directly from Python, or the `datahub-actions` cli directly, then you need to first install the `slack` action plugin in your Python virtualenv. + +``` +pip install "acryl-datahub-actions[slack]" +``` + +Then run the action with a configuration file that you have modified to capture your credentials and configuration. + +##### Sample Slack Action Configuration File + +```yml +name: datahub_slack_action +enabled: true +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} + topic_routes: + mcl: ${METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME:-MetadataChangeLog_Versioned_v1} + pe: ${PLATFORM_EVENT_TOPIC_NAME:-PlatformEvent_v1} + +## 3a. Optional: Filter to run on events (map) +# filter: +# event_type: +# event: +# # Filter event fields by exact-match +# + +# 3b. Optional: Custom Transformers to run on events (array) +# transform: +# - type: +# config: +# # Transformer-specific configs (map) + +action: + type: slack + config: + # Action-specific configs (map) + base_url: ${DATAHUB_ACTIONS_SLACK_DATAHUB_BASE_URL:-http://localhost:9002} + bot_token: ${DATAHUB_ACTIONS_SLACK_BOT_TOKEN} + signing_secret: ${DATAHUB_ACTIONS_SLACK_SIGNING_SECRET} + default_channel: ${DATAHUB_ACTIONS_SLACK_CHANNEL} + suppress_system_activity: ${DATAHUB_ACTIONS_SLACK_SUPPRESS_SYSTEM_ACTIVITY:-true} + +datahub: + server: "http://${DATAHUB_GMS_HOST:-localhost}:${DATAHUB_GMS_PORT:-8080}" +``` + +##### Slack Action Configuration Parameters + +| Field | Required | Default | Description | +| -------------------------- | -------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `base_url` | ❌ | `False` | Whether to print events in upper case. | +| `signing_secret` | ✅ | | Set to the [Slack Signing Secret](#1-the-signing-secret) that you configured in the pre-requisites step above | +| `bot_token` | ✅ | | Set to the [Bot User OAuth Token](#2-the-bot-token) that you configured in the pre-requisites step above | +| `default_channel` | ✅ | | Set to the [Slack Channel ID](#3-the-slack-channel) that you want the action to send messages to | +| `suppress_system_activity` | ❌ | `True` | Set to `False` if you want to get low level system activity events, e.g. when datasets are ingested, etc. Note: this will currently result in a very spammy Slack notifications experience, so this is not recommended to be changed. | + +## Troubleshooting + +If things are configured correctly, you should see logs on the `datahub-actions` container that indicate success in enabling and running the Slack action. + +```shell +docker logs datahub-datahub-actions-1 + +... +[2022-12-04 07:07:53,804] INFO {datahub_actions.plugin.action.slack.slack:96} - Slack notification action configured with bot_token=SecretStr('**********') signing_secret=SecretStr('**********') default_channel='C04CZUSSR5X' base_url='http://localhost:9002' suppress_system_activity=True +[2022-12-04 07:07:54,506] WARNING {datahub_actions.cli.actions:103} - Skipping pipeline datahub_teams_action as it is not enabled +[2022-12-04 07:07:54,506] INFO {datahub_actions.cli.actions:119} - Action Pipeline with name 'ingestion_executor' is now running. +[2022-12-04 07:07:54,507] INFO {datahub_actions.cli.actions:119} - Action Pipeline with name 'datahub_slack_action' is now running. +... +``` + +If the Slack action was not enabled, you would see messages indicating that. +e.g. the following logs below show that neither the Slack or Teams action were enabled. + +```shell +docker logs datahub-datahub-actions-1 + +.... +No user action configurations found. Not starting user actions. +[2022-12-04 06:45:27,509] INFO {datahub_actions.cli.actions:76} - DataHub Actions version: unavailable (installed editable via git) +[2022-12-04 06:45:27,647] WARNING {datahub_actions.cli.actions:103} - Skipping pipeline datahub_slack_action as it is not enabled +[2022-12-04 06:45:27,649] WARNING {datahub_actions.cli.actions:103} - Skipping pipeline datahub_teams_action as it is not enabled +[2022-12-04 06:45:27,649] INFO {datahub_actions.cli.actions:119} - Action Pipeline with name 'ingestion_executor' is now running. +... + +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/concepts.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/concepts.md new file mode 100644 index 00000000..cf65f725 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/concepts.md @@ -0,0 +1,105 @@ +--- +title: Concepts +slug: /actions/concepts +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/concepts.md +--- +# DataHub Actions Concepts + +The Actions framework includes pluggable components for filtering, transforming, and reacting to important DataHub, such as + +- Tag Additions / Removals +- Glossary Term Additions / Removals +- Schema Field Additions / Removals +- Owner Additions / Removals + +& more, in real time. + +DataHub Actions comes with open library of freely available Transformers, Actions, Events, and more. + +Finally, the framework is highly configurable & scalable. Notable highlights include: + +- **Distributed Actions**: Ability to scale-out processing for a single action. Support for running the same Action configuration across multiple nodes to load balance the traffic from the event stream. +- **At-least Once Delivery**: Native support for independent processing state for each Action via post-processing acking to achieve at-least once semantics. +- **Robust Error Handling**: Configurable failure policies featuring event-retry, dead letter queue, and failed-event continuation policy to achieve the guarantees required by your organization. + +### Use Cases + +Real-time use cases broadly fall into the following categories: + +- **Notifications**: Generate organization-specific notifications when a change is made on DataHub. For example, send an email to the governance team when a "PII" tag is added to any data asset. +- **Workflow Integration**: Integrate DataHub into your organization's internal workflows. For example, create a Jira ticket when specific Tags or Terms are proposed on a Dataset. +- **Synchronization**: Syncing changes made in DataHub into a 3rd party system. For example, reflecting Tag additions in DataHub into Snowflake. +- **Auditing**: Audit who is making what changes on DataHub through time. + +and more! + +## Concepts + +The Actions Framework consists of a few core concepts-- + +- **Pipelines** +- **Events** and **Event Sources** +- **Transformers** +- **Actions** + +Each of these will be described in detail below. + +

+ +

+ +**In the Actions Framework, Events flow continuously from left-to-right.** + +### Pipelines + +A **Pipeline** is a continuously running process which performs the following functions: + +1. Polls events from a configured Event Source (described below) +2. Applies configured Transformation + Filtering to the Event +3. Executes the configured Action on the resulting Event + +in addition to handling initialization, errors, retries, logging, and more. + +Each Action Configuration file corresponds to a unique Pipeline. In practice, +each Pipeline has its very own Event Source, Transforms, and Actions. This makes it easy to maintain state for mission-critical Actions independently. + +Importantly, each Action must have a unique name. This serves as a stable identifier across Pipeline run which can be useful in saving the Pipeline's consumer state (ie. resiliency + reliability). For example, the Kafka Event Source (default) uses the pipeline name as the Kafka Consumer Group id. This enables you to easily scale-out your Actions by running multiple processes with the same exact configuration file. Each will simply become different consumers in the same consumer group, sharing traffic of the DataHub Events stream. + +### Events + +**Events** are data objects representing changes that have occurred on DataHub. Strictly speaking, the only requirement that the Actions framework imposes is that these objects must be + +a. Convertible to JSON +b. Convertible from JSON + +So that in the event of processing failures, events can be written and read from a failed events file. + +#### Event Types + +Each Event instance inside the framework corresponds to a single **Event Type**, which is common name (e.g. "EntityChangeEvent_v1") which can be used to understand the shape of the Event. This can be thought of as a "topic" or "stream" name. That being said, Events associated with a single type are not expected to change in backwards-breaking ways across versons. + +### Event Sources + +Events are produced to the framework by **Event Sources**. Event Sources may include their own guarantees, configurations, behaviors, and semantics. They usually produce a fixed set of Event Types. + +In addition to sourcing events, Event Sources are also responsible for acking the successful processing of an event by implementing the `ack` method. This is invoked by the framework once the Event is guaranteed to have reached the configured Action successfully. + +### Transformers + +**Transformers** are pluggable components which take an Event as input, and produce an Event (or nothing) as output. This can be used to enrich the information of an Event prior to sending it to an Action. + +Multiple Transformers can be configured to run in sequence, filtering and transforming an event in multiple steps. + +Transformers can also be used to generate a completely new type of Event (i.e. registered at runtime via the Event Registry) which can subsequently serve as input to an Action. + +Transformers can be easily customized and plugged in to meet an organization's unique requirements. For more information on developing a Transformer, check out [Developing a Transformer](guides/developing-a-transformer.md) + +### Action + +**Actions** are pluggable components which take an Event as input and perform some business logic. Examples may be sending a Slack notification, logging to a file, +or creating a Jira ticket, etc. + +Each Pipeline can be configured to have a single Action which runs after the filtering and transformations have occurred. + +Actions can be easily customized and plugged in to meet an organization's unique requirements. For more information on developing a Action, check out [Developing a Action](guides/developing-an-action.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/audit-events-search-guide.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/audit-events-search-guide.md new file mode 100644 index 00000000..f4cb9998 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/audit-events-search-guide.md @@ -0,0 +1,284 @@ +--- +title: Audit Events Search API V1 +slug: /actions/events/audit-events-search-guide +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/events/audit-events-search-guide.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Audit Events Search API V1 + + + +## Endpoint + +`/openapi/v1/events/audit/search` + +## Overview + +This API allows you to search for audit events that occur within DataHub. Audit events track various actions performed by users and systems, providing a comprehensive history of activities and changes within your DataHub instance. + +## Request Structure + +The Audit Events Search API accepts POST requests with optional query parameters and a required JSON body. + +### Query Parameters + +| Name | Type | Description | Required | Default | +| ---------- | ------- | ----------------------------------------------------------------------- | -------- | ------- | +| startTime | int64 | The timestamp (in ms) to start the search from, defaults to one day ago | No | -1 | +| endTime | int64 | The timestamp (in ms) to end the search at, defaults to current time | No | -1 | +| size | int32 | The maximum number of events to return in one response | No | 10 | +| scrollId | string | The scroll ID used for pagination when fetching subsequent results | No | null | +| includeRaw | boolean | Whether to include the raw event data in the response | No | true | + +### Request Body + +The request body must be a JSON object with the following structure: + +```json +{ + "eventTypes": ["string"], + "entityTypes": ["string"], + "aspectTypes": ["string"], + "actorUrns": ["string"] +} +``` + +| Field | Type | Description | Required | +| ----------- | -------- | -------------------------------------------------------------- | -------- | +| eventTypes | string[] | List of event types to filter by (empty means all event types) | No | +| entityTypes | string[] | List of entity types to filter by (empty means all entities) | No | +| aspectTypes | string[] | List of aspect types to filter by (empty means all aspects) | No | +| actorUrns | string[] | List of actor URNs to filter by (empty means all actors) | No | + +These filters work as `and` filters between each other and `or` filters for elements in the list so: + +```json +{ + "eventTypes": ["CreateAccessTokenEvent", "RevokeAccessTokenEvent"], + "actorUrns": ["urn:li:corpuser:datahub"] +} +``` + +Filters for events that are either `CreateAccessTokenEvent` or `RevokeAccessTokenEvent` AND had `urn:li:corpuser:datahub` as the actor. + +## Response Structure + +The API returns a JSON object with the following structure: + +```json +{ + "nextScrollId": "string", + "count": 0, + "total": 0, + "usageEvents": [ + { + // Event data varies based on event type + } + ] +} +``` + +| Field | Type | Description | +| ------------ | ------ | ------------------------------------------------------------------ | +| nextScrollId | string | ID for retrieving the next page of results (if more are available) | +| count | int32 | Number of events returned in this response | +| total | int32 | Total count of matching events (calculated up to 10,000) | +| usageEvents | array | Array of usage events matching the search criteria | + +## Event Types + +The API supports various event types that track different actions within DataHub. Each event type has its own specific structure, but all share common properties defined in the `UsageEventResult` base type. + +### Common Fields (UsageEventResult) + +All event types include these base fields: + +| Field | Type | Description | +| ---------------- | ------ | ------------------------------------------------------------ | +| eventType | string | Type of the event | +| timestamp | int64 | Timestamp when the event occurred (in milliseconds) | +| actorUrn | string | URN of the actor who performed the action | +| sourceIP | string | IP address from which the action was performed | +| eventSource | enum | Source API of the event (RESTLI, OPENAPI, GRAPHQL, SSO_SCIM) | +| userAgent | string | User agent string from the HTTP request (if applicable) | +| telemetryTraceId | string | Trace ID from system telemetry | +| rawUsageEvent | object | Full raw event contents if includeRaw=true | + +### Specific Event Types + +The API returns different event types, each with its own specific structure in addition to the common fields: + +#### EntityEvent + +Tracks general entity operations. + +```json +{ + "eventType": "EntityEvent", + "timestamp": 1649953100653, + "actorUrn": "urn:li:corpuser:jdoe", + "sourceIP": "192.168.1.1", + "eventSource": "GRAPHQL", + "userAgent": "Mozilla/5.0...", + "telemetryTraceId": "abc123", + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "aspectName": "ownership" +} +``` + +#### Create/Update/Delete Event Types + +Several event types track specific creation, update, and deletion actions: + +- **CreateUserEvent**: Tracks user creation +- **UpdateUserEvent**: Tracks user updates +- **CreateAccessTokenEvent**: Tracks access token creation +- **RevokeAccessTokenEvent**: Tracks access token revocation +- **CreatePolicyEvent**: Tracks policy creation +- **UpdatePolicyEvent**: Tracks policy updates +- **DeletePolicyEvent**: Tracks policy deletes +- **CreateIngestionSourceEvent**: Tracks ingestion source creation +- **UpdateIngestionSourceEvent**: Tracks ingestion source updates +- **DeleteEntityEvent**: Tracks entity deletion +- **UpdateAspectEvent**: Tracks aspect updates + +All these event types share the same structure: + +```json +{ + "eventType": "[Event Type Name]", + "timestamp": 1649953100653, + "actorUrn": "urn:li:corpuser:jdoe", + "sourceIP": "192.168.1.1", + "eventSource": "GRAPHQL", + "userAgent": "Mozilla/5.0...", + "telemetryTraceId": "abc123", + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "aspectName": "ownership" +} +``` + +#### LogInEvent & FailedLogInEvent + +Tracks user login events with a specific login source. + +```json +{ + "eventType": "LogInEvent", + "timestamp": 1649953100653, + "actorUrn": "urn:li:corpuser:jdoe", + "sourceIP": "192.168.1.1", + "eventSource": "GRAPHQL", + "userAgent": "Mozilla/5.0...", + "telemetryTraceId": "abc123", + "loginSource": "PASSWORD_LOGIN" +} +``` + +`loginSource` can be one of: + +- PASSWORD_RESET +- PASSWORD_LOGIN +- FALLBACK_LOGIN +- SIGN_UP_LINK_LOGIN +- GUEST_LOGIN +- SSO_LOGIN +- OIDC_IMPLICIT_LOGIN + +## Usage Examples + +### Basic Search for All Events + +To search for all audit events with default parameters: + +```json +// POST /openapi/v1/events/audit/search +{ + "eventTypes": [], + "entityTypes": [], + "aspectTypes": [], + "actorUrns": [] +} +``` + +### Search for Events by a Specific User + +To search for all events performed by a specific user: + +```json +// POST /openapi/v1/events/audit/search +{ + "eventTypes": [], + "entityTypes": [], + "aspectTypes": [], + "actorUrns": ["urn:li:corpuser:jdoe"] +} +``` + +### Search for Specific Event Types with Time Range + +To search for specific event types within a time range: + +```json +// POST /openapi/v1/events/audit/search?startTime=1649953000000&endTime=1649954000000 +{ + "eventTypes": ["LogInEvent", "CreateUserEvent"], + "entityTypes": [], + "aspectTypes": [], + "actorUrns": [] +} +``` + +### Search for Events on Specific Entity Types + +To search for events related to specific entity types: + +```json +// POST /openapi/v1/events/audit/search +{ + "eventTypes": [], + "entityTypes": ["dataset", "dashboard"], + "aspectTypes": [], + "actorUrns": [] +} +``` + +### Paginating through Results + +To retrieve the first page of results: + +```json +// POST /openapi/v1/events/audit/search?size=25 +{ + "eventTypes": [], + "entityTypes": [], + "aspectTypes": [], + "actorUrns": [] +} +``` + +To retrieve subsequent pages, use the `nextScrollId` from the previous response: + +```json +// POST /openapi/v1/events/audit/search?scrollId=abcdef123456&size=25 +{ + "eventTypes": [], + "entityTypes": [], + "aspectTypes": [], + "actorUrns": [] +} +``` + +## Best Practices + +1. **Use Time Ranges**: Always specify start and end times when searching for events to limit the result set and improve performance. + +2. **Filter Appropriately**: Use the filtering options (eventTypes, entityTypes, etc.) to narrow down your search to only the events you're interested in. + +3. **Paginate Results**: Use the size parameter and scrollId to paginate through large result sets rather than trying to retrieve all events at once. + +4. **Monitor User Activity**: Use the actorUrns filter to track actions by specific users, which is useful for security auditing. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/entity-change-event.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/entity-change-event.md new file mode 100644 index 00000000..34c8f129 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/entity-change-event.md @@ -0,0 +1,732 @@ +--- +title: Entity Change Event V1 +slug: /actions/events/entity-change-event +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/events/entity-change-event.md +--- +# Entity Change Event V1 + +## Event Type + +`EntityChangeEvent_v1` + +## Overview + +This Event is emitted when certain changes are made to an entity (dataset, dashboard, chart, etc) on DataHub. + +## Event Structure + +Entity Change Events are generated in a variety of circumstances, but share a common set of fields. + +### Common Fields + +| Name | Type | Description | Optional | +| ---------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | +| entityUrn | String | The unique identifier for the Entity being changed. For example, a Dataset's urn. | False | +| entityType | String | The type of the entity being changed. Supported values include dataset, chart, dashboard, dataFlow (Pipeline), dataJob (Task), domain, tag, glossaryTerm, corpGroup, & corpUser. | False | +| category | String | The category of the change, related to the kind of operation that was performed. Examples include TAG, GLOSSARY_TERM, DOMAIN, LIFECYCLE, and more. | False | +| operation | String | The operation being performed on the entity given the category. For example, ADD ,REMOVE, MODIFY. For the set of valid operations, see the full catalog below. | False | +| modifier | String | The modifier that has been applied to the entity. The value depends on the category. An example includes the URN of a tag being applied to a Dataset or Schema Field. | True | +| parameters | Dict | Additional key-value parameters used to provide specific context. The precise contents depends on the category + operation of the event. See the catalog below for a full summary of the combinations. | True | +| auditStamp.actor | String | The urn of the actor who triggered the change. | False | +| auditStamp.time | Number | The timestamp in milliseconds corresponding to the event. | False | + +In following sections, we will provide sample events for each scenario in which Entity Change Events are fired. + +### Add Tag Event + +This event is emitted when a Tag has been added to an entity on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TAG", + "operation": "ADD", + "modifier": "urn:li:tag:PII", + "parameters": { + "tagUrn": "urn:li:tag:PII" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Tag Event + +This event is emitted when a Tag has been removed from an entity on DataHub. +Header + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TAG", + "operation": "REMOVE", + "modifier": "urn:li:tag:PII", + "parameters": { + "tagUrn": "urn:li:tag:PII" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Glossary Term Event + +This event is emitted when a Glossary Term has been added to an entity on DataHub. +Header + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "GLOSSARY_TERM", + "operation": "ADD", + "modifier": "urn:li:glossaryTerm:ExampleNode.ExampleTerm", + "parameters": { + "termUrn": "urn:li:glossaryTerm:ExampleNode.ExampleTerm" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Glossary Term Event + +This event is emitted when a Glossary Term has been removed from an entity on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "GLOSSARY_TERM", + "operation": "REMOVE", + "modifier": "urn:li:glossaryTerm:ExampleNode.ExampleTerm", + "parameters": { + "termUrn": "urn:li:glossaryTerm:ExampleNode.ExampleTerm" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Domain Event + +This event is emitted when Domain has been added to an entity on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DOMAIN", + "operation": "ADD", + "modifier": "urn:li:domain:ExampleDomain", + "parameters": { + "domainUrn": "urn:li:domain:ExampleDomain" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Domain Event + +This event is emitted when Domain has been removed from an entity on DataHub. +Header + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DOMAIN", + "operation": "REMOVE", + "modifier": "urn:li:domain:ExampleDomain", + "parameters": { + "domainUrn": "urn:li:domain:ExampleDomain" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Owner Event + +This event is emitted when a new owner has been assigned to an entity on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "OWNER", + "operation": "ADD", + "modifier": "urn:li:corpuser:jdoe", + "parameters": { + "ownerUrn": "urn:li:corpuser:jdoe", + "ownerType": "BUSINESS_OWNER" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Owner Event + +This event is emitted when an existing owner has been removed from an entity on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "OWNER", + "operation": "REMOVE", + "modifier": "urn:li:corpuser:jdoe", + "parameters": { + "ownerUrn": "urn:li:corpuser:jdoe", + "ownerType": "BUSINESS_OWNER" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Structured Property Event + +This event is emitted when a Structured Property has been added to an entity on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "STRUCTURED_PROPERTY", + "operation": "ADD", + "modifier": "urn:li:structuredProperty:prop1", + "parameters": { + "propertyUrn": "urn:li:structuredProperty:prop1", + "propertyValues": "[\"value1\"]" + }, + "version": 0, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Structured Property Event + +This event is emitted when a Structured Property has been removed from an entity on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "STRUCTURED_PROPERTY", + "operation": "REMOVE", + "modifier": "urn:li:structuredProperty:prop1", + "version": 0, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Modify Structured Property Event + +This event is emitted when a Structured Property's values have been modified on an entity in DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "STRUCTURED_PROPERTY", + "operation": "MODIFY", + "modifier": "urn:li:structuredProperty:prop1", + "parameters": { + "propertyUrn": "urn:li:structuredProperty:prop1", + "propertyValues": "[\"value1\",\"value2\"]" + }, + "version": 0, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Modify Deprecation Event + +This event is emitted when the deprecation status of an entity has been modified on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DEPRECATION", + "operation": "MODIFY", + "modifier": "DEPRECATED", + "parameters": { + "status": "DEPRECATED" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Dataset Schema Field Event + +This event is emitted when a new field has been added to a Dataset Schema. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TECHNICAL_SCHEMA", + "operation": "ADD", + "modifier": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "parameters": { + "fieldUrn": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "fieldPath": "newFieldName", + "nullable": false + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Dataset Schema Field Event + +This event is emitted when a new field has been remove from a Dataset Schema. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TECHNICAL_SCHEMA", + "operation": "REMOVE", + "modifier": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "parameters": { + "fieldUrn": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "fieldPath": "newFieldName", + "nullable": false + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Entity Create Event + +This event is emitted when a new entity has been created on DataHub. +Header + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Entity Soft-Delete Event + +This event is emitted when a new entity has been soft-deleted on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "LIFECYCLE", + "operation": "SOFT_DELETE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Entity Hard-Delete Event + +This event is emitted when a new entity has been hard-deleted on DataHub. + +#### Sample Event + +```json +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "LIFECYCLE", + "operation": "HARD_DELETE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +## Action Request Events (Proposals) + +Action Request events represent proposals for changes to entities that may require approval before being applied. These events have entityType "actionRequest" and use the `LIFECYCLE` category with `CREATE` operation. + +### Domain Association Request Event + +This event is emitted when a domain association is proposed for an entity on DataHub. + +#### Sample Event + +```json +{ + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:abc-123", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1234567890 + }, + "version": 0, + "parameters": { + "domains": "[\"urn:li:domain:marketing\"]", + "actionRequestType": "DOMAIN_ASSOCIATION", + "resourceUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "resourceType": "dataset" + } +} +``` + +### Owner Association Request Event + +This event is emitted when an owner association is proposed for an entity on DataHub. + +#### Sample Event + +```json +{ + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:def-456", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1234567890 + }, + "version": 0, + "parameters": { + "owners": "[{\"type\":\"TECHNICAL_OWNER\",\"typeUrn\":\"urn:li:ownershipType:technical_owner\",\"ownerUrn\":\"urn:li:corpuser:jdoe\"}]", + "actionRequestType": "OWNER_ASSOCIATION", + "resourceUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "resourceType": "dataset" + } +} +``` + +### Tag Association Request Event + +This event is emitted when a tag association is proposed for an entity on DataHub. + +#### Sample Event + +```json +{ + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:ghi-789", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1234567890 + }, + "version": 0, + "parameters": { + "actionRequestType": "TAG_ASSOCIATION", + "resourceUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "tagUrn": "urn:li:tag:pii", + "resourceType": "dataset" + } +} +``` + +### Create Glossary Term Request Event + +This event is emitted when a new glossary term creation is proposed on DataHub. + +#### Sample Event + +```json +{ + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:jkl-101", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1234567890 + }, + "version": 0, + "parameters": { + "parentNodeUrn": "urn:li:glossaryNode:123", + "glossaryEntityName": "ExampleTerm", + "actionRequestType": "CREATE_GLOSSARY_TERM", + "resourceType": "glossaryTerm" + } +} +``` + +### Term Association Request Event + +This event is emitted when a glossary term association is proposed for an entity on DataHub. + +#### Sample Event + +```json +{ + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:mno-102", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1234567890 + }, + "version": 0, + "parameters": { + "glossaryTermUrn": "urn:li:glossaryTerm:123", + "actionRequestType": "TERM_ASSOCIATION", + "resourceUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "resourceType": "dataset" + } +} +``` + +### Update Description Request Event + +This event is emitted when an update to an entity's description is proposed on DataHub. + +#### Sample Event + +```json +{ + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:pqr-103", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1234567890 + }, + "version": 0, + "parameters": { + "description": "Example description for a dataset.", + "actionRequestType": "UPDATE_DESCRIPTION", + "resourceUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "resourceType": "dataset" + } +} +``` + +### Structured Property Association Request Event + +This event is emitted when a structured property association is proposed for an entity on DataHub. + +#### Sample Event + +```json +{ + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:stu-104", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1234567890 + }, + "version": 0, + "parameters": { + "structuredProperties": "[{\"propertyUrn\":\"urn:li:structuredProperty:123\",\"values\":[\"value1\",\"value2\"]}]", + "actionRequestType": "STRUCTURED_PROPERTY_ASSOCIATION", + "resourceUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "resourceType": "dataset" + } +} +``` + +### Approval Workflow Request Create Event (Data Access Workflows) + +This event is emitted when a new approval workflow form request is submitted on DataHub. + +Note that `parameters.fields` contains all of the form fields provided when the user originally +submitted the approval request form. It contains a serialized JSON object that must be deserialized from string before accessing! + +#### Sample Event + +```json +{ + "event_type": "EntityChangeEvent_v1", + "event": { + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:88170f40-b4f5-4b05-a997-f5f680248b59", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "time": 1753829419490, + "actor": "urn:li:corpuser:admin" + }, + "version": 0, + "parameters": { + "actorUrn": "urn:li:corpuser:admin", + "actionRequestType": "WORKFLOW_FORM_REQUEST", + "workflowUrn": "urn:li:actionWorkflow:7a6e7a24-7525-4f1e-8d9b-08ba562399d0", + "expiresAtMs": 1756680611743, + "fields": "{\"reason\":[\"Building Tableau Dashboard Q2\"],\"privileges\":[\"SELECT\"],\"projects\":[\"Orders Dashboard Q2\"],\"tags\":[\"urn:li:tag:__default_gold\"]}", + "workflowId": "7a6e7a24-7525-4f1e-8d9b-08ba562399d0", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "entityName": "table", + "qualifiedName": "example.table", + "entityPlatformName": "Snowflake" + } + } +} +``` + +### Approval Workflow Request Step Complete Event (Data Access Workflows) + +This event is emitted when an approval step within a workflow request has been completed (approved, rejected, or requires more information). + +Note that `parameters.fields` contains all of the form fields provided when the user originally +submitted the approval request form. It contains a serialized JSON object that must be deserialized from string before accessing! + +#### Sample Event + +```json +{ + "event_type": "EntityChangeEvent_v1", + "event": { + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:ce424fff-7fab-4731-b106-fab9d69238da", + "category": "LIFECYCLE", + "operation": "MODIFY", + "auditStamp": { + "time": 1753829625133, + "actor": "urn:li:corpuser:admin" + }, + "version": 0, + "parameters": { + "actorUrn": "urn:li:corpuser:admin", + "actionRequestType": "WORKFLOW_FORM_REQUEST", + "stepId": "approval-step-2", + "workflowUrn": "urn:li:actionWorkflow:7a6e7a24-7525-4f1e-8d9b-08ba562399d0", + "expiresAtMs": 1754002407730, + "stepResult": "ACCEPTED", + "fields": "{\"reason\":[\"Test\"],\"privileges\":[\"SELECT\"],\"tags\":[\"urn:li:tag:NeedsDocumentation\"]}", + "workflowId": "7a6e7a24-7525-4f1e-8d9b-08ba562399d0", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "entityName": "table", + "qualifiedName": "example.table", + "entityPlatformName": "Snowflake" + } + } +} +``` + +### Approval Workflow Request Complete Event (Data Access Workflows) + +This event is emitted when an approval workflow request has been fully completed - either approved through all steps or rejected at any step. + +Note that `parameters.fields` contains all of the form fields provided when the user originally +submitted the approval request form. It contains a serialized JSON object that must be deserialized from string before accessing! + +#### Sample Event + +```json +{ + "event_type": "EntityChangeEvent_v1", + "event": { + "entityType": "actionRequest", + "entityUrn": "urn:li:actionRequest:553595ec-2295-4970-89a4-bb7d02e691c0", + "category": "LIFECYCLE", + "operation": "COMPLETED", + "auditStamp": { + "time": 1753829954539, + "actor": "urn:li:corpuser:admin" + }, + "version": 0, + "parameters": { + "result": "ACCEPTED", + "actorUrn": "urn:li:corpuser:admin", + "actionRequestType": "WORKFLOW_FORM_REQUEST", + "workflowUrn": "urn:li:actionWorkflow:7a6e7a24-7525-4f1e-8d9b-08ba562399d0", + "expiresAtMs": 1756335474418, + "fields": "{\"reason\":[\"test\"],\"privileges\":[\"SELECT\"],\"projects\":[\"test\"],\"tags\":[\"urn:li:tag:NeedsDocumentation\"]}", + "operation": "COMPLETE", + "workflowId": "7a6e7a24-7525-4f1e-8d9b-08ba562399d0", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,example.table,PROD)", + "entityName": "table", + "qualifiedName": "example.table", + "entityPlatformName": "Snowflake" + } + } +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/metadata-change-log-event.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/metadata-change-log-event.md new file mode 100644 index 00000000..5d40e6f4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/events/metadata-change-log-event.md @@ -0,0 +1,154 @@ +--- +title: Metadata Change Log Event V1 +slug: /actions/events/metadata-change-log-event +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/events/metadata-change-log-event.md +--- +# Metadata Change Log Event V1 + +## Event Type + +`MetadataChangeLogEvent_v1` + +## Overview + +This event is emitted when any aspect on DataHub Metadata Graph is changed. This includes creates, updates, and removals of both "versioned" aspects and "time-series" aspects. + +> Disclaimer: This event is quite powerful, but also quite low-level. Because it exposes the underlying metadata model directly, it is subject to more frequent structural and semantic changes than the higher level [Entity Change Event](entity-change-event.md). We recommend using that event instead to achieve your use case when possible. + +## Event Structure + +The fields include + +| Name | Type | Description | Optional | +| ------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | +| entityUrn | String | The unique identifier for the Entity being changed. For example, a Dataset's urn. | False | +| entityType | String | The type of the entity being changed. Supported values include dataset, chart, dashboard, dataFlow (Pipeline), dataJob (Task), domain, tag, glossaryTerm, corpGroup, & corpUser. | False | +| entityKeyAspect | Object | The key struct of the entity that was changed. Only present if the Metadata Change Proposal contained the raw key struct. | True | +| changeType | String | The change type. UPSERT, DELETE, CREATE, RESTATE are currently supported. | False | +| aspectName | String | The entity aspect which was changed. | False | +| aspect | Object | The new aspect value. Null if the aspect was deleted. | True | +| aspect.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json`. | False | +| aspect.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| previousAspectValue | Object | The previous aspect value. Null if the aspect did not exist previously. | True | +| previousAspectValue.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json` | False | +| previousAspectValue.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| systemMetadata | Object | The new system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | +| previousSystemMetadata | Object | The previous system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | +| created | Object | Audit stamp about who triggered the Metadata Change and when. | False | +| created.time | Number | The timestamp in milliseconds when the aspect change occurred. | False | +| created.actor | String | The URN of the actor (e.g. corpuser) that triggered the change. | + +### Sample Events + +#### Tag Change Event + +```json +{ + "entityType": "container", + "entityUrn": "urn:li:container:DATABASE", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "value": "{\"tags\":[{\"tag\":\"urn:li:tag:pii\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1651516475595, + "runId": "no-run-id-provided", + "registryName": "unknownRegistry", + "registryVersion": "0.0.0.0-dev", + "properties": null + }, + "previousAspectValue": null, + "previousSystemMetadata": null, + "created": { + "time": 1651516475594, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } +} +``` + +#### Glossary Term Change Event + +```json +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "glossaryTerms", + "aspect": { + "value": "{\"auditStamp\":{\"actor\":\"urn:li:corpuser:datahub\",\"time\":1651516599479},\"terms\":[{\"urn\":\"urn:li:glossaryTerm:CustomerAccount\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1651516599486, + "runId": "no-run-id-provided", + "registryName": "unknownRegistry", + "registryVersion": "0.0.0.0-dev", + "properties": null + }, + "previousAspectValue": null, + "previousSystemMetadata": null, + "created": { + "time": 1651516599480, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } +} +``` + +#### Owner Change Event + +```json +{ + "auditHeader": null, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\":[{\"type\":\"DATAOWNER\",\"owner\":\"urn:li:corpuser:datahub\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:datahub\",\"time\":1651516640488}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1651516640493, + "runId": "no-run-id-provided", + "registryName": "unknownRegistry", + "registryVersion": "0.0.0.0-dev", + "properties": null + }, + "previousAspectValue": { + "value": "{\"owners\":[{\"owner\":\"urn:li:corpuser:jdoe\",\"type\":\"DATAOWNER\"},{\"owner\":\"urn:li:corpuser:datahub\",\"type\":\"DATAOWNER\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:jdoe\",\"time\":1581407189000}}", + "contentType": "application/json" + }, + "previousSystemMetadata": { + "lastObserved": 1651516415088, + "runId": "file-2022_05_02-11_33_35", + "registryName": null, + "registryVersion": null, + "properties": null + }, + "created": { + "time": 1651516640490, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } +} +``` + +## FAQ + +### Where can I find all the aspects and their schemas? + +Great Question! All MetadataChangeLog events are based on the Metadata Model which is comprised of Entities, +Aspects, and Relationships which make up an enterprise Metadata Graph. We recommend checking out the following +resources to learn more about this: + +- [Intro to Metadata Model](/docs/metadata-modeling/metadata-model) + +You can also find a comprehensive list of Entities + Aspects of the Metadata Model under the **Metadata Modeling > Entities** section of the [official DataHub docs](/docs/). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-a-transformer.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-a-transformer.md new file mode 100644 index 00000000..90dfe99f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-a-transformer.md @@ -0,0 +1,135 @@ +--- +title: Developing a Transformer +slug: /actions/guides/developing-a-transformer +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/guides/developing-a-transformer.md +--- +# Developing a Transformer + +In this guide, we will outline each step to developing a custom Transformer for the DataHub Actions Framework. + +## Overview + +Developing a DataHub Actions Transformer is a matter of extending the `Transformer` base class in Python, installing your +Transformer to make it visible to the framework, and then configuring the framework to use the new Transformer. + +## Step 1: Defining a Transformer + +To implement an Transformer, we'll need to extend the `Transformer` base class and override the following functions: + +- `create()` - This function is invoked to instantiate the action, with a free-form configuration dictionary + extracted from the Actions configuration file as input. +- `transform()` - This function is invoked when an Event is received. It should contain the core logic of the Transformer. + and will return the transformed Event, or `None` if the Event should be filtered. + +Let's start by defining a new implementation of Transformer called `CustomTransformer`. We'll keep it simple-- this Transformer will +print the configuration that is provided when it is created, and print any Events that it receives. + +```python +# custom_transformer.py +from datahub_actions.transform.transformer import Transformer +from datahub_actions.event.event_envelope import EventEnvelope +from datahub_actions.pipeline.pipeline_context import PipelineContext +from typing import Optional + +class CustomTransformer(Transformer): + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> "Transformer": + # Simply print the config_dict. + print(config_dict) + return cls(config_dict, ctx) + + def __init__(self, ctx: PipelineContext): + self.ctx = ctx + + def transform(self, event: EventEnvelope) -> Optional[EventEnvelope]: + # Simply print the received event. + print(event) + # And return the original event (no-op) + return event +``` + +## Step 2: Installing the Transformer + +Now that we've defined the Transformer, we need to make it visible to the framework by making +it available in the Python runtime environment. + +The easiest way to do this is to just place it in the same directory as your configuration file, in which case the module name is the same as the file +name - in this case it will be `custom_transformer`. + +### Advanced: Installing as a Package + +Alternatively, create a `setup.py` file in the same directory as the new Transformer to convert it into a package that pip can understand. + +``` +from setuptools import find_packages, setup + +setup( + name="custom_transformer_example", + version="1.0", + packages=find_packages(), + # if you don't already have DataHub Actions installed, add it under install_requires + # install_requires=["acryl-datahub-actions"] +) +``` + +Next, install the package + +```shell +pip install -e . +``` + +inside the module. (alt.`python setup.py`). + +Once we have done this, our class will be referencable via `custom_transformer_example.custom_transformer:CustomTransformer`. + +## Step 3: Running the Action + +Now that we've defined our Transformer, we can create an Action configuration file that refers to the new Transformer. +We will need to provide the fully-qualified Python module & class name when doing so. + +_Example Configuration_ + +```yaml +# custom_transformer_action.yaml +name: "custom_transformer_test" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +transform: + - type: "custom_transformer_example.custom_transformer:CustomTransformer" + config: + # Some sample configuration which should be printed on create. + config1: value1 +action: + # Simply reuse the default hello_world action + type: "hello_world" +``` + +Next, run the `datahub actions` command as usual: + +```shell +datahub actions -c custom_transformer_action.yaml +``` + +If all is well, your Transformer should now be receiving & printing Events. + +### (Optional) Step 4: Contributing the Transformer + +If your Transformer is generally applicable, you can raise a PR to include it in the core Transformer library +provided by DataHub. All Transformers will live under the `datahub_actions/plugin/transform` directory inside the +[datahub-actions](https://github.com/acryldata/datahub-actions) repository. + +Once you've added your new Transformer there, make sure that you make it discoverable by updating the `entry_points` section +of the `setup.py` file. This allows you to assign a globally unique name for you Transformer, so that people can use +it without defining the full module path. + +#### Prerequisites: + +Prerequisites to consideration for inclusion in the core Transformer library include + +- **Testing** Define unit tests for your Transformer +- **Deduplication** Confirm that no existing Transformer serves the same purpose, or can be easily extended to serve the same purpose diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-an-action.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-an-action.md new file mode 100644 index 00000000..b3c5d50b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/guides/developing-an-action.md @@ -0,0 +1,134 @@ +--- +title: Developing an Action +slug: /actions/guides/developing-an-action +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/guides/developing-an-action.md +--- +# Developing an Action + +In this guide, we will outline each step to developing a Action for the DataHub Actions Framework. + +## Overview + +Developing a DataHub Action is a matter of extending the `Action` base class in Python, installing your +Action to make it visible to the framework, and then configuring the framework to use the new Action. + +## Step 1: Defining an Action + +To implement an Action, we'll need to extend the `Action` base class and override the following functions: + +- `create()` - This function is invoked to instantiate the action, with a free-form configuration dictionary + extracted from the Actions configuration file as input. +- `act()` - This function is invoked when an Action is received. It should contain the core logic of the Action. +- `close()` - This function is invoked when the framework has issued a shutdown of the pipeline. It should be used + to cleanup any processes happening inside the Action. + +Let's start by defining a new implementation of Action called `CustomAction`. We'll keep it simple-- this Action will +print the configuration that is provided when it is created, and print any Events that it receives. + +```python +# custom_action.py +from datahub_actions.action.action import Action +from datahub_actions.event.event_envelope import EventEnvelope +from datahub_actions.pipeline.pipeline_context import PipelineContext + +class CustomAction(Action): + @classmethod + def create(cls, config_dict: dict, ctx: PipelineContext) -> "Action": + # Simply print the config_dict. + print(config_dict) + return cls(ctx) + + def __init__(self, ctx: PipelineContext): + self.ctx = ctx + + def act(self, event: EventEnvelope) -> None: + # Do something super important. + # For now, just print. :) + print(event) + + def close(self) -> None: + pass +``` + +## Step 2: Installing the Action + +Now that we've defined the Action, we need to make it visible to the framework by making it +available in the Python runtime environment. + +The easiest way to do this is to just place it in the same directory as your configuration file, in which case the module name is the same as the file +name - in this case it will be `custom_action`. + +### Advanced: Installing as a Package + +Alternatively, create a `setup.py` file in the same directory as the new Action to convert it into a package that pip can understand. + +``` +from setuptools import find_packages, setup + +setup( + name="custom_action_example", + version="1.0", + packages=find_packages(), + # if you don't already have DataHub Actions installed, add it under install_requires + # install_requires=["acryl-datahub-actions"] +) +``` + +Next, install the package + +```shell +pip install -e . +``` + +inside the module. (alt.`python setup.py`). + +Once we have done this, our class will be referencable via `custom_action_example.custom_action:CustomAction`. + +## Step 3: Running the Action + +Now that we've defined our Action, we can create an Action configuration file that refers to the new Action. +We will need to provide the fully-qualified Python module & class name when doing so. + +_Example Configuration_ + +```yaml +# custom_action.yaml +name: "custom_action_test" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +action: + type: "custom_action_example.custom_action:CustomAction" + config: + # Some sample configuration which should be printed on create. + config1: value1 +``` + +Next, run the `datahub actions` command as usual: + +```shell +datahub actions -c custom_action.yaml +``` + +If all is well, your Action should now be receiving & printing Events. + +## (Optional) Step 4: Contributing the Action + +If your Action is generally applicable, you can raise a PR to include it in the core Action library +provided by DataHub. All Actions will live under the `datahub_actions/plugin/action` directory inside the +[datahub-actions](https://github.com/acryldata/datahub-actions) repository. + +Once you've added your new Action there, make sure that you make it discoverable by updating the `entry_points` section +of the `setup.py` file. This allows you to assign a globally unique name for you Action, so that people can use +it without defining the full module path. + +### Prerequisites: + +Prerequisites to consideration for inclusion in the core Actions library include + +- **Testing** Define unit tests for your Action +- **Deduplication** Confirm that no existing Action serves the same purpose, or can be easily extended to serve the same purpose diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/quickstart.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/quickstart.md new file mode 100644 index 00000000..efcf7eeb --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/quickstart.md @@ -0,0 +1,175 @@ +--- +title: Quickstart +slug: /actions/quickstart +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/quickstart.md +--- +# DataHub Actions Quickstart + +## Prerequisites + +The DataHub Actions CLI commands are an extension of the base `datahub` CLI commands. We recommend +first installing the `datahub` CLI: + +```shell +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub +datahub --version +``` + +> Note that the Actions Framework requires a version of `acryl-datahub` >= v0.8.34 + +## Installation + +To install DataHub Actions, you need to install the `acryl-datahub-actions` package from PyPi + +```shell +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub-actions + +# Verify the installation by checking the version. +datahub actions version +``` + +### Hello World + +DataHub ships with a "Hello World" Action which logs all events it receives to the console. +To run this action, simply create a new Action configuration file: + +```yaml +# hello_world.yaml +name: "hello_world" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +action: + type: "hello_world" +``` + +and then run it using the `datahub actions` command: + +```shell +datahub actions -c hello_world.yaml +``` + +You should the see the following output if the Action has been started successfully: + +```shell +Action Pipeline with name 'hello_world' is now running. +``` + +Now, navigate to the instance of DataHub that you've connected to and perform an Action such as + +- Adding / removing a Tag +- Adding / removing a Glossary Term +- Adding / removing a Domain + +If all is well, you should see some events being logged to the console + +```shell +Hello world! Received event: +{ + "event_type": "EntityChangeEvent_v1", + "event": { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "category": "TAG", + "operation": "ADD", + "modifier": "urn:li:tag:pii", + "parameters": {}, + "auditStamp": { + "time": 1651082697703, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + }, + "version": 0, + "source": null + }, + "meta": { + "kafka": { + "topic": "PlatformEvent_v1", + "offset": 1262, + "partition": 0 + } + } +} +``` + +_An example of an event emitted when a 'pii' tag has been added to a Dataset._ + +Woohoo! You've successfully started using the Actions framework. Now, let's see how we can get fancy. + +#### Filtering events + +If we know which Event types we'd like to consume, we can optionally add a `filter` configuration, which +will prevent events that do not match the filter from being forwarded to the action. + +```yaml +# hello_world.yaml +name: "hello_world" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +filter: + event_type: "EntityChangeEvent_v1" +action: + type: "hello_world" +``` + +_Filtering for events of type EntityChangeEvent_v1 only_ + +#### Advanced Filtering + +Beyond simply filtering by event type, we can also filter events by matching against the values of their fields. To do so, +use the `event` block. Each field provided will be compared against the real event's value. An event that matches +**all** of the fields will be forwarded to the action. + +```yaml +# hello_world.yaml +name: "hello_world" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +filter: + event_type: "EntityChangeEvent_v1" + event: + category: "TAG" + operation: "ADD" + modifier: "urn:li:tag:pii" +action: + type: "hello_world" +``` + +_This filter only matches events representing "PII" tag additions to an entity._ + +And more, we can achieve "OR" semantics on a particular field by providing an array of values. + +```yaml +# hello_world.yaml +name: "hello_world" +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} +filter: + event_type: "EntityChangeEvent_v1" + event: + category: "TAG" + operation: ["ADD", "REMOVE"] + modifier: "urn:li:tag:pii" +action: + type: "hello_world" +``` + +_This filter only matches events representing "PII" tag additions to OR removals from an entity. How fancy!_ diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/datahub-cloud-event-source.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/datahub-cloud-event-source.md new file mode 100644 index 00000000..770e130d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/datahub-cloud-event-source.md @@ -0,0 +1,125 @@ +--- +title: DataHub Cloud Event Source +sidebar_label: Cloud Event Source +slug: /actions/sources/datahub-cloud-event-source +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/sources/datahub-cloud-event-source.md +--- +# DataHub Cloud Event Source + +## Prerequisites + +### Compatibility + +The **DataHub Cloud Event Source** is only compatible with versions of DataHub Cloud above `v0.3.7`. + +### Privileges + +By default, users do not have access to the Events API of DataHub Cloud. In order to access the API, the user or service account +associated with the access token used to configure this events source _must_ have the `Get Platform Events` platform privilege, which +can be granted using an [Access Policy](/docs/authorization/access-policies-guide/). + +## Overview + +The DataHub Cloud Event Source allows you to use DataHub Actions with an instance of DataHub Cloud hosted by [DataHub](https://acryl.io). + +Under the hood, the DataHub Cloud Event Source communicates with DataHub Cloud to extract change events in realtime. +The state of progress is automatically saved to DataHub Cloud after messages are processed, allowing you to seamlessly pause and restart the consumer, using the provided `name` to uniquely identify the consumer state. + +On initial startup of a new consumer id, the DataHub event source will automatically begin the _latest_ events by default. Afterwards, the message stream processed offsets will be continually saved. However, the source can also optionally be configured to "look back" in time +by a certain number of days on initial bootstrap using the `lookback_days` parameter. To reset all previously saved offsets for a consumer, +you can set `reset_offsets` to `True`. + +### Processing Guarantees + +This event source implements an "ack" function which is invoked if and only if an event is successfully processed +by the Actions framework, meaning that the event made it through the Transformers and into the Action without +any errors. Under the hood, the "ack" method synchronously commits DataHub Cloud Consumer Offsets on behalf of the Action. This means that by default, the framework provides _at-least once_ processing semantics. That is, in the unusual case that a failure occurs when attempting to commit offsets back to Kafka, that event may be replayed on restart of the Action. + +If you've configured your Action pipeline `failure_mode` to be `CONTINUE` (the default), then events which +fail to be processed will simply be logged to a `failed_events.log` file for further investigation (dead letter queue). The DataHub Cloud Event Source will continue to make progress against the underlying topics and continue to commit offsets even in the case of failed messages. + +If you've configured your Action pipeline `failure_mode` to be `THROW`, then events which fail to be processed result in an Action Pipeline error. This in turn terminates the pipeline before committing offsets back to DataHub Cloud. Thus the message will not be marked as "processed" by the Action consumer. + +## Supported Events + +The DataHub Cloud Event Source produces + +- [Entity Change Event V1](../../managed-datahub/datahub-api/entity-events-api.md) +- [Metadata Change Log V1](../events/metadata-change-log-event.md) (By changing config > topics to include `MetadataChangeLog_Versioned_v1` and `MetadataChangeLog_Timeseries_v1`) + +## Configure the Event Source + +Use the following config(s) to get started with the DataHub Cloud Event Source. + +### Quickstart + +To start listening for new events from now, you can use the following recipe: + +```yml +name: "unique-action-name" +datahub: + server: "https://.acryl.io" + token: "" +source: + type: "datahub-cloud" +action: + # action configs +``` + +Note that the `datahub` configuration block is **required** to connect to your DataHub Cloud instance. + +### Advanced Configurations + +To reset the offsets for the action pipeline and start consuming events from 7 days ago, you can use the following recipe: + +```yml +name: "unique-action-name" +datahub: + server: "https://.acryl.io" + token: "" +source: + type: "datahub-cloud" + config: + topics: ["PlatformEvent_v1"] # Add MetadataChangeLog_Versioned_v1 and / or MetadataChangeLog_Timeseries_v1 to generate raw MCL events. + lookback_days: 7 # Look back 7 days for events + reset_offsets: true # Ignore stored offsets and start fresh + infinite_retry: true # Enable infinite retry for connection failures (default: false) + kill_after_idle_timeout: true # Enable shutdown after idle period + idle_timeout_duration_seconds: 60 # Idle timeout set to 60 seconds + event_processing_time_max_duration_seconds: 45 # Max processing time of 45 seconds per batch +action: + # action configs +``` + +Note that the `datahub` configuration block is **required** to connect to your DataHub Cloud instance. + +
+ View All Configuration Options + +| Field | Required | Default | Description | +| -------------------------------------------- | :------: | :----------------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `topics` | ❌ | `PlatformEvent_v1` | The name of the topic from which events will be consumed. By default only produces `EntityChangeEvent_v1` events. To include `MetadataChangeLogEvent_v1` events, set this value to include ["MetadataChangeLog_Versioned_v1", "MetadataChangeLog_Timeseries_v1"] | +| `lookback_days` | ❌ | None | Optional number of days to look back when polling for events. | +| `reset_offsets` | ❌ | `False` | When set to `True`, the consumer will ignore any stored offsets and start fresh. | +| `infinite_retry` | ❌ | `False` | When set to `True`, the consumer will retry indefinitely on connection failures (HTTPError, ConnectionError, ChunkedEncodingError, Timeout) with exponential backoff (2s to 60s). When `False` (default), it retries up to 15 times before failing. | +| `kill_after_idle_timeout` | ❌ | `False` | If `True`, stops the consumer after being idle for the specified timeout duration. | +| `idle_timeout_duration_seconds` | ❌ | `30` | Duration in seconds after which, if no events are received, the consumer is considered idle. | +| `event_processing_time_max_duration_seconds` | ❌ | `30` | Maximum allowed time in seconds for processing events before timing out. | + +
+ +## FAQ + +1. Is there a way to always start processing from the end of the topics on Actions start? + +Yes, simply set `reset_offsets` to True for a single run of the action. Remember to disable this for subsequent runs if you don't want to miss any events! + +2. What happens if I have multiple actions with the same pipeline `name` running? Can I scale out horizontally? + +Today, there is undefined behavior deploying multiple actions with the same name using the DataHub Cloud Events Source. +All events must be processed by a single running action + +3. How do I handle transient connection failures? + +By default, the consumer will retry up to 15 times on connection failures (HTTPError, ConnectionError, ChunkedEncodingError, Timeout) with exponential backoff (2s to 60s). If you expect longer outages or want to ensure the consumer continues retrying indefinitely, set `infinite_retry: true` in the source configuration. When enabled, the consumer will retry indefinitely with exponential backoff (starting at 2 seconds, increasing up to 60 seconds, then continuing at 60 seconds) until the connection is restored. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/kafka-event-source.md b/docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/kafka-event-source.md new file mode 100644 index 00000000..d73899ca --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/actions/sources/kafka-event-source.md @@ -0,0 +1,134 @@ +--- +title: Kafka Event Source +slug: /actions/sources/kafka-event-source +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/actions/sources/kafka-event-source.md +--- +# Kafka Event Source + +## Overview + +The Kafka Event Source is the default Event Source used within the DataHub Actions Framework. + +Under the hood, the Kafka Event Source uses a Kafka Consumer to subscribe to the topics streaming +out of DataHub (MetadataChangeLog_v1, PlatformEvent_v1). Each Action is automatically placed into a unique +[consumer group](https://docs.confluent.io/platform/current/clients/consumer.html#consumer-groups) based on +the unique `name` provided inside the Action configuration file. + +This means that you can easily scale-out Actions processing by sharing the same Action configuration file across +multiple nodes or processes. As long as the `name` of the Action is the same, each instance of the Actions framework will subscribe as a member in the same Kafka Consumer Group, which allows for load balancing the +topic traffic across consumers which each consume independent [partitions](https://developer.confluent.io/learn-kafka/apache-kafka/partitions/#kafka-partitioning). + +Because the Kafka Event Source uses consumer groups by default, actions using this source will be **stateful**. +This means that Actions will keep track of their processing offsets of the upstream Kafka topics. If you +stop an Action and restart it sometime later, it will first "catch up" by processing the messages that the topic +has received since the Action last ran. Be mindful of this - if your Action is computationally expensive, it may be preferable to start consuming from the end of the log, instead of playing catch up. The easiest way to achieve this is to simply rename the Action inside the Action configuration file - this will create a new Kafka Consumer Group which will begin processing new messages at the end of the log (latest policy). + +### Processing Guarantees + +This event source implements an "ack" function which is invoked if and only if an event is successfully processed +by the Actions framework, meaning that the event made it through the Transformers and into the Action without +any errors. Under the hood, the "ack" method synchronously commits Kafka Consumer Offsets on behalf of the Action. This means that by default, the framework provides _at-least once_ processing semantics. That is, in the unusual case that a failure occurs when attempting to commit offsets back to Kafka, that event may be replayed on restart of the Action. + +If you've configured your Action pipeline `failure_mode` to be `CONTINUE` (the default), then events which +fail to be processed will simply be logged to a `failed_events.log` file for further investigation (dead letter queue). The Kafka Event Source will continue to make progress against the underlying topics and continue to commit offsets even in the case of failed messages. + +If you've configured your Action pipeline `failure_mode` to be `THROW`, then events which fail to be processed result in an Action Pipeline error. This in turn terminates the pipeline before committing offsets back to Kafka. Thus the message will not be marked as "processed" by the Action consumer. + +## Supported Events + +The Kafka Event Source produces + +- [Entity Change Event V1](../events/entity-change-event.md) +- [Metadata Change Log V1](../events/metadata-change-log-event.md) + +## Configure the Event Source + +Use the following config(s) to get started with the Kafka Event Source. + +```yml +name: "pipeline-name" +source: + type: "kafka" + config: + # Connection-related configuration + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} + # Dictionary of freeform consumer configs propagated to underlying Kafka Consumer + consumer_config: + #security.protocol: ${KAFKA_PROPERTIES_SECURITY_PROTOCOL:-PLAINTEXT} + #ssl.keystore.location: ${KAFKA_PROPERTIES_SSL_KEYSTORE_LOCATION:-/mnt/certs/keystore} + #ssl.truststore.location: ${KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION:-/mnt/certs/truststore} + #ssl.keystore.password: ${KAFKA_PROPERTIES_SSL_KEYSTORE_PASSWORD:-keystore_password} + #ssl.key.password: ${KAFKA_PROPERTIES_SSL_KEY_PASSWORD:-keystore_password} + #ssl.truststore.password: ${KAFKA_PROPERTIES_SSL_TRUSTSTORE_PASSWORD:-truststore_password} + # Topic Routing - which topics to read from. + topic_routes: + mcl: ${METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME:-MetadataChangeLog_Versioned_v1} # Topic name for MetadataChangeLogEvent_v1 events. + pe: ${PLATFORM_EVENT_TOPIC_NAME:-PlatformEvent_v1} # Topic name for PlatformEvent_v1 events. +action: + # action configs +``` + +
+ View All Configuration Options + + | Field | Required | Default | Description | + | --- | :-: | :-: | --- | + | `connection.bootstrap` | ✅ | N/A | The Kafka bootstrap URI, e.g. `localhost:9092`. | + | `connection.schema_registry_url` | ✅ | N/A | The URL for the Kafka schema registry, e.g. `http://localhost:8081` | + | `connection.consumer_config` | ❌ | {} | A set of key-value pairs that represents arbitrary Kafka Consumer configs | + | `topic_routes.mcl` | ❌ | `MetadataChangeLog_v1` | The name of the topic containing MetadataChangeLog events | + | `topic_routes.pe` | ❌ | `PlatformEvent_v1` | The name of the topic containing PlatformEvent events | +
+ +## Schema Registry Configuration + +The Kafka Event Source requires a schema registry to deserialize events. There are several ways to configure the schema registry: + +### Default Schema Registry + +When using the default schema registry that comes with DataHub, you can use the internal URL: + +```yml +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: "http://datahub-datahub-gms:8080/schema-registry/api/" +``` + +Note: If you're running this outside the DataHub cluster, you'll need to map this internal URL to an externally accessible URL. + +### External Schema Registry + +For external schema registries (like Confluent Cloud), you'll need to provide the full URL and any necessary authentication: + +```yml +source: + type: "kafka" + config: + connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: "https://your-schema-registry-url" + schema_registry_config: + basic.auth.user.info: "${REGISTRY_API_KEY_ID}:${REGISTRY_API_KEY_SECRET}" +``` + +### AWS Glue Schema Registry + +If you're using AWS Glue Schema Registry, you'll need to configure it differently. See the [AWS deployment guide](../../deploy/aws.md/#aws-glue-schema-registry) for details. + +## FAQ + +1. Is there a way to always start processing from the end of the topics on Actions start? + +Currently, the only way is to change the `name` of the Action in its configuration file. In the future, +we are hoping to add first-class support for configuring the action to be "stateless", ie only process +messages that are received while the Action is running. + +2. Is there a way to asynchronously commit offsets back to Kafka? + +Currently, all consumer offset commits are made synchronously for each message received. For now we've optimized for correctness over performance. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/api-tracing.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/api-tracing.md new file mode 100644 index 00000000..b7c221f9 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/api-tracing.md @@ -0,0 +1,439 @@ +--- +title: API Tracing +slug: /advanced/api-tracing +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/api-tracing.md +--- +# API Tracing + +## Introduction + +DataHub's asynchronous APIs enable high-volume data operations, particularly for bulk ingestion processes. While these +APIs optimize throughput, they previously lacked built-in validation mechanisms for operation status. Consequently, +detecting processing issues required direct monitoring of backend system metrics and logs. + +To address this limitation, DataHub implemented a trace/request ID system that enables end-to-end tracking of write +operations. This tracing mechanism is particularly crucial given DataHub's multi-stage write architecture, where data +propagates through multiple components and persists across distinct storage systems. The trace ID maintains continuity +throughout this complex processing pipeline, providing visibility into the operation's status at each stage of execution. + +The system effectively balances the performance benefits of asynchronous processing with the operational necessity of +request tracking and validation. This enhancement significantly improves observability without compromising the +throughput advantages of bulk operations. + +## Architecture Overview + +Shown below is the write path for an asynchronous write within DataHub. For more information about MCPs please see +the documentation on [MetadataChangeProposal & MetadataChangeLog Events](/docs/advanced/mcp-mcl.md). + +

+ +

+ +A successful write operation requires data persistence in at least one storage system, though typically both primary and +search storage systems must be updated. The storage architecture consists of two main components: + +- Primary Storage: Comprises MySQL, Postgres, or Cassandra, serving as the persistent store for all non-Timeseries aspects. +- Search Storage: Utilizes either Elasticsearch or OpenSearch systems. + +In most operational scenarios, write operations must successfully complete across both storage layers to maintain system +consistency and ensure complete data availability. + +## Trace API + +Known Limitations: + +- Tracing can fail in some cases where a batch of aspects introduces a conflict between a system generated aspect and one + included within a batch. An example, `upstreamLineage` and `siblings` aspects together may conflict with a system generated + `siblings` aspect normally generated from `upstreamLineage` in an MCL hook. + +The trace API's status retrieval functionality requires three key identifiers to locate specific write operations: +the trace ID (unique to the request), the URN, and the aspect name. This combination of identifiers ensures precise +operation tracking within the system. + +For batch operations involving multiple URNs and aspects, a single trace ID is assigned to monitor the entire request. +In asynchronous mode, the system maintains independent status tracking for each aspect within the batch, allowing for +granular operation monitoring. + +The API returns a comprehensive status report that includes: + +- Per-aspect success/failure status +- Detailed status breakdowns for each storage system +- Write states as defined in the [Write States](#Write-States) documentation +- Error information from MCP processing, when applicable, to facilitate debugging + +This structured approach to status reporting enables precise monitoring of complex write operations across the system's +various components. + +### Retrieving the `trace id` + +DataHub's asynchronous APIs provide trace ID information through two distinct mechanisms: + +- HTTP Response Header: A W3C-compliant `traceparent` header is included in all API responses + The complete header value serves as a valid trace ID +- System Metadata: For OpenAPI v3 APIs and those returning systemMetadata, the trace ID is accessible via the + `telemetryTraceId` property within systemMetadata + +While these two trace ID formats differ structurally—with the `traceparent` adhering to W3C's Trace Context +specification—both formats are fully compatible with the Trace API for operation tracking purposes. + +Header Example: + +```text +traceparent: 00-00062c53a468cbd8077e7dd079846870-9199effb49910b4e-01 +``` + +`SystemMetadata` Example: + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "status": { + "value": { + "removed": false + }, + "systemMetadata": { + "properties": { + "telemetryLog": "false", + "telemetryQueueSpanId": "ee9e40edcb66ce4f", + "telemetryTraceId": "00062c53a468cbd8077e7dd079846870", + "telemetryEnqueuedAt": "1737587612508" + } + } + } + } +] +``` + +### Write States + +As mentioned earlier, there are multiple states for an aspect write both storage systems. These states are as follows: + +| Write State | Description | +| ----------------------- | ------------------------------------------------------------------------------------------------------ | +| `ERROR` | This state indicates an error occurred when processing the write request. | +| `PENDING` | A pending state indicates that the write is queued and the consumer has not yet processed the message. | +| `ACTIVE_STATE` | The write was successful and is the current value. | +| `HISTORIC_STATE` | The write was successful, however it has been overwritten by a newer value. | +| `NO_OP` | The write is not applicable for a given storage system. | +| `UNKNOWN` | We are unable to determine the state of the write and no record of its failure exists either. | +| `TRACE_NOT_IMPLEMENTED` | We have not yet implemented tracing a particular aspect type. This applies to Timeseries aspects. | + +### Using the Trace API + +The Trace API is implemented as an OpenAPI endpoint and can be used both programmatically and through the Swagger UI. + +Required Values: + +- `traceId` - The `trace id` associated with the write request. See the previous [Retrieving the `trace id`](#retrieving-the-trace-id) section for how to find this id. +- URN/Aspect names - These are passed as a POST body and should represent at least a subset of the URN/aspects from the initial request. + An example is shown here for a single URN and 2 aspects [`datasetInfo`, `status`]. + ```json + { + "urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": [ + "datasetInfo", + "status" + ] + } + ``` +- Authorization token + +Optional Parameters: + +- `onlyIncludeErrors` (default: `true`) - If this parameter is set to `true`, the response will only include status information on the failed aspects. +- `detailed` (default: `false`) - If set to `true`, will include detailed information from exceptions for failed MCPs. +- `skipCache` (default: `false`) - If set to `true`, will bypass a short-lived cache of the kafka consumer group offsets. + +The following shows a few examples of requests/response pairs. + +- Successful Write + - Request for URN `urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)` and aspect `status` + ```shell + curl -v 'http://localhost:8080/openapi/v1/trace/write/00062c2b698bcb28e92508f8f311802d?onlyIncludeErrors=false&detailed=true&skipCache=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' \ + -d '{ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": [ + "status" + ] + }' | jq + ``` + - Example response + ```json + { + "urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": { + "status": { + "success": true, + "primaryStorage": { + "writeStatus": "ACTIVE_STATE" + }, + "searchStorage": { + "writeStatus": "ACTIVE_STATE" + } + } + } + } + ``` +- Error with exception details + - Example request + ```shell + curl -v 'http://localhost:8080/openapi/v1/trace/write/00062c543e4550c8400e6f6864471a20?onlyIncludeErrors=true&detailed=true&skipCache=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' \ + -d '{"urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": ["status"]}' + ``` + - Example response + ```json + { + "urn:li:dataset:(urn:li:dataPlatform:bigquery,transactions.user_profile,PROD)": { + "status": { + "success": false, + "primaryStorage": { + "writeStatus": "ERROR", + "writeExceptions": [ + { + "message": "Expected version -100000, actual version -1", + "exceptionClass": "com.linkedin.metadata.aspect.plugins.validation.AspectValidationException", + "stackTrace": [ + "com.linkedin.metadata.aspect.plugins.validation.AspectValidationException.forPrecondition(AspectValidationException.java:33)", + "com.linkedin.metadata.aspect.plugins.validation.AspectValidationException.forPrecondition(AspectValidationException.java:25)", + "com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.validateVersionPrecondition(ConditionalWriteValidator.java:152)", + "com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.lambda$validatePreCommitAspects$2(ConditionalWriteValidator.java:100)", + "java.base/java.util.Optional.flatMap(Optional.java:289)", + "com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.validatePreCommitAspects(ConditionalWriteValidator.java:98)", + "com.linkedin.metadata.aspect.plugins.validation.AspectPayloadValidator.validatePreCommit(AspectPayloadValidator.java:38)", + "com.linkedin.metadata.aspect.batch.AspectsBatch.lambda$validatePreCommit$4(AspectsBatch.java:129)", + "java.base/java.util.stream.ReferencePipeline$7$1.accept(ReferencePipeline.java:273)", + "java.base/java.util.ArrayList$ArrayListSpliterator.forEachRemaining(ArrayList.java:1625)", + "java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509)", + "java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:499)", + "java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)", + "java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)", + "java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)", + "java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596)", + "com.linkedin.metadata.aspect.batch.AspectsBatch.validatePreCommit(AspectsBatch.java:130)" + ] + } + ] + }, + "searchStorage": { + "writeStatus": "ERROR", + "writeMessage": "Primary storage write failed." + } + } + } + } + ``` + +### Ingestion Tracing (Experimental) + +Known Limitations: + +- Only OpenAPI is supported +- ASYNC_WAIT/ASYNC - Async modes are impacted by kafka lag. + +Ingestion can be used with tracing enabled however it does require using a `ASYNC_WAIT` emit mode as well as the `OPENAPI`. This +can be enabled by setting a couple environment variables shown below. + +```shell +DATAHUB_REST_EMITTER_DEFAULT_ENDPOINT=OPENAPI \ +DATAHUB_EMIT_MODE=ASYNC_WAIT \ +datahub ingest ... +``` + +### Client-Side Trace Logging + +For debugging purposes, you can enable client-side trace ID logging in the Python REST emitter by setting the +`DATAHUB_EMITTER_TRACE` environment variable. When enabled, the emitter will log trace IDs, timestamps, URNs, and +aspect names for each MCP emission. + +```shell +DATAHUB_EMITTER_TRACE=true \ +datahub ingest ... +``` + +Example log output: + +```text +INFO - MCP trace_id=00062c53a468cbd8077e7dd079846870 timestamp=1737587612508 urn=urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD) aspect=status +``` + +For batch emissions, the log will show all URNs in the batch: + +```text +INFO - MCP batch trace_id=00062c53a468cbd8077e7dd079846870 timestamp=1737587612508 urns=['urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_orders,PROD)'] +``` + +## Trace Performance + +The Trace API's performance profile varies based on operation status: + +Successful Operations: + +- Optimal performance through direct storage access +- Requires single lookup operations from SQL and Elasticsearch +- Bypasses Kafka interaction entirely + +Error State Operations: + +- Performance impact due to required Kafka topic inspection + - Optimization mechanisms implemented: + - Timestamp-based offset seeking for efficient topic traversal + - Parallel trace processing with controlled concurrency + - Offset caching system to enhance response times + - Cache bypass available via skipCache parameter when data currency is critical + +The performance differential between success and error states stems primarily from the additional overhead of Kafka +topic inspection required for error tracking and diagnosis. + +For more detail, please see the [Design Notes](#design-notes) section. + +### Real World Test + +A test was executed with 627,712 aspects in `ASYNC_BATCH` and averaged over 2 runs. The overhead from tracing introduced +a 3.84% increase in runtime. Example test runs shown below. + +Without Tracing: + +```json +{ + "total_records_written": 627712, + "records_written_per_second": 376, + "total_duration_in_seconds": 1665.25, + "mode": "ASYNC_BATCH", + "max_threads": 15, + "gms_version": "v0.3.9.3", + "pending_requests": 0, + "async_batches_prepared": 6290, + "async_batches_split": 0, + "main_thread_blocking_timer": "1186.301 seconds" +} +``` + +With Tracing: + +```json +{ + "total_records_written": 627612, + "records_written_per_second": 368, + "total_duration_in_seconds": 1701.51, + "mode": "ASYNC_BATCH", + "max_threads": 15, + "gms_version": "v0.3.9.3", + "pending_requests": 0, + "async_batches_prepared": 6291, + "async_batches_split": 0, + "main_thread_blocking_timer": "1224.738 seconds" +} +``` + +## Trace Exporters + +At the foundation of the trace instrumentation is OpenTelemetry which has been a part of DataHub for quite some time. As +documented in the [Monitoring](/docs/advanced/monitoring.md) section, OpenTelemetry can be configured to export traces +to external systems. For the Trace API to function, this external system is NOT required. + +### Trace Log Export + +A special log-based OpenTelemetry exporter was implemented for debugging purposes. When selectively activated for a given +request it will print `trace id`s and detailed timing information as the request traverses the different components of DataHub. +The output of these logs is also not required for the Trace API to function, however it leverages the same underlying OpenTelemetry +foundation. + +Activating a trace log is done using one of these methods: + +- HTTP Header: `X-Enable-Trace-Log: true` +- Cookie: `enable-trace-log: true` + - javascript: `document.cookie = "enable-trace-log=true";` + +Example logs for a single request with tracing logging enabled: + +- GMS + +```text +i.d.metadata.context.RequestContext:53 - RequestContext{actorUrn='urn:li:corpuser:datahub', sourceIP='172.18.0.5', requestAPI=OPENAPI, requestID='createAspect([dataset])', userAgent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36'} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: a2898a18f9f0c4f1, ParentId: dd746f079d1232ba, Name: ingestTimeseriesProposal, Duration: 0.03 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={async=true, batch.size=1}, capacity=128, totalAddedValues=2} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 02e058ff616e4c99, ParentId: 7ed88659811a8fdb, Name: produceMetadataChangeProposal, Duration: 0.03 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, messaging.system=kafka, messaging.destination=MetadataChangeProposal_v1, messaging.operation=publish, queue.enqueued_at=1737418391958}, capacity=128, totalAddedValues=5} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 7ed88659811a8fdb, ParentId: dd746f079d1232ba, Name: ingestProposalAsync, Duration: 2.57 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1} +``` + +- MCE Consumer + +```text +c.l.m.k.MetadataChangeProposalsProcessor:89 - Got MCP event key: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD), topic: MetadataChangeProposal_v1, partition: 0, offset: 75, value size: 412, timestamp: 1737418391959 +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: a65075fe0982d873, ParentId: 02e058ff616e4c99, Name: consume, Duration: 0.01 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, queue.duration_ms=4, messaging.system=kafka, messaging.destination=MetadataChangeProposal_v1, messaging.operation=receive, queue.enqueued_at=1737418391958}, capacity=128, totalAddedValues=6} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: dd746f079d1232ba, ParentId: 0000000000000000, Name: POST /openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cclimate.daily_temperature%2CPROD%29/status, Duration: 16.18 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={request.api=OPENAPI, http.status_code=202, user.id=urn:li:corpuser:datahub, http.url=/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cclimate.daily_temperature%2CPROD%29/status, request.id=createAspect([dataset]), http.method=POST}, capacity=128, totalAddedValues=6} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 94a019b95154c0e7, ParentId: 0cb378fe4f5ad185, Name: ingestProposalSync, Duration: 0.01 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=0}, capacity=128, totalAddedValues=1} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 0cb378fe4f5ad185, ParentId: 68df6bc4729dc0a2, Name: ingestTimeseriesProposal, Duration: 0.25 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={async=false, batch.size=1}, capacity=128, totalAddedValues=2} +c.l.m.entity.EntityServiceImpl:988 - Ingesting aspects batch to database: AspectsBatchImpl{items=[ChangeMCP{changeType=UPSERT, urn=urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD), aspectName='status', recordTemplate={removed=false}, systemMetadata={lastObserved=1737418391954, version=1, properties={telemetryLog=true, telemetryQueueSpanId=02e058ff616e4c99, telemetryEnqueu...}]} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 4754a1c02dadec4c, ParentId: ef383b26f0040fc5, Name: retentionService, Duration: 0.09 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: ef383b26f0040fc5, ParentId: 7ae629151400fc18, Name: ingestAspectsToLocalDB, Duration: 18.64 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1, dwizName=com.linkedin.metadata.entity.EntityServiceImpl.ingestAspectsToLocalDB}, capacity=128, totalAddedValues=2} +c.l.m.entity.EntityServiceImpl:1900 - Producing MCL for ingested aspect status, urn urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD) +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: f1a8a1da99f1ae23, ParentId: c5f8b3884060722c, Name: produceMetadataChangeLog, Duration: 0.10 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, messaging.system=kafka, messaging.destination=MetadataChangeLog_Versioned_v1, messaging.operation=publish, queue.enqueued_at=1737418391982}, capacity=128, totalAddedValues=5} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: c5f8b3884060722c, ParentId: 7ae629151400fc18, Name: emitMCL, Duration: 14.32 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 7ae629151400fc18, ParentId: 68df6bc4729dc0a2, Name: ingestProposalSync, Duration: 37.90 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1}, capacity=128, totalAddedValues=1} +c.l.m.k.MetadataChangeProposalsProcessor:128 - Successfully processed MCP event urn: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD) +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 68df6bc4729dc0a2, ParentId: 02e058ff616e4c99, Name: consume, Duration: 39.11 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1, dwizName=com.linkedin.metadata.kafka.MetadataChangeProposalsProcessor.consume}, capacity=128, totalAddedValues=2} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 04dc44653b634df2, ParentId: 02e058ff616e4c99, Name: consume, Duration: 0.03 ms +``` + +- MAE Consumer + +```text +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={messaging.destination_kind=topic, queue.duration_ms=22, messaging.system=kafka, messaging.destination=MetadataChangeLog_Versioned_v1, messaging.operation=receive, queue.enqueued_at=1737418391982}, capacity=128, totalAddedValues=6} +c.l.metadata.kafka.MCLKafkaListener:96 - Invoking MCL hooks for consumer: generic-mae-consumer-job-client urn: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD), aspect name: status, entity type: dataset, change type: UPSERT +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 3c3c055c360dc8e4, ParentId: 1de99215a0e82697, Name: FormAssignmentHook, Duration: 0.06 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.FormAssignmentHook_latency}, capacity=128, totalAddedValues=1} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 8e238d0156baacc4, ParentId: 1de99215a0e82697, Name: IngestionSchedulerHook, Duration: 0.05 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.IngestionSchedulerHook_latency}, capacity=128, totalAddedValues=1} +c.l.m.s.e.update.ESBulkProcessor:85 - Added request id: urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cclimate.daily_temperature%2CPROD%29, operation type: UPDATE, index: datasetindex_v2 +c.l.m.s.e.update.ESBulkProcessor:85 - Added request id: SIHRXj1ktF7qkwPBZO8w0A==, operation type: UPDATE, index: system_metadata_service_v1 +c.l.m.s.e.update.ESBulkProcessor:85 - Added request id: 2p3742l4sFS3wcL82Qh2lQ==, operation type: UPDATE, index: system_metadata_service_v1 +c.l.m.s.e.update.ESBulkProcessor:85 - Added request id: CfZKRLsf25/e3p3mURzlnA==, operation type: UPDATE, index: system_metadata_service_v1 +c.l.m.s.e.update.ESBulkProcessor:85 - Added request id: 8tvhG5ARd5BOdEbqaZkE0g==, operation type: UPDATE, index: system_metadata_service_v1 +c.l.m.s.e.update.ESBulkProcessor:85 - Added request id: rAvQOOBItiKAym622S4dcQ==, operation type: UPDATE, index: system_metadata_service_v1 +c.l.m.s.e.update.ESBulkProcessor:85 - Added request id: YqT6TNy7MAMOAyVXh6abMA==, operation type: UPDATE, index: system_metadata_service_v1 +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 054ac726204b449c, ParentId: 1de99215a0e82697, Name: UpdateIndicesHook, Duration: 47.31 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.UpdateIndicesHook_latency}, capacity=128, totalAddedValues=1} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 14d9ded49a94c7b8, ParentId: 1de99215a0e82697, Name: IncidentsSummaryHook, Duration: 0.09 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.IncidentsSummaryHook_latency}, capacity=128, totalAddedValues=1} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: a92d9e54ade6073b, ParentId: 1de99215a0e82697, Name: EntityChangeEventGeneratorHook, Duration: 9.10 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.EntityChangeEventGeneratorHook_latency}, capacity=128, totalAddedValues=1} +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: c06dc7f131e57fca, ParentId: 1de99215a0e82697, Name: SiblingAssociationHook, Duration: 0.07 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.SiblingAssociationHook_latency}, capacity=128, totalAddedValues=1} +c.l.metadata.kafka.MCLKafkaListener:139 - Successfully completed MCL hooks for consumer: generic-mae-consumer-job-client urn: urn:li:dataset:(urn:li:dataPlatform:snowflake,climate.daily_temperature,PROD) +i.d.metadata.context.TraceContext:366 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, SpanId: 1de99215a0e82697, ParentId: 02e058ff616e4c99, Name: consume, Duration: 58.67 ms +i.d.metadata.context.TraceContext:376 - Trace: 00062c2c3e1403109bbaf3d2e39adcd0, Attributes: AttributesMap{data={batch.size=1, dwizName=com.linkedin.metadata.kafka.MCLKafkaListener.consume}, capacity=128, totalAddedValues=2} +``` + +## Design Notes + +For the initial implementation no specific OpenTelemetry infrastructure is required, however existing environment variables +for OpenTelemetry can continue to be used and will export the new spans if configured. + +The Trace API implementation does not rely on any additional external systems or infrastructure. Due to this design +choice, the trace is determined by inspecting the 3 storage systems (Primary Storage (SQL/Cassandra), Elasticsearch/Opensearch, +Kafka topics) for the `trace id` or related timestamps. + +The `trace id` is stored in systemMetadata in both SQL and ES. For ES specifically, the presence of the `trace id` in +the system metadata index is used as a proxy to determine a successful write to ES. + +The tracing feature will additionally fetch messages from the kafka topics (including the failed MCP topic) for +more detailed error information. Pending states are derived from offsets of the message vs the current offsets of the +consumer groups. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/aspect-versioning.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/aspect-versioning.md new file mode 100644 index 00000000..81571b51 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/aspect-versioning.md @@ -0,0 +1,63 @@ +--- +title: Aspect Versioning +slug: /advanced/aspect-versioning +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/aspect-versioning.md +--- +# Aspect Versioning + +As each version of [metadata aspect](../what/aspect.md) is immutable, any update to an existing aspect results in the creation of a new version. Typically one would expect the version number increases sequentially with the largest version number being the latest version, i.e. `v1` (oldest), `v2` (second oldest), ..., `vN` (latest). However, this approach results in major challenges in both rest.li modeling & transaction isolation and therefore requires a rethinking. + +## Rest.li Modeling + +As it's common to create dedicated rest.li sub-resources for a specific aspect, e.g. `/datasets/{datasetKey}/ownership`, the concept of versions become an interesting modeling question. Should the sub-resource be a [Simple](https://linkedin.github.io/rest.li/modeling/modeling#simple) or a [Collection](https://linkedin.github.io/rest.li/modeling/modeling#collection) type? + +If Simple, the [GET](https://linkedin.github.io/rest.li/user_guide/restli_server#get) method is expected to return the latest version, and the only way to retrieve non-latest versions is through a custom [ACTION](https://linkedin.github.io/rest.li/user_guide/restli_server#action) method, which is going against the [REST](https://en.wikipedia.org/wiki/Representational_state_transfer) principle. As a result, a Simple sub-resource doesn't seem to a be a good fit. + +If Collection, the version number naturally becomes the key so it's easy to retrieve specific version number using the typical GET method. It's also easy to list all versions using the standard [GET_ALL](https://linkedin.github.io/rest.li/user_guide/restli_server#get_all) method or get a set of versions via [BATCH_GET](https://linkedin.github.io/rest.li/user_guide/restli_server#batch_get). However, Collection resources don't support a simple way to get the latest/largest key directly. To achieve that, one must do one of the following + +- a GET_ALL (assuming descending key order) with a page size of 1 +- a [FINDER](https://linkedin.github.io/rest.li/user_guide/restli_server#finder) with special parameters and a page size of 1 +- a custom ACTION method again + +None of these options seems like a natural way to ask for the latest version of an aspect, which is one of the most common use cases. + +## Transaction Isolation + +[Transaction isolation](https://en.wikipedia.org/wiki/Isolation_(database_systems)) is a complex topic so make sure to familiarize yourself with the basics first. + +To support concurrent update of a metadata aspect, the following pseudo DB operations must be run in a single transaction, + +``` +1. Retrieve the current max version (Vmax) +2. Write the new value as (Vmax + 1) +``` + +Operation 1 above can easily suffer from [Phantom Reads](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Phantom_reads). This subsequently leads to Operation 2 computing the incorrect version and thus overwrites an existing version instead of creating a new one. + +One way to solve this is by enforcing [Serializable](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) isolation level in DB at the [cost of performance](https://logicalread.com/optimize-mysql-perf-part-2-mc13/#.XjxSRSlKh1N). In reality, very few DB even supports this level of isolation, especially for distributed document stores. It's more common to support [Repeatable Reads](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Repeatable_reads) or [Read Committed](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Read_committed) isolation levels—sadly neither would help in this case. + +Another possible solution is to transactionally keep track of `Vmax` directly in a separate table to avoid the need to compute that through a `select` (thus prevent Phantom Reads). However, cross-table/document/entity transaction is not a feature supported by all distributed document stores, which precludes this as a generalized solution. + +## Solution: Version 0 + +The solution to both challenges turns out to be surprisingly simple. Instead of using a "floating" version number to represent the latest version, one can use a "fixed/sentinel" version number instead. In this case we choose Version 0 as we want all non-latest versions to still keep increasing sequentially. In other words, it'd be `v0` (latest), `v1` (oldest), `v2` (second oldest), etc. Alternatively, you can also simply view all the non-zero versions as an audit trail. + +Let's examine how Version 0 can solve the aforementioned challenges. + +### Rest.li Modeling + +With Version 0, getting the latest version becomes calling the GET method of a Collection aspect-specific sub-resource with a deterministic key, e.g. `/datasets/{datasetkey}/ownership/0`, which is a lot more natural than using GET_ALL or FINDER. + +### Transaction Isolation + +The pseudo DB operations change to the following transaction block with version 0, + +``` +1. Retrieve v0 of the aspect +2. Retrieve the current max version (Vmax) +3. Write the old value back as (Vmax + 1) +4. Write the new value back as v0 +``` + +While Operation 2 still suffers from potential Phantom Reads and thus corrupting existing version in Operation 3, Repeatable Reads isolation level will ensure that the transaction fails due to [Lost Update](https://codingsight.com/the-lost-update-problem-in-concurrent-transactions/) detected in Operation 4. Note that this happens to also be the [default isolation level](https://dev.mysql.com/doc/refman/8.0/en/innodb-transaction-isolation-levels.html) for InnoDB in MySQL. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/backfilling.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/backfilling.md new file mode 100644 index 00000000..eaeb5c5e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/backfilling.md @@ -0,0 +1,9 @@ +--- +title: Backfilling Search Index & Graph DB +slug: /advanced/backfilling +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/backfilling.md +--- +# Backfilling Search Index & Graph DB + +WIP diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/bootstrap-mcps.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/bootstrap-mcps.md new file mode 100644 index 00000000..e36332c6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/bootstrap-mcps.md @@ -0,0 +1,189 @@ +--- +title: Bootstrap MetadataChangeProposals (MCPs) +slug: /advanced/bootstrap-mcps +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/bootstrap-mcps.md +--- +# Bootstrap MetadataChangeProposals (MCPs) + +Bootstrap MCPs are templated MCPs which are loaded when the `system-update` job runs. This allows adding +entities and aspects to DataHub at install time with the ability to customize them via environment variable +overrides. + +The built-in bootstrap MCP process can also be extended with custom MCPs. This can streamline deployment +scenarios where a set of standard ingestion recipes, data platforms, users groups, or other configuration +can be applied without the need for developing custom scripts. + +## Process Overview + +When DataHub is installed or upgraded, a job runs called `system-update`, this job is responsible for data +migration (particularly Elasticsearch indices) and ensuring the data is prepared for the next version of +DataHub. This is the job which will also apply the bootstrap MCPs. + +The `system-update` job, depending on configuration, can be split into two sequences of steps. If they are +not split, then all steps are blocking. + +1. An initial blocking sequence which is run prior to the new version of GMS and other components +2. Second sequence of steps where GMS and other components are allowed to run while additional data migration steps are + continued in the background + +When applying bootstrap MCPs `system-update` will perform the following steps: + +1. The `bootstrap_mcps.yaml` file is read, either from a default classpath location, `bootstrap_mcps.yaml`, or a filesystem location + provided by an environment variable, `SYSTEM_UPDATE_BOOTSTRAP_MCP_CONFIG`. +2. Depending on the mode of blocking or non-blocking each entry in the configuration file will be executed in sequence. +3. The template MCP file is loaded either from the classpath, or a filesystem location, and the template values are applied. +4. The rendered template MCPs are executed with the options specified in the `bootstrap_mcps.yaml`. + +## `bootstrap_mcps.yaml` Configuration + +The `bootstrap_mcps.yaml` file has the following format. + +```yaml +bootstrap: + templates: + - name: + version: + force: false + blocking: false + async: true + optional: false + mcps_location: + values_env: +``` + +Each entry in the list of templates points to a single yaml file which can contain one or more MCP objects. The +execution of the template MCPs is tracked by name and version to prevent re-execution. The MCP objects are executed once +unless `force=true` for each `name`/`version` combination. + +See the following table of options for descriptions of each field in the template configuration. + +| Field | Default | Required | Description | +| ------------- | ------- | -------- | ---------------------------------------------------------------------------------------------------------- | +| name | | `true` | The name for the collection of template MCPs. | +| version | | `true` | A string version for the collection of template MCPs. | +| force | `false` | `false` | Ignores the previous run history, will not skip execution if run previously. | +| blocking | `false` | `false` | Run before GMS and other components during upgrade/install if running in split blocking/non-blocking mode. | +| async | `true` | `false` | Controls whether the MCPs are executed for sync or async ingestion. | +| optional | `false` | `false` | Whether to ignore a failure or fail the entire `system-update` job. | +| mcps_location | | `true` | The location of the file which contains the template MCPs | +| values_env | | `false` | The environment variable which contains override template values. | + +## Template MCPs + +Template MCPs are stored in a yaml file which uses the mustache templating library to populate values from an optional environment +variable. Defaults can be provided inline making override only necessary when providing install/upgrade time configuration. + +In general the file contains a list of MCPs which follow the schema definition for MCPs exactly. Any valid field for an MCP +is accepted, including optional fields such as `headers`. + +### Example: Native Group + +An example template MCP collection, configuration, and values environment variable is shown below which would create a native group. + +```yaml +- entityUrn: urn:li:corpGroup:{{group.id}} + entityType: corpGroup + aspectName: corpGroupInfo + changeType: UPSERT + aspect: + description: {{group.description}}{{^group.description}}Default description{{/group.description}} + displayName: {{group.displayName}} + created: {{&auditStamp}} + members: [] # required as part of the aspect's schema definition + groups: [] # required as part of the aspect's schema definition + admins: [] # required as part of the aspect's schema definition +- entityUrn: urn:li:corpGroup:{{group.id}} + entityType: corpGroup + aspectName: origin + changeType: UPSERT + aspect: + type: NATIVE +``` + +Creating an entry in the `bootstrap_mcps.yaml` to populate the values from the environment variable `DATAHUB_TEST_GROUP_VALUES` + +```yaml +- name: test-group + version: v1 + mcps_location: "bootstrap_mcps/test-group.yaml" + values_env: "DATAHUB_TEST_GROUP_VALUES" +``` + +An example json values are loaded from environment variable in `DATAHUB_TEST_GROUP_VALUES` might look like the following. + +```json +{ + "group": { + "id": "mygroup", + "displayName": "My Group", + "description": "Description of the group" + } +} +``` + +Using standard mustache template semantics the values in the environment would be inserted into the yaml structure +and ingested when the `system-update` runs. + +#### Default values + +In the example above, the group's `description` if not provided would default to `Default description` if not specified +in the values contain in the environment variable override following the standard mustache template semantics. + +#### AuditStamp + +A special template reference, `{{&auditStamp}}` can be used to inject an `auditStamp` into the aspect. This can be used to +populate required fields of type `auditStamp` calculated from when the MCP is applied. This will insert an inline json representation +of the `auditStamp` into the location and avoid escaping html characters per standard mustache template indicated by the `&` character. + +### Ingestion Template MCPs + +Ingestion template MCPs are slightly more complicated since the ingestion `recipe` is stored as a json string within the aspect. +For ingestion recipes, special handling was added so that they can be described naturally in yaml instead of the normally encoded json string. + +This means that in the example below, the structure beneath the `aspect.config.recipe` path will be automatically converted +to the required json structure and stored as a string. + +```yaml +- entityType: dataHubIngestionSource + entityUrn: urn:li:dataHubIngestionSource:demo-data + aspectName: dataHubIngestionSourceInfo + changeType: UPSERT + aspect: + type: "demo-data" + name: "demo-data" + config: + recipe: + source: + type: "datahub-gc" + config: {} + executorId: default +``` + +## `bootstrap_mcps.yaml` Override + +Additionally, the `bootstrap_mcps.yaml` can be overridden. +This might be useful for applying changes to the version when using helm defined template values. + +```yaml +bootstrap: + templates: + - name: myMCPTemplate + version: v1 + mcps_location: + values_env: + revision_env: REVISION_ENV +``` + +In the above example, we've added a `revision_env` which allows overriding the MCP bootstrap definition itself (excluding `revision_env`). + +In this example we could configure `REVISION_ENV` to contain a timestamp or hash: `{"version":"2024060600"}` +This value can be changed/incremented each time the helm supplied template values change. This ensures the MCP is updated +with the latest values during deployment. + +## Known Limitations + +- Supported change types: + - UPSERT + - CREATE + - CREATE_ENTITY diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/browse-paths-upgrade.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/browse-paths-upgrade.md new file mode 100644 index 00000000..82dbba49 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/browse-paths-upgrade.md @@ -0,0 +1,143 @@ +--- +title: Browse Paths Upgrade (August 2022) +slug: /advanced/browse-paths-upgrade +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/browse-paths-upgrade.md +--- +# Browse Paths Upgrade (August 2022) + +## Background + +Up to this point, there's been a historical constraint on all entity browse paths. Namely, each browse path has been +required to end with a path component that represents "simple name" for an entity. For example, a Browse Path for a +Snowflake Table called "test_table" may look something like this: + +``` +/prod/snowflake/warehouse1/db1/test_table +``` + +In the UI, we artificially truncate the final path component when you are browsing the Entity hierarchy, so your browse experience +would be: + +`prod` > `snowflake` > `warehouse1`> `db1` > `Click Entity` + +As you can see, the final path component `test_table` is effectively ignored. It could have any value, and we would still ignore +it in the UI. This behavior serves as a workaround to the historical requirement that all browse paths end with a simple name. + +This data constraint stands in opposition the original intention of Browse Paths: to provide a simple mechanism for organizing +assets into a hierarchical folder structure. For this reason, we've changed the semantics of Browse Paths to better align with the original intention. +Going forward, you will not be required to provide a final component detailing the "name". Instead, you will be able to provide a simpler path that +omits this final component: + +``` +/prod/snowflake/warehouse1/db1 +``` + +and the browse experience from the UI will continue to work as you would expect: + +`prod` > `snowflake` > `warehouse1`> `db1` > `Click Entity`. + +With this change comes a fix to a longstanding bug where multiple browse paths could not be attached to a single URN. Going forward, +we will support producing multiple browse paths for the same entity, and allow you to traverse via multiple paths. For example + +```python +browse_path = BrowsePathsClass( + paths=["/powerbi/my/custom/path", "/my/other/custom/path"] +) +return MetadataChangeProposalWrapper( + entityType="dataset", + changeType="UPSERT", + entityUrn="urn:li:dataset:(urn:li:dataPlatform:custom,MyFileName,PROD), + aspectName="browsePaths", + aspect=browse_path, +) +``` + +_Using the Python Emitter SDK to produce multiple Browse Paths for the same entity_ + +We've received multiple bug reports, such as [this issue](https://github.com/datahub-project/datahub/issues/5525), and requests to address these issues with Browse, and thus are deciding +to do it now before more workarounds are created. + +## What this means for you + +Once you upgrade to DataHub `v0.8.45` you will immediately notice that traversing your Browse Path hierarchy will require +one extra click to find the entity. This is because we are correctly displaying the FULL browse path, including the simple name mentioned above. + +There will be 2 ways to upgrade to the new browse path format. Depending on your ingestion sources, you may want to use one or both: + +1. Migrate default browse paths to the new format by restarting DataHub +2. Upgrade your version of the `datahub` CLI to push new browse path format (version `v0.8.45`) + +Each step will be discussed in detail below. + +### 1. Migrating default browse paths to the new format + +To migrate those Browse Paths that are generated by DataHub by default (when no path is provided), simply restart the `datahub-gms` container / pod with a single +additional environment variable: + +``` +UPGRADE_DEFAULT_BROWSE_PATHS_ENABLED=true +``` + +And restart the `datahub-gms` instance. This will cause GMS to perform a boot-time migration of all your existing Browse Paths +to the new format, removing the unnecessarily name component at the very end. + +If the migration is successful, you'll see the following in your GMS logs: + +``` +18:58:17.414 [main] INFO c.l.m.b.s.UpgradeDefaultBrowsePathsStep:60 - Successfully upgraded all browse paths! +``` + +After this one-time migration is complete, you should be able to navigate the Browse hierarchy exactly as you did previously. + +> Note that certain ingestion sources actively produce their own Browse Paths, which overrides the default path +> computed by DataHub. +> +> In these cases, getting the updated Browse Path will require re-running your ingestion process with the updated +> version of the connector. This is discussed in more detail in the next section. + +### 2. Upgrading the `datahub` CLI to push new browse paths + +If you are actively ingesting metadata from one or more of following sources + +1. Sagemaker +2. Looker / LookML +3. Feast +4. Kafka +5. Mode +6. PowerBi +7. Pulsar +8. Tableau +9. Business Glossary + +You will need to upgrade the DataHub CLI to >= `v0.8.45` and re-run metadata ingestion. This will generate the new browse path format +and overwrite the existing paths for entities that were extracted from these sources. + +### If you are producing custom Browse Paths + +If you've decided to produce your own custom Browse Paths to organize your assets (e.g. via the Python Emitter SDK), you'll want to change the code to produce those paths +to truncate the final path component. For example, if you were previously emitting a browse path like this: + +``` +"my/custom/browse/path/suffix" +``` + +You can simply remove the final "suffix" piece: + +``` +"my/custom/browse/path" +``` + +Your users will be able to find the entity by traversing through these folders in the UI: + +`my` > `custom` > `browse`> `path` > `Click Entity`. + +> Note that if you are using the Browse Path Transformer you _will_ be impacted in the same way. It is recommended that you revisit the +> paths that you are producing, and update them to the new format. + +## Support + +The DataHub team will be on standby to assist you in your migration. Please +join [#release-0_8_0](https://datahubspace.slack.com/archives/C0244FHMHJQ) channel and reach out to us if you find +trouble with the upgrade or have feedback on the process. We will work closely to make sure you can continue to operate +DataHub smoothly. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/db-retention.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/db-retention.md new file mode 100644 index 00000000..6072c5da --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/db-retention.md @@ -0,0 +1,87 @@ +--- +title: Configuring Database Retention +slug: /advanced/db-retention +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/db-retention.md +--- +# Configuring Database Retention + +## Goal + +DataHub stores different versions of [metadata aspects](/docs/what/aspect) as they are ingested +using a database (or key-value store). These multiple versions allow us to look at an aspect's historical changes and +rollback to a previous version if incorrect metadata is ingested. However, every stored version takes additional storage +space, while possibly bringing less value to the system. We need to be able to impose a **retention** policy on these +records to keep the size of the DB in check. + +Goal of the retention system is to be able to **configure and enforce retention policies** on documents at each of these +various levels: + +- global +- entity-level +- aspect-level + +## What type of retention policies are supported? + +We support 3 types of retention policies for aspects: + +| Policy | Versions Kept | +| :-----------: | :-----------------------------------: | +| Indefinite | All versions | +| Version-based | Latest _N_ versions | +| Time-based | Versions ingested in last _N_ seconds | + +**Note:** The latest version (version 0) is never deleted. This ensures core functionality of DataHub is not impacted while applying retention. + +## When is the retention policy applied? + +As of now, retention policies are applied in two places: + +1. **System-update (datahub-upgrade)**: The default retention policies are ingested when you run the non-blocking system-update (e.g. `IngestRetentionPolicies` step). The default YAML is bundled in the datahub-upgrade module (`boot/retention.yaml`). If no policy existed before or the existing policy was updated and `entityService.retention.applyOnBootstrap` is true, a batch apply runs to apply the policy (or policies) to **all** records in the database. +2. **Ingest**: On every ingest, if an existing aspect got updated, it applies the retention policy to the urn-aspect pair being ingested. + +We are planning to support a cron-based application of retention in the near future to ensure that the time-based retention is applied correctly. + +## How to configure? + +We have enabled with feature by default. Please set **ENTITY_SERVICE_ENABLE_RETENTION=false** when +creating the datahub-gms container/k8s pod to prevent the retention policies from taking effect. + +When system-update runs (datahub-upgrade), retention policies are loaded as follows: + +1. First, the default bundled **version-based** retention (e.g. keep **20 latest aspects** for all entity-aspect pairs, and tighter policies for specific status aspects). +2. Second, YAML files from the plugin path (`datahub.plugin.retention.path`, e.g. `/etc/datahub/plugins/retention`) are read and merged with the default set. + +For docker, we set docker-compose to mount `${HOME}/.datahub` directory to `/etc/datahub` directory +within the containers, so you can customize the initial set of retention policies by creating +a `${HOME}/.datahub/plugins/retention/retention.yaml` file. + +We will support a standardized way to do this in Kubernetes setup in the near future. + +The format for the YAML file is as follows: + +```yaml +- entity: "*" # denotes that policy will be applied to all entities + aspect: "*" # denotes that policy will be applied to all aspects + config: + retention: + version: + maxVersions: 20 +- entity: "dataset" + aspect: "datasetProperties" + config: + retention: + version: + maxVersions: 20 + time: + maxAgeInSeconds: 2592000 # 30 days +``` + +Note, it searches for the policies corresponding to the entity, aspect pair in the following order: + +1. entity, aspect +2. \*, aspect +3. entity, \* +4. _, _ + +By restarting datahub-gms after creating the plugin yaml file, the new set of retention policies will be applied. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/derived-aspects.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/derived-aspects.md new file mode 100644 index 00000000..e90d91f9 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/derived-aspects.md @@ -0,0 +1,9 @@ +--- +title: Derived Aspects +slug: /advanced/derived-aspects +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/derived-aspects.md +--- +# Derived Aspects + +WIP diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/entity-hierarchy.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/entity-hierarchy.md new file mode 100644 index 00000000..961bf4c4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/entity-hierarchy.md @@ -0,0 +1,9 @@ +--- +title: Entity Hierarchy +slug: /advanced/entity-hierarchy +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/entity-hierarchy.md +--- +# Entity Hierarchy + +WIP diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/field-path-spec-v2.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/field-path-spec-v2.md new file mode 100644 index 00000000..78cb8c02 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/field-path-spec-v2.md @@ -0,0 +1,372 @@ +--- +title: SchemaFieldPath Specification (Version 2) +slug: /advanced/field-path-spec-v2 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/field-path-spec-v2.md +--- +# SchemaFieldPath Specification (Version 2) + +This document outlines the formal specification for the fieldPath member of +the [SchemaField](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl) +model. This specification (version 2) takes into account the unique requirements of supporting a wide variety of nested +types, unions and optional fields and is a substantial improvement over the current implementation (version 1). + +## Requirements + +The `fieldPath` field is currently used by datahub for not just rendering the schema fields in the UI, but also as a +primary identifier of a field in other places such +as [EditableSchemaFieldInfo](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl#L12), +usage stats and data profiles. Therefore, it must satisfy the following requirements. + +- must be unique across all fields within a schema. +- make schema navigation in the UI more intuitive. +- allow for identifying the type of schema the field is part of, such as a `key-schema` or a `value-schema`. +- allow for future-evolution + +## Existing Convention(v1) + +The existing convention is to simply use the field's name as the `fieldPath` for simple fields, and use the `dot` +delimited names for nested fields. This scheme does not satisfy the [requirements](#requirements) stated above. The +following example illustrates where the `uniqueness` requirement is not satisfied. + +### Example: Ambiguous field path + +Consider the following `Avro` schema which is a `union` of two record types `A` and `B`, each having a simple field with +the same name `f` that is of type `string`. The v1 naming scheme cannot differentiate if a `fieldPath=f` is referring to +the record type `A` or `B`. + +``` +[ + { + "type": "record", + "name": "A", + "fields": [{ "name": "f", "type": "string" } ] + }, { + "type": "record", + "name": "B", + "fields": [{ "name": "f", "type": "string" } ] + } +] +``` + +## The FieldPath encoding scheme(v2) + +The syntax for V2 encoding of the `fieldPath` is captured in the following grammar. The `FieldPathSpec` is essentially +the type annotated path of the member, with each token along the path representing one level of nested member, +starting from the most-enclosing type, leading up to the member. In the case of `unions` that have `one-of` semantics, +the corresponding field will be emitted once for each `member` of the union as its `type`, along with one path +corresponding to the `union` itself. + +### Formal Spec: + +``` + := .. // when part of a key-schema + | . // when part of a value schema + := [version=] // [version=2.0] for v2 + := [key=True] // when part of a key schema + := + // this is the type prefixed path field (nested if repeats). + := . // type prefixed path of a field. + := . | + := [type=] + := [type=] + := | union | array | map + := int | float | double | string | fixed | enum +``` + +For the [example above](#example-ambiguous-field-path), this encoding would produce the following 2 unique paths +corresponding to the `A.f` and `B.f` fields. + +```python +unique_v2_field_paths = [ + "[version=2.0].[type=union].[type=A].[type=string].f", + "[version=2.0].[type=union].[type=B].[type=string].f" +] +``` + +NOTE: + +- this encoding always ensures uniqueness within a schema since the full type annotation leading to a field is encoded + in the fieldPath itself. +- processing a fieldPath, such as from UI, gets simplified simply by walking each token along the path from + left-to-right. +- adding PartOfKeySchemaToken allows for identifying if the field is part of key-schema. +- adding VersionToken allows for future evolvability. +- to represent `optional` fields, which sometimes are modeled as `unions` in formats like `Avro`, instead of treating it + as a `union` member, set the `nullable` member of `SchemaField` to `True`. + +## Examples + +### Primitive types + +```python +avro_schema = """ +{ + "type": "string" +} +""" +unique_v2_field_paths = [ + "[version=2.0].[type=string]" +] +``` + +### Records + +**Simple Record** + +```python +avro_schema = """ +{ + "type": "record", + "name": "some.event.E", + "namespace": "some.event.N", + "doc": "this is the event record E" + "fields": [ + { + "name": "a", + "type": "string", + "doc": "this is string field a of E" + }, + { + "name": "b", + "type": "string", + "doc": "this is string field b of E" + } + ] +} +""" + +unique_v2_field_paths = [ + "[version=2.0].[type=E].[type=string].a", + "[version=2.0].[type=E].[type=string].b", +] +``` + +**Nested Record** + +```python +avro_schema = """ +{ + "type": "record", + "name": "SimpleNested", + "namespace": "com.linkedin", + "fields": [{ + "name": "nestedRcd", + "type": { + "type": "record", + "name": "InnerRcd", + "fields": [{ + "name": "aStringField", + "type": "string" + } ] + } + }] +} +""" + +unique_v2_field_paths = [ + "[version=2.0].[key=True].[type=SimpleNested].[type=InnerRcd].nestedRcd", + "[version=2.0].[key=True].[type=SimpleNested].[type=InnerRcd].nestedRcd.[type=string].aStringField", +] +``` + +**Recursive Record** + +```python +avro_schema = """ +{ + "type": "record", + "name": "Recursive", + "namespace": "com.linkedin", + "fields": [{ + "name": "r", + "type": { + "type": "record", + "name": "R", + "fields": [ + { "name" : "anIntegerField", "type" : "int" }, + { "name": "aRecursiveField", "type": "com.linkedin.R"} + ] + } + }] +} +""" + +unique_v2_field_paths = [ + "[version=2.0].[type=Recursive].[type=R].r", + "[version=2.0].[type=Recursive].[type=R].r.[type=int].anIntegerField", + "[version=2.0].[type=Recursive].[type=R].r.[type=R].aRecursiveField" +] +``` + +```python +avro_schema =""" +{ + "type": "record", + "name": "TreeNode", + "fields": [ + { + "name": "value", + "type": "long" + }, + { + "name": "children", + "type": { "type": "array", "items": "TreeNode" } + } + ] +} +""" +unique_v2_field_paths = [ + "[version=2.0].[type=TreeNode].[type=long].value", + "[version=2.0].[type=TreeNode].[type=array].[type=TreeNode].children", +] +``` + +### Unions + +```python +avro_schema = """ +{ + "type": "record", + "name": "ABUnion", + "namespace": "com.linkedin", + "fields": [{ + "name": "a", + "type": [{ + "type": "record", + "name": "A", + "fields": [{ "name": "f", "type": "string" } ] + }, { + "type": "record", + "name": "B", + "fields": [{ "name": "f", "type": "string" } ] + } + ] + }] +} +""" +unique_v2_field_paths: List[str] = [ + "[version=2.0].[key=True].[type=ABUnion].[type=union].a", + "[version=2.0].[key=True].[type=ABUnion].[type=union].[type=A].a", + "[version=2.0].[key=True].[type=ABUnion].[type=union].[type=A].a.[type=string].f", + "[version=2.0].[key=True].[type=ABUnion].[type=union].[type=B].a", + "[version=2.0].[key=True].[type=ABUnion].[type=union].[type=B].a.[type=string].f", +] +``` + +### Arrays + +```python +avro_schema = """ +{ + "type": "record", + "name": "NestedArray", + "namespace": "com.linkedin", + "fields": [{ + "name": "ar", + "type": { + "type": "array", + "items": { + "type": "array", + "items": [ + "null", + { + "type": "record", + "name": "Foo", + "fields": [ { + "name": "a", + "type": "long" + } ] + } + ] + } + } + }] +} +""" +unique_v2_field_paths: List[str] = [ + "[version=2.0].[type=NestedArray].[type=array].[type=array].[type=Foo].ar", + "[version=2.0].[type=NestedArray].[type=array].[type=array].[type=Foo].ar.[type=long].a", +] +``` + +### Maps + +```python +avro_schema = """ +{ + "type": "record", + "name": "R", + "namespace": "some.namespace", + "fields": [ + { + "name": "a_map_of_longs_field", + "type": { + "type": "map", + "values": "long" + } + } + ] +} +""" +unique_v2_field_paths = [ + "[version=2.0].[type=R].[type=map].[type=long].a_map_of_longs_field", +] + + +``` + +### Mixed Complex Type Examples + +```python +# Combines arrays, unions and records. +avro_schema = """ +{ + "type": "record", + "name": "ABFooUnion", + "namespace": "com.linkedin", + "fields": [{ + "name": "a", + "type": [ { + "type": "record", + "name": "A", + "fields": [{ "name": "f", "type": "string" } ] + }, { + "type": "record", + "name": "B", + "fields": [{ "name": "f", "type": "string" } ] + }, { + "type": "array", + "items": { + "type": "array", + "items": [ + "null", + { + "type": "record", + "name": "Foo", + "fields": [{ "name": "f", "type": "long" }] + } + ] + } + }] + }] +} +""" + +unique_v2_field_paths: List[str] = [ + "[version=2.0].[type=ABFooUnion].[type=union].a", + "[version=2.0].[type=ABFooUnion].[type=union].[type=A].a", + "[version=2.0].[type=ABFooUnion].[type=union].[type=A].a.[type=string].f", + "[version=2.0].[type=ABFooUnion].[type=union].[type=B].a", + "[version=2.0].[type=ABFooUnion].[type=union].[type=B].a.[type=string].f", + "[version=2.0].[type=ABFooUnion].[type=union].[type=array].[type=array].[type=Foo].a", + "[version=2.0].[type=ABFooUnion].[type=union].[type=array].[type=array].[type=Foo].a.[type=long].f", +] +``` + +For more examples, see +the [unit-tests for AvroToMceSchemaConverter](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/tests/unit/test_schema_util.py). + +### Backward-compatibility + +While this format is not directly compatible with the v1 format, the v1 equivalent can easily be constructed from the v2 +encoding by stripping away all the v2 tokens enclosed in the square-brackets `[]`. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/high-cardinality.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/high-cardinality.md new file mode 100644 index 00000000..c87c4217 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/high-cardinality.md @@ -0,0 +1,52 @@ +--- +title: High Cardinality Relationships +slug: /advanced/high-cardinality +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/high-cardinality.md +--- +# High Cardinality Relationships + +As explained in [What is a Relationship](../what/relationship.md), the raw metadata for forming relationships is captured directly inside of a [Metadata Aspect](../what/aspect.md). The most natural way to model this is using an array, e.g. a group membership aspect contains an array of user [URNs](../what/urn.md). However, this poses some challenges when the cardinality of the relationship is expected to be large (say, greater than 10,000). The aspect becomes large in size, which leads to slow update and retrieval. It may even exceed the underlying limit of the document store, which is often in the range of a few MBs. Furthermore, sending large messages (> 1MB) over Kafka requires special tuning and is generally discouraged. + +Depending on the type of relationships, there are different strategies for dealing with high cardinality. + +### 1:N Relationships + +When `N` is large, simply store the relationship as a reverse pointer on the `N` side, instead of an `N`-element array on the `1` side. In other words, instead of doing this + +``` +record MemberList { + members: array[UserUrn] +} +``` + +do this + +``` +record Membership { + group: GroupUrn +} +``` + +One drawback with this approach is that batch updating the member list becomes multiple DB operations and non-atomic. If the list is provided by an external metadata provider via [MCEs](../what/mxe.md), this also means that multiple MCEs will be required to update the list, instead of having one giant array in a single MCE. + +### M:N Relationships + +When one side of the relation (`M` or `N`) has low cardinality, you can apply the same trick in [1:N Relationship] by creating the array on the side with low-cardinality. For example, assuming a user can only be part of a small number of groups but each group can have a large number of users, the following model will be more efficient than the reverse. + +``` +record Membership { + groups: array[GroupUrn] +} +``` + +When both `M` and `N` are of high cardinality (e.g. millions of users, each belongs to million of groups), the only way to store such relationships efficiently is by creating a new "Mapping Entity" with a single aspect like this + +``` +record UserGroupMap { + user: UserUrn + group: GroupUrn +} +``` + +This means that the relationship now can only be created & updated at a single source-destination pair granularity. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/jar-extraction-tmpfs-optimization.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/jar-extraction-tmpfs-optimization.md new file mode 100644 index 00000000..da0bfc3d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/jar-extraction-tmpfs-optimization.md @@ -0,0 +1,345 @@ +--- +title: WAR Extraction to tmpfs Optimization +slug: /advanced/jar-extraction-tmpfs-optimization +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/jar-extraction-tmpfs-optimization.md +--- +# WAR Extraction to tmpfs Optimization + +## Overview + +This optimization improves DataHub GMS startup performance by **15-30%** through two key techniques: + +1. **Extracting the WAR to a tmpfs (RAM disk)** - Eliminates nested JAR overhead +2. **Using Spring Boot's classpath.idx for deterministic class loading** - Ensures classes load in consistent order + +## How It Works + +### 1. WAR Extraction to tmpfs + +Instead of running Java from the packaged WAR directly: + +``` +Traditional: java -jar war.war + → Java decompresses nested JARs on first class load + → Slow filesystem I/O + → Filesystem order randomness affects startup +``` + +This optimization extracts to RAM disk first: + +``` +Optimized: Extract WAR → /tmp/gms/extraction (tmpfs) + → Read classes from RAM + → No decompression overhead + → 2-3× faster class loading +``` + +### 2. Deterministic Class Loading with classpath.idx + +Spring Boot 3.2+ includes `BOOT-INF/classpath.idx` - a pre-computed ordered list of all JARs and their dependencies. + +**Without optimization:** + +- Classpath built from filesystem directory listing +- Filesystem order varies between boots (some filesystems randomize) +- May affect class resolution order if there are duplicate classes +- Variable startup times depending on filesystem state + +**With optimization:** + +- Uses Spring's pre-computed classpath.idx ordering +- Same class loading order every boot +- Consistent performance +- Deterministic behavior for reproducible environments + +**How it works:** + +```bash +# classpath.idx format: +- "BOOT-INF/lib/spring-core-6.0.jar" +- "BOOT-INF/lib/spring-context-6.0.jar" +- "BOOT-INF/lib/spring-data-commons-3.0.jar" +... (100+ entries in dependency order) + +# Extracted to absolute paths: +/tmp/gms-work/BOOT-INF/classes (application classes first) +/tmp/gms-work/BOOT-INF/lib/spring-core-6.0.jar +/tmp/gms-work/BOOT-INF/lib/spring-context-6.0.jar +/tmp/gms-work/BOOT-INF/lib/spring-data-commons-3.0.jar +... (all as single colon-separated classpath) +``` + +## Performance Impact + +| Metric | Improvement | +| ------------------- | ---------------------------------------------------- | +| **Startup Time** | 15-30% faster | +| **Class Loading** | 2-3× faster (RAM vs filesystem) | +| **Consistency** | Deterministic ordering, no random variations | +| **Memory Overhead** | +150-300MB temporary (freed after startup completes) | + +### Example Timing + +``` +Without optimization (filesystem WAR): + - WAR decompression: 8-15s + - Class discovery: 5-10s + - Total startup: ~20-30s + +With optimization (tmpfs extraction): + - WAR extraction to RAM: 2-3s + - Class discovery (from classpath.idx): 1-2s + - Total startup: ~15-20s + +Net gain: 5-15 seconds faster +``` + +## Configuration + +### Enable in Helm + +Add to your `values.yaml`: + +```yaml +# Enable WAR extraction to tmpfs for faster startup +extractJarEnabled: true +``` + +Or via command line: + +```bash +helm install datahub ... --set extractJarEnabled=true +``` + +### What Gets Created + +When `extractJarEnabled: true`: + +1. **tmpfs emptyDir volume** (1Gi, Memory-backed) + + - Mounted at `/tmp/gms-work` in the container + - Automatically cleaned up after pod termination + +2. **EXTRACT_JAR_ENABLED environment variable** + + - Tells startup script to use extraction + - Triggers classpath.idx loading + +3. **Startup logging** + - WAR size logged for validation + - Available RAM checked for safety + - Extraction time measured + +## Requirements + +| Requirement | Minimum | Recommended | Notes | +| ----------------------- | ------- | ----------- | ------------------------------------ | +| **Available RAM** | 500MB | 2GB+ | Per pod; extraction is temporary | +| **WAR File Size** | N/A | < 500MB | Exceeding 1Gi tmpfs limit will fail | +| **Spring Boot Version** | 3.2+ | Latest | Requires classpath.idx support | +| **Kubernetes** | 1.20+ | 1.24+ | For reliable emptyDir medium: Memory | + +### Pre-Flight Checks + +The startup script performs these checks: + +```bash +[STARTUP] JAR extraction enabled. WAR size: 250MB, Available RAM: 7200MB +[STARTUP] Extracting WAR to tmpfs: /tmp/gms-work +[STARTUP] Generating deterministic classpath from BOOT-INF/classpath.idx +[STARTUP] WAR extracted in 2843ms +``` + +### Warning Conditions + +⚠️ **WAR Size Warning** (> 1Gi): + +``` +[WARN] WAR size (1200MB) exceeds tmpfs limit (1Gi). Extraction may fail +``` + +**Action:** Increase tmpfs sizeLimit in values.yaml or reduce WAR size + +⚠️ **Low RAM Warning** (< 500MB): + +``` +[WARN] Low available RAM (256MB). Extraction may fail or trigger swap +``` + +**Action:** Allocate more resources to the pod or disable optimization + +## Architecture Details + +### Classpath.idx Processing + +The startup script processes classpath.idx in 4 steps: + +**Step 1: Convert to absolute paths** + +```bash +- "BOOT-INF/lib/spring-core.jar" +↓ +/tmp/gms-work/BOOT-INF/lib/spring-core.jar +``` + +**Step 2: Prepend application classes** + +```bash +/tmp/gms-work/BOOT-INF/classes (application code - loaded first) +/tmp/gms-work/BOOT-INF/lib/... (library JARs - in dependency order) +``` + +**Step 3: Join into single classpath** + +```bash +/tmp/gms-work/BOOT-INF/classes:/tmp/gms-work/BOOT-INF/lib/jar1.jar:/tmp/gms-work/BOOT-INF/lib/jar2.jar:... +``` + +**Step 4: Create Java argfile** + +```bash +# Avoids shell variable size limits (32KB-256KB depending on system) +cat > java.args < threshold) +- [ ] Parallel class loading for multi-core systems +- [ ] Extraction pre-warming in init containers +- [ ] Compression of classpath.idx for smaller WARs + +## References + +- [Spring Boot Executable Archives](https://docs.spring.io/spring-boot/docs/current/reference/html/deployment.html#deployment-install) +- [Spring Boot 3.2+ classpath.idx](https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-3.2-Release-Notes) +- [Java Argfiles](https://docs.oracle.com/javase/9/tools/java.htm#JSWOR-GUID-4856361B-8BFD-4964-AE84-121F5F6CF111) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/mcp-mcl.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/mcp-mcl.md new file mode 100644 index 00000000..80e5bf0f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/mcp-mcl.md @@ -0,0 +1,298 @@ +--- +title: MetadataChangeProposal & MetadataChangeLog Events +slug: /advanced/mcp-mcl +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/mcp-mcl.md +--- +# MetadataChangeProposal & MetadataChangeLog Events + +## Overview & Vision + +As of release v0.8.7, two new important event streams have been introduced: MetadataChangeProposal & MetadataChangeLog. These topics serve as a more generic (and more appropriately named) versions of the classic MetadataChangeEvent and MetadataAuditEvent events, used for a) proposing and b) logging changes to the DataHub Metadata Graph. + +With these events, we move towards a more generic world, in which Metadata models are not strongly-typed parts of the event schemas themselves. This provides flexibility, allowing for the core models comprising the Metadata Graph to be added and changed dynamically, without requiring structural updates to Kafka or REST API schemas used for ingesting and serving Metadata. + +Moreover, we've focused in on the "aspect" as the atomic unit of write in DataHub. MetadataChangeProposal & MetadataChangeLog with carry only a single aspect in their payload, as opposed to the list of aspects carried by today's MCE & MAE. This more accurately reflects the atomicity contract of the metadata model, hopefully lessening confusion about transactional guarantees for multi-aspect writes in addition to making it simpler to tune into the metadata changes a consumer cares about. + +Making these events more generic does not come for free; we give up some in the form of Restli and Kafka-native schema validation and defer this responsibility to DataHub itself, who is the sole enforcer of the graph model contracts. Additionally, we add an extra step to unbundling the actual metadata by requiring a double-deserialization: that of the event / response body itself and another of the nested Metadata aspect. + +To mitigate these downsides, we are committed to providing cross-language client libraries capable of doing the hard work for you. We intend to publish these as strongly-typed artifacts generated from the "default" model set DataHub ships with. This stands in addition to an initiative to introduce an OpenAPI layer in DataHub's backend (gms) which would provide a strongly typed model. + +Ultimately, we intend to realize a state in which the Entities and Aspect schemas can be altered without requiring generated code and without maintaining a single mega-model schema (looking at you, Snapshot.pdl). The intention is that changes to the metadata model become even easier than they are today. + +### Synchronous Ingestion Architecture + +

+ +

+ +### Asynchronous Ingestion Architecture + +

+ +

+ +## Modeling + +A Metadata Change Proposal is defined (in PDL) as follows + +```protobuf +record MetadataChangeProposal { + + /** + * Kafka audit header. See go/kafkaauditheader for more info. + */ + auditHeader: optional KafkaAuditHeader + + /** + * Type of the entity being written to + */ + entityType: string + + /** + * Urn of the entity being written + **/ + entityUrn: optional Urn, + + /** + * Key aspect of the entity being written + */ + entityKeyAspect: optional GenericAspect + + /** + * Type of change being proposed + */ + changeType: ChangeType + + /** + * Aspect of the entity being written to + * Not filling this out implies that the writer wants to affect the entire entity + * Note: This is only valid for CREATE and DELETE operations. + **/ + aspectName: optional string + + aspect: optional GenericAspect + + /** + * A string->string map of custom properties that one might want to attach to an event + **/ + systemMetadata: optional SystemMetadata + + /** + * Headers - intended to mimic http headers + */ + headers: optional map[string, string] +} +``` + +Each proposal is comprised of the following: + +1. entityType + + Refers to the type of the entity e.g. dataset, chart + +2. entityUrn + + Urn of the entity being updated. Note, **exactly one** of entityUrn or entityKeyAspect must be filled out to correctly identify an entity. + +3. entityKeyAspect + + Key aspect of the entity. Instead of having a string URN, we will support identifying entities by their key aspect structs. Note, this is not supported as of now. + +4. changeType + + Type of change you are proposing: one of + + - UPSERT: Insert if not exists, update otherwise + - CREATE: Insert aspect if not exists, fail otherwise + - CREATE_ENTITY: Insert if entity does not exist, fail otherwise + - UPDATE: Update if exists, fail otherwise + - DELETE: Delete + - PATCH: Patch the aspect instead of doing a full replace + + Only UPSERT, CREATE, CREATE_ENTITY, DELETE, PATCH are supported as of now. + +5. aspectName + + Name of the aspect. Must match the name in the "@Aspect" annotation. + +6. aspect + + To support strongly typed aspects, without having to keep track of a union of all existing aspects, we introduced a new object called GenericAspect. + + ```xml + record GenericAspect { + value: bytes + contentType: string + } + ``` + + It contains the type of serialization and the serialized value. Note, currently we only support "application/json" as contentType but will be adding more forms of serialization in the future. Validation of the serialized object happens in GMS against the schema matching the aspectName. + +7. systemMetadata + + Extra metadata about the proposal like run_id or updated timestamp. + +8. headers + + Optional headers which are meant to mimic http headers. These are currently used for implementing conditional write logic. + +GMS processes the proposal and produces the Metadata Change Log, which looks like this. + +```protobuf +record MetadataChangeLog includes MetadataChangeProposal { + + previousAspectValue: optional GenericAspect + + previousSystemMetadata: optional SystemMetadata + +} +``` + +It includes all fields in the proposal, but also has the previous version of the aspect value and system metadata. This allows the MCL processor to know the previous value before deciding to update all indices. + +## Topics + +Following the change in our event models, we introduced 4 new topics. The old topics will get deprecated as we fully migrate to this model. + +1. **MetadataChangeProposal_v1, FailedMetadataChangeProposal_v1** + + Analogous to the MCE topic, proposals that get produced into the MetadataChangeProposal_v1 topic, will get ingested to GMS asynchronously, and any failed ingestion will produce a failed MCP in the FailedMetadataChangeProposal_v1 topic. + +2. **MetadataChangeLog_Versioned_v1** + + Analogous to the MAE topic, MCLs for versioned aspects will get produced into this topic. Since versioned aspects have a source of truth that can be separately backed up, the retention of this topic is short (by default 7 days). Note both this and the next topic are consumed by the same MCL processor. + +3. **MetadataChangeLog_Timeseries_v1** + + Analogous to the MAE topics, MCLs for timeseries aspects will get produced into this topic. Since timeseries aspects do not have a source of truth, but rather gets ingested straight to elasticsearch, we set the retention of this topic to be longer (90 days). You can backup timeseries aspect by replaying this topic. + +## Configuration + +With MetadataChangeProposal and MetadataChangeLog, we will introduce a new mechanism for configuring the association between Metadata Entities & Aspects. Specifically, the Snapshot.pdl model will no longer encode this information by way of [Rest.li](http://rest.li) union. Instead, a more explicit yaml file will provide these links. This file will be leveraged at runtime to construct the in-memory Entity Registry which contains the global Metadata schema along with some additional metadata. + +An example of the configuration file that will be used for MCP & MCL, which defines a "dataset" entity that is associated with to two aspects: "datasetKey" and "datasetProfile". + +``` +# entity-registry.yml + +entities: + - name: dataset + keyAspect: datasetKey + aspects: + - datasetProfile +``` + +## Features + +### Conditional Writes + +Conditional write semantics use extra information contained in the MCP `headers` field to possibly avoid writing new aspects +if the conditions are not met. + +#### If-Version-Match + +Each time an aspect is updated a `version` is incremented to represent the change to the aspect. This `version` is stored and returned +in `SystemMetadata`. + +A writer can provide a header with the expected `version` when initiating the request. If the expected `version` does not +match the actual `version` stored in the database, the write will fail. This prevents overwriting an aspect that has +been modified by another process. + +Note: If the aspect doesn't exist yet, then the `version` is `-1`. A writer can use this `version` to only create +an aspect if it doesn't. Also see _Change Types: [`CREATE`, `CREATE_ENTITY`]_ section below. + +#### If-Modified-Since / If-Unmodified-Since + +A writer may also specify time-based conditions using http header semantics. Similar to version based conditional writes +this method can be used to prevent the write if the target aspect was modified after a reading the aspect. Per the +http specification dates must comply with ISO-8601 standard. + +`If-Unmodified-Since`: +A writer can specify that the aspect must NOT have been modified after a specific time, following [If-Unmodified-Since](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Unmodified-Since) http headers. + +`If-Modified-Since` +A writer can specify that the aspect must have been modified after a specific time, following [If-Modified-Since](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-Modified-Since) http headers. + +#### Change Types: [`CREATE`, `CREATE_ENTITY`] + +Another form of conditional writes which considers the existence of an aspect or entity uses the following Change Types. + +`CREATE` - Create the aspect if it doesn't already exist. + +`CREATE_ENTITY` - Create the aspect if no aspects exist for the entity. + +By default, a validation exception is thrown if the `CREATE`/`CREATE_ENTITY` constraint is violated. If the write operation +should be dropped without considering it an exception, then add the following header: `If-None-Match: *` to the MCP. + +### Synchronous ElasticSearch Updates + +The writes to the elasticsearch are asynchronous by default. A writer can add a custom header +`X-DataHub-Sync-Index-Update` to the MCP `headers` with value set to `true` to enable a synchronous update of +elasticsearch for specific MCPs that may benefit from it. + +## Aspect Size Validation + +Validates aspect sizes to protect against very large aspect sizes being created or consumed. + +**Debugging flags - disabled by default.** See [Environment Variables - Aspect Size Validation](../deploy/environment-vars.md#aspect-size-validation) for configuration details and usage guidance. + +```yaml +datahub: + validation: + aspectSize: + prePatch: + enabled: false # Validates existing aspects from DB before patch application + warnSizeBytes: null # Optional: logs warning at this size without blocking (for observability) + maxSizeBytes: 16000000 # 16MB - same as INGESTION_MAX_SERIALIZED_STRING_LENGTH + oversizedRemediation: IGNORE # IGNORE (skip write, log warning) or DELETE (skip write and delete aspect) + postPatch: + enabled: false # Validates aspects after patch, before DB write + warnSizeBytes: null # Optional: logs warning at this size without blocking (for observability) + maxSizeBytes: 16000000 # 16MB - same as INGESTION_MAX_SERIALIZED_STRING_LENGTH + oversizedRemediation: IGNORE # IGNORE (skip write, log warning) or DELETE (skip write and delete aspect) +``` + +**Size Thresholds:** + +- `warnSizeBytes` (optional): Logs warning when exceeded but allows write to proceed. Useful for observability during gradual adoption. Should be lower than `maxSizeBytes`. If set higher than `maxSizeBytes`, writes are skipped before the warning triggers. +- `maxSizeBytes`: Skips writes when exceeded and applies the configured remediation strategy. + +**Remediation Strategies:** + +- `IGNORE`: Logs warning, skips write, routes MCP to FailedMetadataChangeProposal topic. +- `DELETE`: Logs warning, skips write, routes MCP to FailedMetadataChangeProposal topic, and deletes the aspect. + +## Change Data Capture (CDC) Mode for Generating MCLs + +### Overview + +DataHub supports an optional CDC (Change Data Capture) mode for generating MetadataChangeLogs. In CDC mode, MCLs are generated from database change events captured directly from the metadata storage layer, rather than being produced by GMS after processing MCPs. This mechanism **guarantees that MCLs are generated in the same order as database writes**, providing stronger ordering guarantees for metadata changes. + +### CDC vs Traditional MCL Generation + +**Traditional Mode (Default)**: + +- GMS processes MCPs and writes to the database +- GMS immediately produces MCLs to Kafka after successful database writes +- MCL order matches MCP processing order + +**CDC Mode**: + +- GMS processes MCPs and writes to the database (MCL emission disabled) +- CDC system (Debezium) captures database changes as they occur +- MCE Consumer reads CDC events and generates MCLs via EntityService +- MCL order matches database transaction commit order + +### Architecture + +In CDC mode, the flow is: + +``` +MCP → GMS (write to DB, no MCL) → CDC Source → CDC Topic → MCE Consumer → MCL generation via EntityService → MCL Topics +``` + +For detailed configuration instructions, see: + +- [CDC Configuration Guide](../how/configure-cdc.md) +- [Environment Variables Reference](../deploy/environment-vars.md#change-data-capture-cdc-configuration) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/micrometer-best-practices.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/micrometer-best-practices.md new file mode 100644 index 00000000..4afb4000 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/micrometer-best-practices.md @@ -0,0 +1,271 @@ +--- +title: Micrometer Best Practices Guide +slug: /advanced/micrometer-best-practices +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/micrometer-best-practices.md +--- +# Micrometer Best Practices Guide + +This guide provides practical recommendations for using the Micrometer library with Prometheus in production environments, addressing common pitfalls and unclear documentation areas. + +## Metric Types and When to Use Them + +### Counters + +**Use for:** Values that only increase (requests processed, errors occurred, bytes sent) + +```java +// ✅ Good - Initialize once, reuse everywhere +private final Counter requestCounter = Counter.builder("http_requests_total") + .description("Total HTTP requests") + .register(meterRegistry); + +// Usage +requestCounter.increment(); +requestCounter.increment(5.0); +``` + +**Anti-patterns:** + +- Don't use for values that can decrease +- Don't create new counters for each operation + +### Gauges + +**Use for:** Current state values (active connections, queue size, temperature) + +```java +// ✅ Good - Gauge tied to an object's state +private final AtomicInteger activeConnections = new AtomicInteger(0); + +Gauge.builder("active_connections") + .description("Number of active database connections") + .register(meterRegistry, activeConnections, AtomicInteger::get); + +// ✅ Good - Using a supplier function +Gauge.builder("jvm_memory_used") + .register(meterRegistry, this, obj -> Runtime.getRuntime().totalMemory()); +``` + +### Timers + +**Use for:** Measuring duration and frequency of events + +```java +// ✅ Good - Initialize once +private final Timer requestTimer = Timer.builder("http_request_duration") + .description("HTTP request duration") + .register(meterRegistry); + +// Usage patterns +Timer.Sample sample = Timer.start(meterRegistry); +// ... do work ... +sample.stop(requestTimer); + +// Or using recordCallable +requestTimer.recordCallable(() -> { + // ... timed operation ... + return result; +}); +``` + +## Tags/Labels: The Complete Guide + +### Static Tags (Recommended) + +Initialize metrics with known, finite tag sets at application startup: + +```java +// ✅ Excellent - Finite, predictable tag values +private final Counter httpRequestsCounter = Counter.builder("http_requests_total") + .description("Total HTTP requests") + .tag("method", "GET") + .tag("endpoint", "/api/users") + .register(meterRegistry); + +// ✅ Good - Create a map for multiple static tag combinations +private final Map methodCounters = Map.of( + "GET", createCounter("GET"), + "POST", createCounter("POST"), + "PUT", createCounter("PUT") +); +``` + +### Dynamic Tags (Use with Extreme Caution) + +**The Cardinal Sin: Unbounded Dynamic Tags** + +```java +// ❌ NEVER DO THIS - Creates unlimited metrics +Counter.builder("user_requests") + .tag("user_id", userId) // If userId is unbounded, this will cause memory leaks + .register(meterRegistry) + .increment(); +``` + +**Why this is problematic:** + +- Each unique tag combination creates a new time series in Prometheus +- Prometheus performance degrades with high cardinality + +**Safe Dynamic Tag Patterns:** + +```java +// ✅ Acceptable - Bounded dynamic tags with known finite values +private final Map statusCounters = new ConcurrentHashMap<>(); + +public void recordHttpResponse(String method, int statusCode) { + // Limit to standard HTTP methods and status code ranges + String normalizedMethod = normalizeHttpMethod(method); + String statusClass = statusCode / 100 + "xx"; // 2xx, 4xx, 5xx + + String key = normalizedMethod + "_" + statusClass; + statusCounters.computeIfAbsent(key, k -> + Counter.builder("http_responses_total") + .tag("method", normalizedMethod) + .tag("status_class", statusClass) + .register(meterRegistry) + ).increment(); +} + +private String normalizeHttpMethod(String method) { + return Set.of("GET", "POST", "PUT", "DELETE", "PATCH") + .contains(method.toUpperCase()) ? method.toUpperCase() : "OTHER"; +} +``` + +### Tag Value Guidelines + +- **Keep values short and normalized**: Long, detailed tag values often indicate high cardinality - normalize them into categories +- **Limit cardinality**: Aim for < 1000 unique combinations per metric +- **Use enums when possible**: Ensures finite, known values + +```java +// ✅ Good - Using enums for guaranteed finite values +public enum DatabaseOperation { + SELECT, INSERT, UPDATE, DELETE +} + +Counter.builder("database_operations_total") + .tag("operation", operation.name().toLowerCase()) + .register(meterRegistry); +``` + +## Naming Conventions + +### Metric Names + +Follow Prometheus naming conventions: + +```java +// ✅ Good metric names +"http_requests_total" // Counter: use _total suffix +"http_request_duration_seconds" // Timer/Histogram: include unit +"database_connections_active" // Gauge: describe current state +"jvm_memory_used_bytes" // Include units in name +"cache_hits_total" +"queue_size_current" + +// ❌ Poor metric names +"httpRequests" // Use snake_case, not camelCase +"requests" // Too generic +"duration" // No unit specified +"count" // Not descriptive +``` + +### Tag Names + +```java +// ✅ Good tag names +.tag("method", "GET") // Clear, standard HTTP concept +.tag("status_code", "200") // Standard terminology +.tag("endpoint", "/api/users") // Clear path identifier +.tag("service", "user-api") // Service identification + +// ❌ Poor tag names +.tag("m", "GET") // Too abbreviated +.tag("httpMethod", "GET") // Avoid camelCase +.tag("path_template", "/api/users/{id}") // Underscores in values OK, not in keys +``` + +## Common Anti-Patterns and Solutions + +### 1. Creating Metrics in Hot Paths + +**Why this is bad:** + +- **Warning spam**: Micrometer logs a warning on first duplicate registration, then debug messages thereafter - problematic if debug logging is enabled +- **Performance overhead**: Registration involves mutex locks and map operations on every request +- **Concurrency bugs**: Race conditions in Micrometer's registration code can cause inconsistent metric state +- **Memory leaks**: High cardinality tags create unbounded metric instances that never get cleaned up + +```java +// ❌ Bad - Creates metrics repeatedly in request handling +public void handleRequest(String userId) { + Counter.builder("user_requests") + .tag("user", userId) + .register(meterRegistry) + .increment(); +} + +// ✅ Good - Pre-create metrics outside hot paths +private final Counter requestsCounter = Counter.builder("requests_total") + .description("Total requests processed") + .register(meterRegistry); + +public void handleRequest(String userId) { + requestsCounter.increment(); + + // If you need user-specific metrics, use bounded patterns: + String userTier = getUserTier(userId); // "free", "premium", "enterprise" + getUserTierCounter(userTier).increment(); +} +``` + +### 2. High Cardinality Tags + +```java +// ❌ Bad - Each user creates new time series +.tag("user_id", request.getUserId()) +.tag("session_id", request.getSessionId()) +.tag("request_id", request.getRequestId()) + +// ✅ Good - Bounded, meaningful dimensions +.tag("user_tier", getUserTier(request.getUserId())) // "free", "premium", "enterprise" +.tag("feature", request.getFeature()) // Limited set of features +.tag("region", request.getRegion()) // Limited set of regions +``` + +### 3. Not Using Description + +```java +// ❌ Missing context +Counter.builder("requests").register(meterRegistry); + +// ✅ Good - Clear description for operators +Counter.builder("http_requests_total") + .description("Total number of HTTP requests processed by the application") + .register(meterRegistry); +``` + +## Memory and Performance Considerations + +### Registry Management + +```java +// ✅ Use a single registry instance across your application +@Configuration +public class MicrometerConfig { + + @Bean + @Primary + public MeterRegistry meterRegistry() { + PrometheusMeterRegistry registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + // Configure common tags that apply to all metrics + registry.config().commonTags("application", "datahub", "environment", activeProfile); + return registry; + } +} +``` + +Benefits: simplified configuration, consistent tagging, single endpoint management. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/monitoring.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/monitoring.md new file mode 100644 index 00000000..a633bd04 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/monitoring.md @@ -0,0 +1,1144 @@ +--- +title: Monitoring DataHub +slug: /advanced/monitoring +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/monitoring.md +--- +# Monitoring DataHub + +## Overview + +Monitoring DataHub's system components is essential for maintaining operational excellence, troubleshooting performance issues, +and ensuring system reliability. This comprehensive guide covers how to implement observability in DataHub through tracing and metrics, +and how to extract valuable insights from your running instances. + +## Why Monitor DataHub? + +Effective monitoring enables you to: + +- Identify Performance Bottlenecks: Pinpoint slow queries or API endpoints +- Debug Issues Faster: Trace requests across distributed components to locate failures +- Meet SLAs: Track and alert on key performance indicators + +## Observability Components + +DataHub's observability strategy consists of two complementary approaches: + +1. Metrics Collection + + **Purpose:** Aggregate statistical data about system behavior over time + **Technology:** Transitioning from DropWizard/JMX to Micrometer + + **Current State:** DropWizard metrics exposed via JMX, collected by Prometheus + **Future Direction:** Native Micrometer integration for Spring-based metrics + **Compatibility:** Prometheus-compatible format with support for other metrics backends + + Key Metrics Categories: + + - Performance Metrics: Request latency, throughput, error rates + - Resource Metrics: CPU, memory utilization + - Application Metrics: Cache hit rates, queue depths, processing times + - Business Metrics: Entity counts, ingestion rates, search performance + +2. Distributed Tracing + + **Purpose:** Track individual requests as they flow through multiple services and components + **Technology:** OpenTelemetry-based instrumentation + + - Provides end-to-end visibility of request lifecycles + - Automatically instruments popular libraries (Kafka, JDBC, Elasticsearch) + - Supports multiple backend systems (Jaeger, Zipkin, etc.) + - Enables custom span creation with minimal code changes + + Key Benefits: + + - Visualize request flow across microservices + - Identify latency hotspots + - Understand service dependencies + - Debug complex distributed transactions + +## GraphQL Instrumentation (Micrometer) + +### Overview + +DataHub provides comprehensive instrumentation for its GraphQL API through Micrometer metrics, enabling detailed performance +monitoring and debugging capabilities. The instrumentation system offers flexible configuration options to balance between +observability depth and performance overhead. + +### Why Path-Level GraphQL Instrumentation Matters + +Traditional GraphQL monitoring only tells you "the search query is slow" but not **why**. Without path-level instrumentation, +you're blind to which specific fields are causing performance bottlenecks in complex nested queries. + +### Real-World Example + +Consider this GraphQL query: + +```graphql +query getSearchResults { + search(input: { query: "sales data" }) { + searchResults { + entity { + ... on Dataset { + name + owner { + # Path: /search/searchResults/entity/owner + corpUser { + displayName + } + } + lineage { + # Path: /search/searchResults/entity/lineage + upstreamCount + downstreamCount + upstreamEntities { + urn + name + } + } + schemaMetadata { + # Path: /search/searchResults/entity/schemaMetadata + fields { + fieldPath + description + } + } + } + } + } + } +} +``` + +### What Path-Level Instrumentation Reveals + +With path-level metrics, you discover: + +- `/search/searchResults/entity/owner` - 50ms (fast, well-cached) +- `/search/searchResults/entity/lineage` - 2500ms (SLOW! hitting graph database) +- `/search/searchResults/entity/schemaMetadata` - 150ms (acceptable) + +**Without path metrics**: "Search query takes 3 seconds" +**With path metrics**: "Lineage resolution is the bottleneck" + +### Key Benefits + +#### 1. **Surgical Optimization** + +Instead of guessing, you know exactly which resolver needs optimization. Maybe lineage needs better caching or pagination. + +#### 2. **Smart Query Patterns** + +Identify expensive patterns like: + +```yaml +# These paths consistently slow: +/*/lineage/upstreamEntities/* +/*/siblings/*/platform +# Action: Add field-level caching or lazy loading +``` + +#### 3. **Client-Specific Debugging** + +Different clients request different fields. Path instrumentation shows: + +- Web UI requests are slow (requesting everything) +- API integrations timeout (requesting deep lineage) + +#### 4. **N+1 Query Detection** + +Spot resolver patterns that indicate N+1 problems: + +``` +/users/0/permissions - 10ms +/users/1/permissions - 10ms +/users/2/permissions - 10ms +... (100 more times) +``` + +### Configuration Strategy + +Start targeted to minimize overhead: + +```yaml +# Focus on known slow operations +fieldLevelOperations: "searchAcrossEntities,getDataset" + +# Target expensive resolver paths +fieldLevelPaths: "/**/lineage/**,/**/relationships/**,/**/privileges" +``` + +### Architecture + +The GraphQL instrumentation is implemented through `GraphQLTimingInstrumentation`, which extends GraphQL Java's instrumentation framework. It provides: + +- **Request-level metrics**: Overall query performance and error tracking +- **Field-level metrics**: Detailed timing for individual field resolvers +- **Smart filtering**: Configurable targeting of specific operations or field paths +- **Low overhead**: Minimal performance impact through efficient instrumentation + +### Metrics Collected + +#### Request-Level Metrics + +**Metric: `graphql.request.duration`** + +- **Type**: Timer with percentiles (p50, p95, p99) +- **Tags**: + - `operation`: Operation name (e.g., "getSearchResultsForMultiple") + - `operation.type`: Query, mutation, or subscription + - `success`: true/false based on error presence + - `field.filtering`: Filtering mode applied (DISABLED, ALL_FIELDS, BY_OPERATION, BY_PATH, BY_BOTH) +- **Use Case**: Monitor overall GraphQL performance, identify slow operations + +**Metric: `graphql.request.errors`** + +- **Type**: Counter +- **Tags**: + - `operation`: Operation name + - `operation.type`: Query, mutation, or subscription +- **Use Case**: Track error rates by operation + +#### Field-Level Metrics + +**Metric: `graphql.field.duration`** + +- **Type**: Timer with percentiles (p50, p95, p99) +- **Tags**: + - `parent.type`: GraphQL parent type (e.g., "Dataset", "User") + - `field`: Field name being resolved + - `operation`: Operation name context + - `success`: true/false + - `path`: Field path (optional, controlled by `fieldLevelPathEnabled`) +- **Use Case**: Identify slow field resolvers, optimize data fetching + +**Metric: `graphql.field.errors`** + +- **Type**: Counter +- **Tags**: Same as field duration (minus success tag) +- **Use Case**: Track field-specific error patterns + +**Metric: `graphql.fields.instrumented`** + +- **Type**: Counter +- **Tags**: + - `operation`: Operation name + - `filtering.mode`: Active filtering mode +- **Use Case**: Monitor instrumentation coverage and overhead + +### Configuration Guide + +#### Master Controls + +```yaml +graphQL: + metrics: + # Master switch for all GraphQL metrics + enabled: ${GRAPHQL_METRICS_ENABLED:true} + + # Enable field-level resolver metrics + fieldLevelEnabled: ${GRAPHQL_METRICS_FIELD_LEVEL_ENABLED:false} +``` + +#### Selective Field Instrumentation + +Field-level metrics can add significant overhead for complex queries. DataHub provides multiple strategies to control which fields are instrumented: + +##### 1. **Operation-Based Filtering** + +Target specific GraphQL operations known to be slow or critical: + +```yaml +fieldLevelOperations: "getSearchResultsForMultiple,searchAcrossLineageStructure" +``` + +##### 2. **Path-Based Filtering** + +Use path patterns to instrument specific parts of your schema: + +```yaml +fieldLevelPaths: "/search/results/**,/user/*/permissions,/**/lineage/*" +``` + +**Path Pattern Syntax**: + +- `/user` - Exact match for the user field +- `/user/*` - Direct children of user (e.g., `/user/name`, `/user/email`) +- `/user/**` - User field and all descendants at any depth +- `/*/comments/*` - Comments field under any parent + +##### 3. **Combined Filtering** + +When both operation and path filters are configured, only fields matching BOTH criteria are instrumented: + +```yaml +# Only instrument search results within specific operations +fieldLevelOperations: "searchAcrossEntities" +fieldLevelPaths: "/searchResults/**" +``` + +#### Advanced Options + +```yaml +# Include field paths as metric tags (WARNING: high cardinality risk) +fieldLevelPathEnabled: false + +# Include metrics for trivial property access +trivialDataFetchersEnabled: false +``` + +### Filtering Modes Explained + +The instrumentation automatically determines the most efficient filtering mode: + +1. **DISABLED**: Field-level metrics completely disabled +2. **ALL_FIELDS**: No filtering, all fields instrumented (highest overhead) +3. **BY_OPERATION**: Only instrument fields within specified operations +4. **BY_PATH**: Only instrument fields matching path patterns +5. **BY_BOTH**: Most restrictive - both operation and path must match + +### Performance Considerations + +#### Impact Assessment + +Field-level instrumentation overhead varies by: + +- **Query complexity**: More fields = more overhead +- **Resolver performance**: Fast resolvers have higher relative overhead +- **Filtering effectiveness**: Better targeting = less overhead + +#### Best Practices + +1. **Start Conservative**: Begin with field-level metrics disabled + + ```yaml + fieldLevelEnabled: false + ``` + +2. **Target Known Issues**: Enable selectively for problematic operations + + ```yaml + fieldLevelEnabled: true + fieldLevelOperations: "slowSearchQuery,complexLineageQuery" + ``` + +3. **Use Path Patterns Wisely**: Focus on expensive resolver paths + + ```yaml + fieldLevelPaths: "/search/**,/**/lineage/**" + ``` + +4. **Avoid Path Tags in Production**: High cardinality risk + + ```yaml + fieldLevelPathEnabled: false # Keep this false + ``` + +5. **Monitor Instrumentation Overhead**: Track the `graphql.fields.instrumented` metric + +### Example Configurations + +#### Development Environment (Full Visibility) + +```yaml +graphQL: + metrics: + enabled: true + fieldLevelEnabled: true + fieldLevelOperations: "" # All operations + fieldLevelPathEnabled: true # Include paths for debugging + trivialDataFetchersEnabled: true +``` + +#### Production - Targeted Monitoring + +```yaml +graphQL: + metrics: + enabled: true + fieldLevelEnabled: true + fieldLevelOperations: "getSearchResultsForMultiple,searchAcrossLineage" + fieldLevelPaths: "/search/results/*,/lineage/upstream/**,/lineage/downstream/**" + fieldLevelPathEnabled: false + trivialDataFetchersEnabled: false +``` + +#### Production - Minimal Overhead + +```yaml +graphQL: + metrics: + enabled: true + fieldLevelEnabled: false # Only request-level metrics +``` + +### Debugging Slow Queries + +When investigating GraphQL performance issues: + +1. **Enable request-level metrics first** to identify slow operations +2. **Temporarily enable field-level metrics** for the slow operation: + ```yaml + fieldLevelOperations: "problematicQuery" + ``` +3. **Analyze field duration metrics** to find bottlenecks +4. **Optionally enable path tags** (briefly) for precise identification: + ```yaml + fieldLevelPathEnabled: true # Temporary only! + ``` +5. **Optimize identified resolvers** and disable detailed instrumentation + +### Integration with Monitoring Stack + +The GraphQL metrics integrate seamlessly with DataHub's monitoring infrastructure: + +- **Prometheus**: Metrics exposed at `/actuator/prometheus` +- **Grafana**: Create dashboards showing: + - Request rates and latencies by operation + - Error rates and types + - Field resolver performance heatmaps + - Top slow operations and fields + +Example Prometheus queries: + +```promql +# Average request duration by operation +rate(graphql_request_duration_seconds_sum[5m]) +/ rate(graphql_request_duration_seconds_count[5m]) + +# Field resolver p99 latency +histogram_quantile(0.99, + rate(graphql_field_duration_seconds_bucket[5m]) +) + +# Error rate by operation +rate(graphql_request_errors_total[5m]) +``` + +## Kafka Consumer Instrumentation (Micrometer) + +### Overview + +DataHub provides comprehensive instrumentation for Kafka message consumption through Micrometer metrics, enabling +real-time monitoring of message queue latency and consumer performance. This instrumentation is critical for +maintaining data freshness SLAs and identifying processing bottlenecks across DataHub's event-driven architecture. + +### Why Kafka Queue Time Monitoring Matters + +Traditional Kafka lag monitoring only tells you "we're behind by 10,000 messages" +Without queue time metrics, you can't answer critical questions like "are we meeting our 5-minute data freshness SLA?" +or "which consumer groups are experiencing delays?" + +#### Real-World Impact + +Consider these scenarios: + +Variable Production Rate: + +- Morning: 100 messages/second → 1000 message lag = 10 seconds old +- Evening: 10 messages/second → 1000 message lag = 100 seconds old +- Same lag count, vastly different business impact! + +Burst Traffic Patterns: + +- Bulk ingestion creates 1M message backlog +- Are these messages from the last hour (recoverable) or last 24 hours (SLA breach)? + +Consumer Group Performance: + +- Real-time processors need < 1 minute latency +- Analytics consumers can tolerate 1 hour latency +- Different groups require different monitoring thresholds + +### Architecture + +Kafka queue time instrumentation is implemented across all DataHub consumers: + +- MetadataChangeProposals (MCP) Processor - SQL entity updates + - BatchMetadataChangeProposals (MCP) Processor - Bulk SQL entity updates +- MetadataChangeLog (MCL) Processor & Hooks - Elasticsearch & downstream aspect operations +- DataHubUsageEventsProcessor - Usage analytics events +- PlatformEventProcessor - Platform operations & external consumers + +Each consumer automatically records queue time metrics using the message's embedded timestamp. + +### Metrics Collected + +#### Core Metric + +Metric: `kafka.message.queue.time` + +- Type: Timer with configurable percentiles and SLO buckets +- Unit: Milliseconds +- Tags: + - topic: Kafka topic name (e.g., "MetadataChangeProposal_v1") + - consumer.group: Consumer group ID (e.g., "generic-mce-consumer") +- Use Case: Monitor end-to-end latency from message production to SQL transaction + +#### Statistical Distribution + +The timer automatically tracks: + +- Count: Total messages processed +- Sum: Cumulative queue time +- Max: Highest queue time observed +- Percentiles: p50, p95, p99, p99.9 (configurable) +- SLO Buckets: Percentage of messages meeting latency targets + +#### Configuration Guide + +Default Configuration: + +```yaml +kafka: + consumer: + metrics: + # Percentiles to calculate + percentiles: "0.5,0.95,0.99,0.999" + + # Service Level Objective buckets (seconds) + slo: "300,1800,3600,10800,21600,43200" # 5m,30m,1h,3h,6h,12h + + # Maximum expected queue time + maxExpectedValue: 86400 # 24 hours (seconds) +``` + +#### Key Monitoring Patterns + +SLA Compliance Monitoring: + +```promql +# Percentage of messages processed within 5-minute SLA +sum(rate(kafka_message_queue_time_seconds_bucket{le="300"}[5m])) by (topic) +/ sum(rate(kafka_message_queue_time_seconds_count[5m])) by (topic) * 100 +``` + +Consumer Group Comparison: + +```promql +# P99 queue time by consumer group +histogram_quantile(0.99, + sum by (consumer_group, le) ( + rate(kafka_message_queue_time_seconds_bucket[5m]) + ) +) +``` + +#### Performance Considerations + +Metric Cardinality: + +The instrumentation is designed for low cardinality: + +- Only two tags: `topic` and `consumer.group` +- No partition-level tags (avoiding explosion with high partition counts) +- No message-specific tags + +Overhead Assessment: + +- CPU Impact: Minimal - single timestamp calculation per message +- Memory Impact: ~5KB per topic/consumer-group combination +- Network Impact: Negligible - metrics aggregated before export + +#### Migration from Legacy Metrics + +The new Micrometer-based queue time metrics coexist with the legacy DropWizard `kafkaLag` histogram: + +- Legacy: `kafkaLag` histogram via JMX +- New: `kafka.message.queue.time` timer via Micrometer +- Migration: Both metrics collected during transition period +- Future: Legacy metrics will be deprecated in favor of Micrometer + +The new metrics provide: + +- Better percentile accuracy +- SLO bucket tracking +- Multi-backend support +- Dimensional tagging + +## DataHub Request Hook Latency Instrumentation (Micrometer) + +### Overview + +DataHub provides comprehensive instrumentation for measuring the latency from initial request submission to post-MCL +(Metadata Change Log) hook execution. This metric is crucial for understanding the end-to-end processing time of metadata +changes, including both the time spent in Kafka queues and the time taken to process through the system to the final hooks. + +### Why Hook Latency Monitoring Matters + +Traditional metrics only show individual component performance. Request hook latency provides the complete picture of how long +it takes for a metadata change to be fully processed through DataHub's pipeline: + +- Request Submission: When a metadata change request is initially submitted +- Queue Time: Time spent in Kafka topics waiting to be consumed +- Processing Time: Time for the change to be persisted and processed +- Hook Execution: Final execution of MCL hooks + +This end-to-end view is essential for: + +- Meeting data freshness SLAs +- Identifying bottlenecks in the metadata pipeline +- Understanding the impact of system load on processing times +- Ensuring timely updates to downstream systems + +### Configuration + +Hook latency metrics are configured separately from Kafka consumer metrics to allow fine-tuning based on your specific requirements: + +```yaml +datahub: + metrics: + # Measures the time from request to post-MCL hook execution + hookLatency: + # Percentiles to calculate for latency distribution + percentiles: "0.5,0.95,0.99,0.999" + + # Service Level Objective buckets (seconds) + # These define the latency targets you want to track + slo: "300,1800,3000,10800,21600,43200" # 5m, 30m, 1h, 3h, 6h, 12h + + # Maximum expected latency (seconds) + # Values above this are considered outliers + maxExpectedValue: 86000 # 24 hours +``` + +### Metrics Collected + +#### Core Metric + +Metric: `datahub.request.hook.queue.time` + +- Type: Timer with configurable percentiles and SLO buckets +- Unit: Milliseconds +- Tags: + - `hook`: Name of the MCL hook being executed (e.g., "IngestionSchedulerHook", "SiblingsHook") +- Use Case: Monitor the complete latency from request submission to hook exe + +#### Key Monitoring Patterns + +SLA Compliance by Hook: + +Monitor which hooks are meeting their latency SLAs: + +```promql +# Percentage of requests processed within 5-minute SLA per hook +sum(rate(datahub_request_hook_queue_time_seconds_bucket{le="300"}[5m])) by (hook) +/ sum(rate(datahub_request_hook_queue_time_seconds_count[5m])) by (hook) * 100 +``` + +Hook Performance Comparison: + +Identify which hooks have the highest latency: + +```promql +# P99 latency by hook +histogram_quantile(0.99, + sum by (hook, le) ( + rate(datahub_request_hook_queue_time_seconds_bucket[5m]) + ) +) +``` + +Latency Trends: + +Track how hook latency changes over time: + +```promql +# Average hook latency trend +avg by (hook) ( + rate(datahub_request_hook_queue_time_seconds_sum[5m]) + / rate(datahub_request_hook_queue_time_seconds_count[5m]) +) +``` + +#### Implementation Details + +The hook latency metric leverages the trace ID embedded in the system metadata of each request: + +1. Trace ID Generation: Each request generates a unique trace ID with an embedded timestamp +1. Propagation: The trace ID flows through the entire processing pipeline via system metadata +1. Measurement: When an MCL hook executes, the metric calculates the time difference between the current time and the trace ID timestamp +1. Recording: The latency is recorded as a timer metric with the hook name as a tag + +#### Performance Considerations + +- Overhead: Minimal - only requires trace ID extraction and time calculation per hook execution +- Cardinality: Low - only one tag (hook name) with typically < 20 unique values +- Accuracy: High - measures actual wall-clock time from request to hook execution + +#### Relationship to Kafka Queue Time Metrics + +While Kafka queue time metrics (`kafka.message.queue.time`) measure the time messages spend in Kafka topics, request hook +latency metrics provide the complete picture: + +- Kafka Queue Time: Time from message production to consumption +- Hook Latency: Time from initial request to final hook execution + +Together, these metrics help identify where delays occur: + +- High Kafka queue time but low hook latency: Bottleneck in Kafka consumption +- Low Kafka queue time but high hook latency: Bottleneck in processing or persistence +- Both high: System-wide performance issues + +## Aspect Size Validation Metrics + +Emitted on all aspect writes (REST, GraphQL, MCP) to track sizes and detect oversized aspects. + +**Metrics:** + +- `aspectSizeValidation.prePatch.sizeDistribution` - Size distribution of existing aspects (tags: aspectName, sizeBucket) +- `aspectSizeValidation.postPatch.sizeDistribution` - Size distribution of aspects being written (tags: aspectName, sizeBucket) +- `aspectSizeValidation.prePatch.oversized` - Oversized aspects found in database (tags: aspectName, remediation) +- `aspectSizeValidation.postPatch.oversized` - Oversized aspects rejected during writes (tags: aspectName, remediation) +- `aspectSizeValidation.prePatch.warning` - Aspects approaching limit in database (tags: aspectName) +- `aspectSizeValidation.postPatch.warning` - Aspects approaching limit during writes (tags: aspectName) + +**Configuration:** + +See [Aspect Size Validation](mcp-mcl.md#aspect-size-validation) for details. + +```yaml +datahub: + validation: + aspectSize: + metrics: + sizeBuckets: [1048576, 5242880, 10485760, 15728640] +``` + +Default buckets (1MB, 5MB, 10MB, 15MB) create ranges: 0-1MB, 1MB-5MB, 5MB-10MB, 10MB-15MB, 15MB+ + +## Cache Monitoring (Micrometer) + +### Overview + +Micrometer provides automatic instrumentation for cache implementations, offering deep insights into cache performance +and efficiency. This instrumentation is crucial for DataHub, where caching significantly impacts query performance and system load. + +### Automatic Cache Metrics + +When caches are registered with Micrometer, comprehensive metrics are automatically collected without code changes: + +#### Core Metrics + +- **`cache.size`** (Gauge) - Current number of entries in the cache +- **`cache.gets`** (Counter) - Cache access attempts, tagged with: + - `result=hit` - Successful cache hits + - `result=miss` - Cache misses requiring backend fetch +- **`cache.puts`** (Counter) - Number of entries added to cache +- **`cache.evictions`** (Counter) - Number of entries evicted +- **`cache.eviction.weight`** (Counter) - Total weight of evicted entries (for size-based eviction) + +#### Derived Metrics + +Calculate key performance indicators using Prometheus queries: + +```promql +# Cache hit rate (should be >80% for hot caches) +sum(rate(cache_gets_total{result="hit"}[5m])) by (cache) / +sum(rate(cache_gets_total[5m])) by (cache) + +# Cache miss rate +1 - (cache_hit_rate) + +# Eviction rate (indicates cache pressure) +rate(cache_evictions_total[5m]) +``` + +### DataHub Cache Configuration + +DataHub uses multiple cache layers, each automatically instrumented: + +#### 1. Entity Client Cache + +```yaml +cache.client.entityClient: + enabled: true + maxBytes: 104857600 # 100MB + entityAspectTTLSeconds: + corpuser: + corpUserInfo: 20 # Short TTL for frequently changing data + corpUserKey: 300 # Longer TTL for stable data + structuredProperty: + propertyDefinition: 300 + structuredPropertyKey: 86400 # 1 day for very stable data +``` + +#### 2. Usage Statistics Cache + +```yaml +cache.client.usageClient: + enabled: true + maxBytes: 52428800 # 50MB + defaultTTLSeconds: 86400 # 1 day + # Caches expensive usage calculations +``` + +#### 3. Search & Lineage Cache + +```yaml +cache.search.lineage: + ttlSeconds: 86400 # 1 day +``` + +### Monitoring Best Practices + +#### Key Indicators to Watch + +1. **Hit Rate by Cache Type** + + ```promql + # Alert if hit rate drops below 70% + cache_hit_rate < 0.7 + ``` + +2. **Memory Pressure** + ```promql + # High eviction rate relative to puts + rate(cache_evictions_total[5m]) / rate(cache_puts_total[5m]) > 0.1 + ``` + +## Thread Pool Executor Monitoring (Micrometer) + +### Overview + +Micrometer automatically instruments Java `ThreadPoolExecutor` instances, providing crucial visibility into concurrency +bottlenecks and resource utilization. For DataHub's concurrent operations, this monitoring is essential for maintaining +performance under load. + +### Automatic Executor Metrics + +#### Pool State Metrics + +- **`executor.pool.size`** (Gauge) - Current number of threads in pool +- **`executor.pool.core`** (Gauge) - Core (minimum) pool size +- **`executor.pool.max`** (Gauge) - Maximum allowed pool size +- **`executor.active`** (Gauge) - Threads actively executing tasks + +#### Queue Metrics + +- **`executor.queued`** (Gauge) - Tasks waiting in queue +- **`executor.queue.remaining`** (Gauge) - Available queue capacity + +#### Performance Metrics + +- **`executor.completed`** (Counter) - Total completed tasks +- **`executor.seconds`** (Timer) - Task execution time distribution +- **`executor.rejected`** (Counter) - Tasks rejected due to saturation + +### DataHub Executor Configurations + +#### 1. GraphQL Query Executor + +```yaml +graphQL.concurrency: + separateThreadPool: true + corePoolSize: 20 # Base threads + maxPoolSize: 200 # Scale under load + keepAlive: 60 # Seconds before idle thread removal + # Handles complex GraphQL query resolution +``` + +#### 2. Batch Processing Executors + +```yaml +entityClient.restli: + get: + batchConcurrency: 2 # Parallel batch processors + batchQueueSize: 500 # Task buffer + batchThreadKeepAlive: 60 + ingest: + batchConcurrency: 2 + batchQueueSize: 500 +``` + +#### 3. Search & Analytics Executors + +```yaml +timeseriesAspectService.query: + concurrency: 10 # Parallel query threads + queueSize: 500 # Buffered queries +``` + +### Critical Monitoring Patterns + +#### Saturation Detection + +```promql +# Thread pool utilization (>0.8 indicates pressure) +executor_active / executor_pool_size > 0.8 + +# Queue filling up (>0.7 indicates backpressure) +executor_queued / (executor_queued + executor_queue_remaining) > 0.7 +``` + +#### Rejection & Starvation + +```promql +# Task rejections (should be zero) +rate(executor_rejected_total[1m]) > 0 + +# Thread starvation (all threads busy for extended period) +avg_over_time(executor_active[5m]) >= executor_pool_core +``` + +#### Performance Analysis + +```promql +# Average task execution time +rate(executor_seconds_sum[5m]) / rate(executor_seconds_count[5m]) + +# Task throughput by executor +rate(executor_completed_total[5m]) +``` + +### Tuning Guidelines + +#### Symptoms & Solutions + +| Symptom | Metric Pattern | Solution | +| --------------- | ------------------------ | ------------------------------- | +| High latency | `executor_queued` rising | Increase pool size | +| Rejections | `executor_rejected` > 0 | Increase queue size or pool max | +| Memory pressure | Many idle threads | Reduce `keepAlive` time | +| CPU waste | Low `executor_active` | Reduce core pool size | + +#### Capacity Planning + +1. **Measure baseline**: Monitor under normal load +2. **Stress test**: Identify saturation points +3. **Set alerts**: + - Warning at 70% utilization + - Critical at 90% utilization +4. **Auto-scale**: Consider dynamic pool sizing based on queue depth + +## Distributed Tracing + +Traces let us track the life of a request across multiple components. Each trace is consisted of multiple spans, which +are units of work, containing various context about the work being done as well as time taken to finish the work. By +looking at the trace, we can more easily identify performance bottlenecks. + +We enable tracing by using the [OpenTelemetry java instrumentation library](https://github.com/open-telemetry/opentelemetry-java-instrumentation). +This project provides a Java agent JAR that is attached to java applications. The agent injects bytecode to capture +telemetry from popular libraries. + +Using the agent we are able to + +1. Plug and play different tracing tools based on the user's setup: Jaeger, Zipkin, or other tools +2. Get traces for Kafka, JDBC, and Elasticsearch without any additional code +3. Track traces of any function with a simple `@WithSpan` annotation + +You can enable the agent by setting env variable `ENABLE_OTEL` to `true` for GMS and MAE/MCE consumers. In our +example [docker-compose](https://github.com/datahub-project/datahub/blob/master/docker/monitoring/docker-compose.monitoring.yml), we export metrics to a local Jaeger +instance by setting env variable `OTEL_TRACES_EXPORTER` to `jaeger` +and `OTEL_EXPORTER_JAEGER_ENDPOINT` to `http://jaeger-all-in-one:14250`, but you can easily change this behavior by +setting the correct env variables. Refer to +this [doc](https://github.com/open-telemetry/opentelemetry-java/blob/main/sdk-extensions/autoconfigure/README.md) for +all configs. + +Once the above is set up, you should be able to see a detailed trace as a request is sent to GMS. We added +the `@WithSpan` annotation in various places to make the trace more readable. You should start to see traces in the +tracing collector of choice. Our example [docker-compose](https://github.com/datahub-project/datahub/blob/master/docker/monitoring/docker-compose.monitoring.yml) deploys +an instance of Jaeger with port 16686. The traces should be available at http://localhost:16686. + +### Configuration Note + +We recommend using either `grpc` or `http/protobuf`, configured using `OTEL_EXPORTER_OTLP_PROTOCOL`. Avoid using `http` will not work as expected due to the size of +the generated spans. + +## Micrometer + +DataHub is transitioning to Micrometer as its primary metrics framework, representing a significant upgrade in observability +capabilities. Micrometer is a vendor-neutral application metrics facade that provides a simple, consistent API for the most +popular monitoring systems, allowing you to instrument your JVM-based application code without vendor lock-in. + +### Why Micrometer? + +1. Native Spring Integration + + As DataHub uses Spring Boot, Micrometer provides seamless integration with: + + - Auto-configuration of common metrics + - Built-in metrics for HTTP requests, JVM, caches, and more + - Spring Boot Actuator endpoints for metrics exposure + - Automatic instrumentation of Spring components + +2. Multi-Backend Support + + Unlike the legacy DropWizard approach that primarily targets JMX, Micrometer natively supports: + + - Prometheus (recommended for cloud-native deployments) + - JMX (for backward compatibility) + - StatsD + - CloudWatch + - Datadog + - New Relic + - And many more... + +3. Dimensional Metrics + + Micrometer embraces modern dimensional metrics with **labels/tags**, enabling: + + - Rich querying and aggregation capabilities + - Better cardinality control + - More flexible dashboards and alerts + - Natural integration with cloud-native monitoring systems + +## Micrometer Transition Plan + +DataHub is undertaking a strategic transition from DropWizard metrics (exposed via JMX) to Micrometer, a modern vendor-neutral metrics facade. +This transition aims to provide better cloud-native monitoring capabilities while maintaining backward compatibility for existing +monitoring infrastructure. + +### Current State + +What We Have Now: + +- Primary System: DropWizard metrics exposed through JMX +- Collection Method: Prometheus-JMX exporter scrapes JMX metrics +- Dashboards: Grafana dashboards consuming JMX-sourced metrics +- Code Pattern: MetricUtils class for creating counters and timers +- Integration: Basic Spring integration with manual metric creation + +

+ +

+ +Limitations: + +- JMX-centric approach limits monitoring backend options +- No unified observability (separate instrumentation for metrics and traces) +- No support for dimensional metrics and tags +- Manual instrumentation required for most components +- Legacy naming conventions without proper tagging + +### Transition State + +What We're Building: + +- Primary System: Micrometer with native Prometheus support +- Collection Method: Direct Prometheus scraping via /actuator/prometheus +- Unified Telemetry: Single instrumentation point for both metrics and traces +- Modern Patterns: Dimensional metrics with rich tagging +- Multi-Backend: Support for Prometheus, StatsD, CloudWatch, Datadog, etc. +- Auto-Instrumentation: Automatic metrics for Spring components + +

+ +

+ +Key Decisions and Rationale: + +1. Dual Registry Approach + + **Decision:** Run both systems in parallel with tag-based routing + + **Rationale:** + + - Zero downtime or disruption + - Gradual migration at component level + - Easy rollback if issues arise + +2. Prometheus as Primary Target + + **Decision:** Focus on Prometheus for new metrics + + **Rationale:** + + - Industry standard for cloud-native applications + - Rich query language and ecosystem + - Better suited for dimensional metrics + +3. Observation API Adoption + + **Decision:** Promote Observation API for new instrumentation + + **Rationale:** + + - Single instrumentation for metrics + traces + - Reduced code complexity + - Consistent naming across telemetry types + +### Future State + +

+ +

+ +Once fully adopted, Micrometer will transform DataHub's observability from a collection of separate tools into a unified platform. +This means developers can focus on building features while getting comprehensive telemetry "for free." + +Intelligent and Adaptive Monitoring + +- Dynamic Instrumentation: Enable detailed metrics for specific entities or operations on-demand without code changes +- Environment-Aware Metrics: Automatically route metrics to Prometheus in Kubernetes, CloudWatch in AWS, or Azure Monitor in Azure +- Built-in SLO Tracking: Define Service Level Objectives declaratively and automatically track error budgets + +Developer and Operator Experience + +- Adding @Observed to a method automatically generates latency percentiles, error rates, and distributed trace spans +- Every service exposes golden signals (latency, traffic, errors, saturation) out-of-the-box +- Business metrics (entity ingestion rates, search performance) seamlessly correlate with system metrics +- Self-documenting telemetry where metrics, traces, and logs tell a coherent operational story + +## DropWizard & JMX + +We originally decided to use [Dropwizard Metrics](https://metrics.dropwizard.io/4.2.0/) to export custom metrics to JMX, +and then use [Prometheus-JMX exporter](https://github.com/prometheus/jmx_exporter) to export all JMX metrics to +Prometheus. This allows our code base to be independent of the metrics collection tool, making it easy for people to use +their tool of choice. You can enable the agent by setting env variable `ENABLE_PROMETHEUS` to `true` for GMS and MAE/MCE +consumers. Refer to this example [docker-compose](https://github.com/datahub-project/datahub/blob/master/docker/monitoring/docker-compose.monitoring.yml) for setting the +variables. + +In our example [docker-compose](https://github.com/datahub-project/datahub/blob/master/docker/monitoring/docker-compose.monitoring.yml), we have configured prometheus to +scrape from 4318 ports of each container used by the JMX exporter to export metrics. We also configured grafana to +listen to prometheus and create useful dashboards. By default, we provide two +dashboards: [JVM dashboard](https://grafana.com/grafana/dashboards/14845) and DataHub dashboard. + +In the JVM dashboard, you can find detailed charts based on JVM metrics like CPU/memory/disk usage. In the DataHub +dashboard, you can find charts to monitor each endpoint and the kafka topics. Using the example implementation, go +to http://localhost:3001 to find the grafana dashboards! (Username: admin, PW: admin) + +To make it easy to track various metrics within the code base, we created MetricUtils class. This util class creates a +central metric registry, sets up the JMX reporter, and provides convenient functions for setting up counters and timers. +You can run the following to create a counter and increment. + +```java +metricUtils.counter(this.getClass(),"metricName").increment(); +``` + +You can run the following to time a block of code. + +```java +try(Timer.Context ignored=metricUtils.timer(this.getClass(),"timerName").timer()){ + ...block of code + } +``` + +#### Enable monitoring through docker-compose + +We provide some example configuration for enabling monitoring in +this [directory](https://github.com/datahub-project/datahub/tree/master/docker/monitoring). Take a look at the docker-compose +files, which adds necessary env variables to existing containers, and spawns new containers (Jaeger, Prometheus, +Grafana). + +You can add in the above docker-compose using the `-f <>` when running docker-compose commands. +For instance, + +```shell +docker-compose \ + -f quickstart/docker-compose.quickstart.yml \ + -f monitoring/docker-compose.monitoring.yml \ + pull && \ +docker-compose -p datahub \ + -f quickstart/docker-compose.quickstart.yml \ + -f monitoring/docker-compose.monitoring.yml \ + up +``` + +We set up quickstart.sh, dev.sh, and dev-without-neo4j.sh to add the above docker-compose when MONITORING=true. For +instance `MONITORING=true ./docker/quickstart.sh` will add the correct env variables to start collecting traces and +metrics, and also deploy Jaeger, Prometheus, and Grafana. We will soon support this as a flag during quickstart. + +## Health check endpoint + +For monitoring healthiness of your DataHub service, `/admin` endpoint can be used. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/partial-update.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/partial-update.md new file mode 100644 index 00000000..98dbe489 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/partial-update.md @@ -0,0 +1,9 @@ +--- +title: Supporting Partial Aspect Update +slug: /advanced/partial-update +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/partial-update.md +--- +# Supporting Partial Aspect Update + +WIP diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/patch.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/patch.md new file mode 100644 index 00000000..9eea3c0b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/patch.md @@ -0,0 +1,695 @@ +--- +title: Emitting Patch Updates to DataHub +slug: /advanced/patch +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/advanced/patch.md' +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Emitting Patch Updates to DataHub + +## Why Would You Use Patch + +By default, most of the SDK tutorials and APIs involve applying full upserts at the aspect level, e.g. replacing the aspect entirely. +This means that when you want to change even a single field within an aspect without modifying others, you need to do a read-modify-write to avoid overwriting existing fields. +To support these scenarios, DataHub supports `PATCH` operations to perform targeted changes for individual fields or values within arrays of fields are possible without impacting other existing metadata. + +:::note + +PATCH support is now supported generically via [OpenAPI](../api/openapi/openapi-usage-guide.md#generic-patching). Traditional PATCH support is only available for a selected set of aspects. The complete list of Aspects that are supported are maintained by the `SUPPORTED_TEMPLATES` constant [here](https://github.com/datahub-project/datahub/blob/master/entity-registry/src/main/java/com/linkedin/metadata/aspect/patch/template/AspectTemplateEngine.java#L23). + +::: + +## How To Use Patches + +The Patch builders are available in both Python and Java SDKs: + + + + +The Python Patch builders are entity-oriented and located in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/9588440549f3d99965085e97b214a7dabc181ed2/metadata-ingestion/src/datahub/specific) module and located in the `datahub.specific` module. +Patch builder helper classes exist for + +- [Datasets](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/dataset.py) +- [Charts](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/chart.py) +- [Dashboards](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/dashboard.py) +- [Data Jobs (Tasks)](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/datajob.py) +- [Data Products](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/specific/dataproduct.py) + +And we are gladly accepting contributions for Containers, Data Flows (Pipelines), Tags, Glossary Terms, Domains, and ML Models. + + + + +The Java Patch builders are aspect-oriented and located in the [datahub-client](https://github.com/datahub-project/datahub/tree/master/metadata-integration/java/datahub-client/src/main/java/datahub/client/patch) module under the `datahub.client.patch` namespace. + + + + +### Add & Remove Owners for Dataset + +To add & remove specific owners for a dataset: + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_owner_patch.py +from datahub.emitter.mce_builder import make_dataset_urn, make_group_urn, make_user_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import OwnerClass, OwnershipTypeClass +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn( + platform="snowflake", name="fct_users_created", env="PROD" +) + +# Create Dataset Patch to Add + Remove Owners +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.add_owner( + OwnerClass(make_user_urn("user-to-add-id"), OwnershipTypeClass.TECHNICAL_OWNER) +) +patch_builder.remove_owner(make_group_urn("group-to-remove-id")) +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + + +### Add & Remove Tags for Dataset + +To add & remove specific tags for a dataset: + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_tag_patch.py +from datahub.emitter.mce_builder import make_tag_urn +from datahub.metadata.schema_classes import TagAssociationClass +from datahub.sdk import DataHubClient, DatasetUrn +from datahub.specific.dataset import DatasetPatchBuilder + +client = DataHubClient.from_env() + +# Create the Dataset updater. +patch_builder = DatasetPatchBuilder( + DatasetUrn(platform="snowflake", name="fct_users_created", env="PROD") +) +patch_builder.add_tag(TagAssociationClass(make_tag_urn("tag-to-add-id"))) +patch_builder.remove_tag("urn:li:tag:tag-to-remove-id") + +# Do the update. +client.entities.update(patch_builder) + +``` + +And for a specific schema field within the Dataset: + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_field_add_tag_patch.py +from datahub.emitter.mce_builder import make_dataset_urn, make_tag_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import TagAssociationClass +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn( + platform="snowflake", name="fct_users_created", env="PROD" +) + +# Create Dataset Patch to Add + Remove Tag for 'profile_id' column +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.for_field("profile_id").add_tag( + TagAssociationClass(make_tag_urn("tag-to-add-id")) +) +patch_builder.for_field("profile_id").remove_tag("urn:li:tag:tag-to-remove-id") +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + + +### Add & Remove Glossary Terms for Dataset + +To add & remove specific glossary terms for a dataset: + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_glossary_term_patch.py +from datahub.emitter.mce_builder import make_dataset_urn, make_term_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import GlossaryTermAssociationClass +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn( + platform="snowflake", name="fct_users_created", env="PROD" +) + +# Create Dataset Patch to Add + Remove Term for 'profile_id' column +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.add_term(GlossaryTermAssociationClass(make_term_urn("term-to-add-id"))) +patch_builder.remove_term(make_term_urn("term-to-remove-id")) +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + +And for a specific schema field within the Dataset: + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_field_add_glossary_term_patch.py +from datahub.emitter.mce_builder import make_dataset_urn, make_term_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import GlossaryTermAssociationClass +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn( + platform="snowflake", name="fct_users_created", env="PROD" +) + +# Create Dataset Patch to Add + Remove Term for 'profile_id' column +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.for_field("profile_id").add_term( + GlossaryTermAssociationClass(make_term_urn("term-to-add-id")) +) +patch_builder.for_field("profile_id").remove_term( + "urn:li:glossaryTerm:term-to-remove-id" +) +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + +### Add & Remove Structured Properties for Dataset + +To add & remove structured properties for a dataset: + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_structured_properties_patch.py +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + +# Create Dataset Patch to Add and Remove Structured Properties +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.add_structured_property( + "urn:li:structuredProperty:retentionTimeInDays", 12 +) +patch_builder.remove_structured_property( + "urn:li:structuredProperty:customClassification" +) +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + + +### Add & Remove Upstream Lineage for Dataset + +To add & remove a lineage edge connecting a dataset to it's upstream or input at both the dataset and schema field level: + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_upstream_lineage_patch.py +from datahub.emitter.mce_builder import make_dataset_urn, make_schema_field_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import ( + DatasetLineageTypeClass, + FineGrainedLineageClass, + FineGrainedLineageUpstreamTypeClass, + UpstreamClass, +) +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn( + platform="snowflake", name="fct_users_created", env="PROD" +) +upstream_to_remove_urn = make_dataset_urn( + platform="s3", name="fct_users_old", env="PROD" +) +upstream_to_add_urn = make_dataset_urn(platform="s3", name="fct_users_new", env="PROD") + +# Create Dataset Patch to Add & Remove Upstream Lineage Edges +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.remove_upstream_lineage(upstream_to_remove_urn) +patch_builder.add_upstream_lineage( + UpstreamClass(upstream_to_add_urn, DatasetLineageTypeClass.TRANSFORMED) +) + +# ...And also include schema field lineage +upstream_field_to_add_urn = make_schema_field_urn(upstream_to_add_urn, "profile_id") +downstream_field_to_add_urn = make_schema_field_urn(dataset_urn, "profile_id") + +patch_builder.add_fine_grained_lineage( + FineGrainedLineageClass( + FineGrainedLineageUpstreamTypeClass.FIELD_SET, + FineGrainedLineageUpstreamTypeClass.FIELD_SET, + [upstream_field_to_add_urn], + [downstream_field_to_add_urn], + ) +) + +upstream_field_to_remove_urn = make_schema_field_urn( + upstream_to_remove_urn, "profile_id" +) +downstream_field_to_remove_urn = make_schema_field_urn(dataset_urn, "profile_id") + +patch_builder.remove_fine_grained_lineage( + FineGrainedLineageClass( + FineGrainedLineageUpstreamTypeClass.FIELD_SET, + FineGrainedLineageUpstreamTypeClass.FIELD_SET, + [upstream_field_to_remove_urn], + [downstream_field_to_remove_urn], + ) +) + +patch_mcps = patch_builder.build() + + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + + +### Add & Remove Read-Only Custom Properties for Dataset + +To add & remove specific custom properties for a dataset: + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_remove_custom_properties_patch.py +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + +# Create Dataset Patch to Add + Remove Custom Properties +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.add_custom_property("cluster_name", "datahubproject.acryl.io") +patch_builder.remove_custom_property("retention_time") +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + +```java +# Inlined from /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class DatasetCustomPropertiesAddRemove { + + private DatasetCustomPropertiesAddRemove() {} + + /** + * Applies Add and Remove property operations on an existing custom properties aspect without + * affecting any other properties + * + * @param args + * @throws IOException + * @throws ExecutionException + * @throws InterruptedException + */ + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + MetadataChangeProposal datasetPropertiesProposal = + new DatasetPropertiesPatchBuilder() + .urn(UrnUtils.toDatasetUrn("hive", "fct_users_deleted", "PROD")) + .addCustomProperty("cluster_name", "datahubproject.acryl.io") + .removeCustomProperty("retention_time") + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + try { + Future response = emitter.emit(datasetPropertiesProposal); + + System.out.println(response.get().getResponseContent()); + } catch (Exception e) { + log.error("Failed to emit metadata to DataHub", e); + throw e; + } finally { + emitter.close(); + } + } +} + +``` + + + + +### Add & Remove Data Job Lineage + + + + +```python +# Inlined from /metadata-ingestion/examples/library/datajob_add_lineage_patch.py +from datahub.emitter.mce_builder import ( + make_data_job_urn, + make_dataset_urn, + make_schema_field_urn, +) +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.schema_classes import ( + FineGrainedLineageClass as FineGrainedLineage, + FineGrainedLineageDownstreamTypeClass as FineGrainedLineageDownstreamType, + FineGrainedLineageUpstreamTypeClass as FineGrainedLineageUpstreamType, +) +from datahub.specific.datajob import DataJobPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create DataJob URN +datajob_urn = make_data_job_urn( + orchestrator="airflow", flow_id="dag_abc", job_id="task_456" +) + +# Create DataJob Patch to Add Lineage +patch_builder = DataJobPatchBuilder(datajob_urn) +patch_builder.add_input_dataset( + make_dataset_urn(platform="kafka", name="SampleKafkaDataset", env="PROD") +) +patch_builder.add_output_dataset( + make_dataset_urn(platform="hive", name="SampleHiveDataset", env="PROD") +) +patch_builder.add_input_datajob( + make_data_job_urn(orchestrator="airflow", flow_id="dag_abc", job_id="task_123") +) +patch_builder.add_input_dataset_field( + make_schema_field_urn( + parent_urn=make_dataset_urn( + platform="hive", name="fct_users_deleted", env="PROD" + ), + field_path="user_id", + ) +) +patch_builder.add_output_dataset_field( + make_schema_field_urn( + parent_urn=make_dataset_urn( + platform="hive", name="fct_users_created", env="PROD" + ), + field_path="user_id", + ) +) + +# Update column-level lineage through the Data Job +lineage1 = FineGrainedLineage( + upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, + upstreams=[ + make_schema_field_urn(make_dataset_urn("postgres", "raw_data.users"), "user_id") + ], + downstreamType=FineGrainedLineageDownstreamType.FIELD, + downstreams=[ + make_schema_field_urn( + make_dataset_urn("postgres", "analytics.user_metrics"), + "user_id", + ) + ], + transformOperation="IDENTITY", + confidenceScore=1.0, +) +patch_builder.add_fine_grained_lineage(lineage1) +patch_builder.remove_fine_grained_lineage(lineage1) +# Replaces all existing fine-grained lineages +patch_builder.set_fine_grained_lineages([lineage1]) + +patch_mcps = patch_builder.build() + +# Emit DataJob Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + +```java +# Inlined from /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DataJobLineageAdd.java +package io.datahubproject.examples; + +import com.linkedin.common.urn.DataJobUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DataJobInputOutputPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class DataJobLineageAdd { + + private DataJobLineageAdd() {} + + /** + * Adds lineage to an existing DataJob without affecting any lineage + * + * @param args + * @throws IOException + * @throws ExecutionException + * @throws InterruptedException + */ + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + String token = ""; + try (RestEmitter emitter = + RestEmitter.create(b -> b.server("http://localhost:8080").token(token))) { + MetadataChangeProposal dataJobIOPatch = + new DataJobInputOutputPatchBuilder() + .urn( + UrnUtils.getUrn( + "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)")) + .addInputDatasetEdge( + DatasetUrn.createFromString( + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)")) + .addOutputDatasetEdge( + DatasetUrn.createFromString( + "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleHiveDataset,PROD)")) + .addInputDatajobEdge( + DataJobUrn.createFromString( + "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)")) + .addInputDatasetField( + UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD),user_id)")) + .addOutputDatasetField( + UrnUtils.getUrn( + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),user_id)")) + .build(); + + Future response = emitter.emit(dataJobIOPatch); + + System.out.println(response.get().getResponseContent()); + } catch (Exception e) { + log.error("Failed to emit metadata to DataHub", e); + throw new RuntimeException(e); + } + } +} + +``` + + + + +## Advanced: How Patch works + +To understand how patching works, it's important to understand a bit about our [models](../what/aspect.md). Entities are comprised of Aspects +which can be reasoned about as JSON representations of the object models. To be able to patch these we utilize [JsonPatch](https://jsonpatch.com/). The components of a JSON Patch are the path, operation, and value. + +### Path + +The JSON path refers to a value within the schema. This can be a single field or can be an entire object reference depending on what the path is. +For our patches we are primarily targeting single fields or even single array elements within a field. To be able to target array elements by id, we go through a translation process +of the schema to transform arrays into maps. This allows a path to reference a particular array element by key rather than by index, for example a specific tag urn being added to a dataset. + +#### Examples + +A patch path for targeting an upstream dataset: + +`/upstreams/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)` + +Breakdown: + +- `/upstreams` -> References the upstreams field of the UpstreamLineage aspect, this is an array of Upstream objects where the key is the Urn +- `/urn:...` -> The dataset to be targeted by the operation + +A patch path for targeting a fine-grained lineage upstream: + +`/fineGrainedLineages/TRANSFORM/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD),foo)/urn:li:query:queryId/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD),bar)` + +Breakdown: + +- `/fineGrainedLineages` -> References the fineGrainedLineages field on an UpstreamLineage, this is an array of FineGrainedLineage objects keyed on transformOperation, downstream urn, and query urn +- `/TRANSFORM` -> transformOperation, one of the fields determining the key for a fineGrainedLineage +- `/urn:li:schemaField:...` -> The downstream schemaField referenced in this schema, part of the key for a fineGrainedLineage +- `/urn:li:query:...` -> The query urn this relationship was derived from, part of the key for a fineGrainedLineage +- `/urn:li:schemaField:` -> The upstream urn that is being targeted by this patch operation + +This showcases that in some cases the key for objects is simple, in others in can be complex to determine, but for our fully supported use cases we have +SDK support on both the Java and Python side that will generate these patches for you as long as you supply the required method parameters. +Path is generally the most complicated portion of a patch to reason about as it requires intimate knowledge of the schema and models. + +### Operation + +Operation is a limited enum of a few supported types pulled directly from the JSON Patch spec. DataHub only supports `ADD` and `REMOVE` of these options +as the other patch operations do not currently have a use case within our system. + +#### Add + +Add is a bit of a misnomer for the JSON Patch spec, it is not an explicit add but an upsert/replace. If the path specified does not exist, it will be created, +but if the path already exists the value will be replaced. Patch operations apply at a path level so it is possible to do full replaces of arrays or objects in the schema +using adds, but generally the most useful use case for patch is to add elements to arrays without affecting the other elements as full upserts are supported by standard ingestion. + +#### Remove + +Remove operations require the path specified to be present, or an error will be thrown, otherwise they operate as one would expect. The specified path will be removed from the aspect. + +### Value + +Value is the actual information that will be stored at a path. If the path references an object then this will include the JSON key value pairs for that object. + +#### Examples + +An example UpstreamLineage object value: + +```json +{ + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,my-bucket/my-folder/my-file.txt,PROD)", + "type": "TRANSFORMED" +} +``` + +For the previous path example (`/upstreams/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)`), this object would represent the UpstreamLineage object for that path. +This specifies the required fields to properly represent that object. Note: by modifying this path, you could reference a single field within the UpstreamLineage object itself, like so: + +```json +{ + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created_upstream,PROD)/type", + "op": "ADD", + "value": "VIEW" +} +``` + +### Implementation details + +#### Template Classes + +Template classes are the mechanism that maps fields to their corresponding JSON paths. Since DataMaps are not true JSON, first we convert a RecordTemplate to a JSON String, +perform any additional process to map array fields to their keys, apply the patch, and then convert the JSON object back to a RecordTemplate to work with the rest of the application. + +The template classes we currently support can be found in the `entity-registry` module. They are split up by aspect, with the GenericTemplate applying to any non-directly supported aspects. +The GenericTemplate allows for use cases that we have not gotten around to directly support yet, but puts more burden on the user to generate patches correctly. + +The template classes are utilized in `EntityServiceImpl` where a MCP is determined to be either a patch or standard upsert which then routes through to the stored templates registered on the EntityRegistry. +The core logical flow each Template runs through is set up in the `Template` interface, with some more specific logic in the lower level interfaces for constructing/deconstructing array field keys. +Most of the complexity around these classes is knowledge of schema and JSON path traversals. + +##### ArrayMergingTemplate & CompoundKeyTemplate + +`ArrayMergingTemplate` is utilized for any aspect which has array fields and may either be used directly or use `CompoundKeyTemplate`. `ArrayMergingTemplate` is the simpler one that can only be used directly for +single value keys. `CompoundKeyTemplate` allows for support of multi-field keys. For more complex examples like FineGrainedLineage, further logic is needed to construct a key as it is not generalizable to other aspects, see `UpstreamLineageTemplate` for full special case implementation. + +#### PatchBuilders + +There are patch builder SDK classes for constructing patches in both Java and Python. The Java patch builders all extend `AbstractMultiFieldPatchBuilder` which sets up the +base functionality for patch builder subtypes. Each implementation of this abstract class is targeted at a particular aspect and contains specific field based update methods +for the most common use cases. On the Python side patch builders live in the `src/specific/` directory and are organized by entity type. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/pdl-best-practices.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/pdl-best-practices.md new file mode 100644 index 00000000..c9631789 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/pdl-best-practices.md @@ -0,0 +1,9 @@ +--- +title: PDL Best Practices +slug: /advanced/pdl-best-practices +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/pdl-best-practices.md +--- +# PDL Best Practices + +WIP diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/advanced/writing-mcps.md b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/writing-mcps.md new file mode 100644 index 00000000..a7f71e7a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/advanced/writing-mcps.md @@ -0,0 +1,165 @@ +--- +title: Saving MCPs to a File +slug: /advanced/writing-mcps +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/advanced/writing-mcps.md +--- +# Saving MCPs to a File + +## What is an MCP? + +A `MetadataChangeProposal` (MCP) represents an atomic unit of change in the DataHub Metadata Graph. Each MCP carries a single aspect in its payload and is used to propose changes to DataHub's metadata. + +- Represents a single aspect change +- Used for proposing metadata changes to DataHub +- Serves as the basic building block for metadata ingestion + +For more information, please see guides on [DataHub Metadata Events](../what/mxe.md) and [MCPs](mcp-mcl.md). + +## Why Write MCPs as Files? + +MCPs in JSON file format are particularly valuable because they represent the lowest and most granular form of events in DataHub. There are two main use cases for using previously saved MCP files: + +### Testing + +MCPs allow you to easily ingest metadata. You can: + +- Use it for entity ingestion by running a simple command, without a dependency on a ingestion connector: + ```bash + datahub ingest mcps .json + ``` +- Create reproducible test cases for metadata ingestion +- Write and run tests when contributing to DataHub (see DataHub Testing Guide for more details) + +### Debugging + +MCPs are valuable for debugging because they let you: + +- Examine entities in your DataHub instance at a granular level +- Export existing entities to MCP files for analysis +- Verify entity structures and relationships before ingestion + +For example, if you want to understand the structure of entities in your DataHub instance, you can emit them as MCP files and examine their contents in detail. + +## Saving MCPs to a file + +### Exporting from Ingestion Source + +You can export MCPs from an ingestion source (such as BigQuery, Snowflake, etc.) to a file using the `file` sink type in your recipe. This approach is useful when you want to: + +- Save MCPs for later ingestion +- Examine existing entities in the source +- Debug ingestion issues + +To get started, create a recipe file (e.g., `export_mcps.yaml`) specifying your target source and the file `sink` type: + +```yaml +source: + type: bigquery # Replace with your source type + config: ... # Add your source configuration here +sink: + type: "file" + config: + filename: "mcps.json" +``` + +Run the ingestion with the following command: + +```python +datahub ingest -c export_mcps.yaml +``` + +This command will extract all entities from your source and write them to `mcps.json` in MCP format. + +For more details about the `file` sink type, please refer to [Metadata File](../../metadata-ingestion/sink_docs/metadata-file.md) + +### Exporting from DataHub Instance + +You can also export MCPs directly from an existing DataHub instance using a similar recipe approach. This method is particularly useful when you need to: + +- Examine entities already in your DataHub instance +- Create test cases based on real data +- Debug entity relationships + +The process is similar to exporting from an ingestion source, with the only difference being that you'll use `datahub` as the source type. +Create a recipe file (e.g., `export_mcps.yaml`) with this configuration: + +```yaml +source: + type: datahub + config: + # Add your DataHub connection configuration here + server: "http://localhost:8080" + token: "your-access-token" # If authentication is required + +sink: + type: "file" + config: + filename: "mcps.json" +``` + +Run the ingestion: + +```python +datahub ingest -c export_mcps.yaml +``` + +This will extract all entities from your DataHub instance and save them to `mcps.json` in MCP format. + +### Creating MCPs with Python SDK + +You can use the `write_metadata_file` helper to generate MCPs programmatically: + +```python +from datahub.ingestion.sink.file import write_metadata_file +from pathlib import Path +from datahub.metadata.schema_classes import DatasetPropertiesClass +from datahub.emitter.mcp import MetadataChangeProposalWrapper + +records = [ + MetadataChangeProposalWrapper( + entityType="dataset", + entityUrn="urn:li:dataset:(urn:li:dataPlatform:hive,example_dataset,PROD)", + changeType="UPSERT", + aspectName="datasetProperties", + aspect=DatasetPropertiesClass( + description="Example dataset description", + customProperties={"encoding": "utf-8"} + )) + +] +write_metadata_file( + file=Path("mcps.json"), + records=records, +) +``` + +Edit `records` to create the event and entities for your needs. + +Run the Python script to generate your defined MCPs and save them to a file: + +```bash +python .py +``` + +For example, the above script will generate an MCP file with a single dataset entity. + +```json +[ + { + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,example_dataset,PROD)", + "changeType": "UPSERT", + "aspectName": "datasetProperties", + "aspect": { + "json": { + "customProperties": { + "encoding": "utf-8" + }, + "description": "Example dataset description", + "tags": [] + } + } + } +] +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/datahub-apis.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/datahub-apis.md new file mode 100644 index 00000000..050e5317 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/datahub-apis.md @@ -0,0 +1,110 @@ +--- +title: DataHub APIs and SDKs Overview +sidebar_label: APIs and SDKs Overview +slug: /api/datahub-apis +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/datahub-apis.md +--- +# DataHub APIs and SDKs Overview + +DataHub has several APIs to manipulate metadata on the platform. Here's the list of APIs and their pros and cons to help you choose the right one for your use case. + +| API | Definition | Pros | Cons | +| ---------------------------------------------------------- | ---------------------------------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| **[Python SDK](/metadata-ingestion/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | +| **[Java SDK](/metadata-integration/java/as-a-library.md)** | SDK | Highly flexible, Good for bulk execution | Requires an understanding of the metadata change event | +| **[GraphQL API](docs/api/graphql/getting-started.md)** | GraphQL interface | Intuitive; mirrors UI capabilities | Less flexible than SDKs; requires knowledge of GraphQL syntax | +| **[OpenAPI](docs/api/openapi/openapi-usage-guide.md)** | Lower-level API for advanced users | Most powerful and flexible | Can be hard to use for straightforward use cases; no corresponding SDKs, but OpenAPI spec is generated within the product | + +In general, **Python and Java SDKs** are our most recommended tools for extending and customizing the behavior of your DataHub instance, especially for programmatic use cases. + +:::warning +About async usage of APIs - DataHub's asynchronous APIs perform only basic schema validation when receiving MCP requests, similar to direct production to MCP Kafka topics. While requests must conform to the MCP schema to be accepted, actual processing happens later in the pipeline. Any processing failures that occur after the initial acceptance are captured in the Failed MCP topic, but these failures are not immediately surfaced to the API caller since they happen asynchronously. +::: + +## Python and Java SDK + +We offer an SDK for both Python and Java that provide full functionality when it comes to CRUD operations and any complex functionality you may want to build into DataHub. We recommend using the SDKs for most use cases. Here are the examples of how to use the SDKs: + +- Define a lineage between data entities +- Executing bulk operations - e.g. adding tags to multiple datasets +- Creating custom metadata entities + +Learn more about the SDKs: + +- **[Python SDK →](/metadata-ingestion/as-a-library.md)** +- **[Java SDK →](/metadata-integration/java/as-a-library.md)** + +## GraphQL API + +The `graphql` API serves as the primary API used by the DataHub frontend. It is generally assumed that accesses to the GraphQL API are coming in from the frontend so it often comes along with default caching, synchronous operations, and other UI targeted expectations. Care should be taken when used programmatically to fetch and update due to this since operations are intentionally limited in scope. Intended as a higher-level API that simplifies the most common operations. + +The GraphQL API can be useful if you're getting started with DataHub since it's more user-friendly and straightfoward, especially when using GraphiQL. Here are some examples of how to use the GraphQL API: + +- Search for datasets with conditions +- Query for relationships between entities + +Learn more about the GraphQL API: + +- **[GraphQL API →](docs/api/graphql/getting-started.md)** + +## DataHub API Comparison + +DataHub supports several APIs, each with its own unique usage and format. +Here's an overview of what each API can do. + +> Last Updated : Feb 16 2024 + +| Feature | GraphQL | Python SDK | OpenAPI | +| -------------------------------------------------------- | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| Create a Dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md) | ✅ | +| Delete a Dataset (Soft Delete) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | +| Delete a Dataset (Hard Delete) | 🚫 | ✅ [[Guide]](/docs/api/tutorials/datasets.md#delete-dataset) | ✅ | +| Search a Dataset | ✅ [[Guide]](/docs/how/search.md#graphql) | ✅ | ✅ | +| Read a Dataset Deprecation | ✅ | ✅ | ✅ | +| Read Dataset Entities (V2) | ✅ | ✅ | ✅ | +| Create a Tag | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#create-tags) | ✅ | +| Read a Tag | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#read-tags) | ✅ | +| Add Tags to a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-dataset) | ✅ | +| Add Tags to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags-to-a-column-of-a-dataset) | ✅ | +| Remove Tags from a Dataset | ✅ [[Guide]](/docs/api/tutorials/tags.md#remove-tags) | ✅ [[Guide]](/docs/api/tutorials/tags.md#add-tags#remove-tags) | ✅ | +| Create Glossary Terms | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms) | ✅ [[Guide]](/docs/api/tutorials/terms.md#create-terms) | ✅ | +| Read Terms from a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms) | ✅ [[Guide]](/docs/api/tutorials/terms.md#read-terms) | ✅ | +| Add Terms to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-column-of-a-dataset) | ✅ | +| Add Terms to a Dataset | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset) | ✅ [[Guide]](/docs/api/tutorials/terms.md#add-terms-to-a-dataset) | ✅ | +| Create Domains | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain) | ✅ [[Guide]](/docs/api/tutorials/domains.md#create-domain) | ✅ | +| Read Domains | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#read-domains) | ✅ | +| Add Domains to a Dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#add-domains) | ✅ | +| Remove Domains from a Dataset | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains) | ✅ [[Guide]](/docs/api/tutorials/domains.md#remove-domains) | ✅ | +| Create / Upsert Users | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users) | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-users) | ✅ | +| Create / Upsert Group | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group) | ✅ [[Guide]](/docs/api/tutorials/owners.md#upsert-group) | ✅ | +| Read Owners of a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md#read-owners) | ✅ | +| Add Owner to a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md#add-owners#remove-owners) | ✅ | +| Remove Owner from a Dataset | ✅ [[Guide]](/docs/api/tutorials/owners.md#remove-owners) | ✅ [[Guide]](/docs/api/tutorials/owners.md) | ✅ | +| Add Lineage | ✅ [[Guide]](/docs/api/tutorials/lineage.md) | ✅ [[Guide]](/docs/api/tutorials/lineage.md#add-lineage) | ✅ | +| Add Column Level (Fine Grained) Lineage | 🚫 | ✅ [[Guide]](docs/api/tutorials/lineage.md#add-column-level-lineage) | ✅ | +| Add Documentation (Description) to a Column of a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-column) | ✅ | +| Add Documentation (Description) to a Dataset | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ [[Guide]](/docs/api/tutorials/descriptions.md#add-description-on-dataset) | ✅ | +| Add / Remove / Replace Custom Properties on a Dataset | 🚫 | ✅ [[Guide]](/docs/api/tutorials/custom-properties.md) | ✅ | +| Add ML Feature to ML Feature Table | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlfeaturetable) | ✅ | +| Add ML Feature to MLModel | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlfeature-to-mlmodel) | ✅ | +| Add ML Group to MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#add-mlgroup-to-mlfeaturetable) | ✅ | +| Create MLFeature | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeature) | ✅ | +| Create MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable) | ✅ | +| Create MLModel | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodel) | ✅ | +| Create MLModelGroup | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlmodelgroup) | ✅ | +| Create MLPrimaryKey | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlprimarykey) | ✅ | +| Create MLFeatureTable | 🚫 | ✅ [[Guide]](/docs/api/tutorials/ml.md#create-mlfeaturetable) | ✅ | +| Read MLFeature | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeature) | ✅ | +| Read MLFeatureTable | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlfeaturetable) | ✅ | +| Read MLModel | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodel) | ✅ | +| Read MLModelGroup | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlmodelgroup) | ✅ | +| Read MLPrimaryKey | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey) | ✅ [[Guide]](/docs/api/tutorials/ml.md#read-mlprimarykey) | ✅ | +| Create Data Product | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/dataproduct_create.py) | ✅ | +| Create Lineage Between Chart and Dashboard | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_chart_dashboard.py) | ✅ | +| Create Lineage Between Dataset and Chart | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_chart.py) | ✅ | +| Create Lineage Between Dataset and DataJob | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_dataset_job_dataset.py) | ✅ | +| Create Finegrained Lineage as DataJob for Dataset | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_datajob_finegrained.py) | ✅ | +| Create Finegrained Lineage for Dataset | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_dataset_finegrained.py) | ✅ | +| Create DataJob with Dataflow | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_job_dataflow.py) | ✅ | +| Create Programmatic Pipeline | 🚫 | ✅ [[Code]](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/programatic_pipeline.py) | ✅ | diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/getting-started.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/getting-started.md new file mode 100644 index 00000000..076cfded --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/getting-started.md @@ -0,0 +1,171 @@ +--- +title: Getting Started With GraphQL +slug: /api/graphql/getting-started +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/graphql/getting-started.md +--- +# Getting Started With GraphQL + +## Reading an Entity: Queries + +DataHub provides the following `graphql` queries for retrieving entities in your Metadata Graph. + +### Query + +The following `graphql` query retrieves the `urn` and `name` of the `properties` of a specific dataset + +```json +{ + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)") { + urn + properties { + name + } + } +} +``` + +In addition to the URN and properties, you can also fetch other types of metadata for an asset, such as owners, tags, domains, and terms of an entity. +For more information on, please refer to the following links." + +- [Querying for Owners of a Dataset](/docs/api/tutorials/owners.md#read-owners) +- [Querying for Tags of a Dataset](/docs/api/tutorials/tags.md#read-tags) +- [Querying for Domain of a Dataset](/docs/api/tutorials/domains.md#read-domains) +- [Querying for Glossary Terms of a Dataset](/docs/api/tutorials/terms.md#read-terms) +- [Querying for Deprecation of a dataset](/docs/api/tutorials/deprecation.md#read-deprecation) +- [Querying for all DataJobs that belong to a DataFlow](/docs/lineage/airflow.md#get-all-datajobs-associated-with-a-dataflow) + +### Search + +To perform full-text search against an Entity of a particular type, use the search(input: `SearchInput!`) `graphql` Query. +The following `graphql` query searches for datasets that match a specific query term. + +```json +{ + search(input: { type: DATASET, query: "my sql dataset", start: 0, count: 10 }) { + start + count + total + searchResults { + entity { + urn + type + ...on Dataset { + name + } + } + } + } +} +``` + +The `search` field is used to indicate that we want to perform a search. +The `input` argument specifies the search criteria, including the type of entity being searched, the search query term, the start index of the search results, and the count of results to return. + +The `query` term is used to specify the search term. +The search term can be a simple string, or it can be a more complex query using patterns. + +- `*` : Search for all entities. +- `*[string]` : Search for all entities that contain aspects **starting with** the specified \[string\]. +- `[string]*` : Search for all entities that contain aspects **ending with** the specified \[string\]. +- `*[string]*` : Search for all entities that **match** aspects named \[string\]. +- `[string]` : Search for all entities that **contain** the specified \[string\]. + +:::note +Note that by default Elasticsearch only allows pagination through 10,000 entities via the search API. +If you need to paginate through more, you can change the default value for the `index.max_result_window` setting in Elasticsearch, or using the scroll API to read from the index directly. +::: + +## Modifying an Entity: Mutations + +:::note +Mutations which change Entity metadata are subject to [DataHub Access Policies](../../authorization/policies.md). +This means that DataHub's server will check whether the requesting actor is authorized to perform the action. +::: + +:::note +GraphQL's mutations in DataHub are primarily designed to support user interface interactions and should generally be +avoided in programmatic use cases. While mutations are implemented and available through the GraphQL API, +they are not intended for high-throughput scenarios or bulk operations commonly found in data integration workflows. + +For programmatic metadata management, data ingestion, and bulk operations, use the **Python SDK** instead. +The Python SDK is available as part of the `acryl-datahub` package and includes comprehensive examples for common use +cases. For detailed usage instructions, see the [Python SDK documentation](../../../metadata-ingestion/as-a-library.md). +::: + +To update an existing Metadata Entity, simply use the `update(urn: String!, input: EntityUpdateInput!)` GraphQL Query. +For example, to update a Dashboard entity, you can issue the following GraphQL mutation: + +```json +mutation updateDashboard { + updateDashboard( + urn: "urn:li:dashboard:(looker,baz)", + input: { + editableProperties: { + description: "My new desription" + } + } + ) { + urn + } +} +``` + +For more information, please refer to following links. + +- [Adding Tags](/docs/api/tutorials/tags.md#add-tags) +- [Adding Glossary Terms](/docs/api/tutorials/terms.md#add-terms) +- [Adding Domain](/docs/api/tutorials/domains.md#add-domains) +- [Adding Owners](/docs/api/tutorials/owners.md#add-owners) +- [Removing Tags](/docs/api/tutorials/tags.md#remove-tags) +- [Removing Glossary Terms](/docs/api/tutorials/terms.md#remove-terms) +- [Removing Domain](/docs/api/tutorials/domains.md#remove-domains) +- [Removing Owners](/docs/api/tutorials/owners.md#remove-owners) +- [Updating Deprecation](/docs/api/tutorials/deprecation.md#update-deprecation) +- [Editing Description (i.e. Documentation) on Datasets](/docs/api/tutorials/descriptions.md#add-description-on-dataset) +- [Editing Description (i.e. Documentation) on Columns](/docs/api/tutorials/descriptions.md#add-description-on-column) +- [Soft Deleting](/docs/api/tutorials/datasets.md#delete-dataset) + +Please refer to [DataHub API Comparison](/docs/api/datahub-apis.md#datahub-api-comparison) to navigate to the use-case oriented guide. + +## Handling Errors + +In GraphQL, requests that have errors do not always result in a non-200 HTTP response body. Instead, errors will be +present in the response body inside a top-level `errors` field. + +This enables situations in which the client is able to deal gracefully will partial data returned by the application server. +To verify that no error has returned after making a GraphQL request, make sure you check _both_ the `data` and `errors` fields that are returned. + +To catch a GraphQL error, simply check the `errors` field side the GraphQL response. It will contain a message, a path, and a set of extensions +which contain a standard error code. + +```json +{ + "errors": [ + { + "message": "Failed to change ownership for resource urn:li:dataFlow:(airflow,dag_abc,PROD). Expected a corp user urn.", + "locations": [ + { + "line": 1, + "column": 22 + } + ], + "path": ["addOwners"], + "extensions": { + "code": 400, + "type": "BAD_REQUEST", + "classification": "DataFetchingException" + } + } + ] +} +``` + +With the following error codes officially supported: + +| Code | Type | Description | +| ---- | ------------ | --------------------------------------------------------------------------------------------- | +| 400 | BAD_REQUEST | The query or mutation was malformed. | +| 403 | UNAUTHORIZED | The current actor is not authorized to perform the requested action. | +| 404 | NOT_FOUND | The resource is not found. | +| 500 | SERVER_ERROR | An internal error has occurred. Check your server logs or contact your DataHub administrator. | diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-best-practices.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-best-practices.md new file mode 100644 index 00000000..d3d1af8a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-best-practices.md @@ -0,0 +1,1082 @@ +--- +title: GraphQL Best Practices +slug: /api/graphql/graphql-best-practices +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/graphql/graphql-best-practices.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# GraphQL Best Practices + +## Introduction: + +DataHub’s GraphQL API is designed to power the UI. The following guidelines are written with this use-case in mind. + +## General Best Practices + +### Query Optimizations + +> One of GraphQL's biggest advantages over a traditional REST API is its support for **declarative data fetching**. Each component can (and should) query exactly the fields it requires to render, with no superfluous data sent over the network. If instead your root component executes a single, enormous query to obtain data for all of its children, it might query on behalf of components that _aren't even rendered_ given the current state. This can result in a delayed response, and it drastically reduces the likelihood that the query's result can be reused by a **server-side response cache**. [[ref](https://www.apolloGraphQL.com/docs/react/data/operation-best-practices#query-only-the-data-you-need-where-you-need-it)] + +1. Minimize over-fetching by only requesting data needed to be displayed. +2. Limit result counts and use pagination (additionally see section below on `Deep Pagination`). +3. Avoid deeply nested queries and instead break out queries into separate requests for the nested objects. + +### Client-side Caching + +Clients, such as Apollo Client (javascript, python `apollo-client-python`), offer [client-side caching](https://www.apolloGraphQL.com/docs/react/caching/overview) to prevent requests to the service and are able to understand the content of the GraphQL query. This enables more advanced caching vs HTTP response caching. + +### Reuse Pieces of Query Logic with Fragments + +One powerful feature of GraphQL that we recommend you use is [fragments](https://hygraph.com/learn/GraphQL/fragments). Fragments allow you to define pieces of a query that you can reuse across any client-side query that you define. Basically, you can define a set of fields that you want to query, and reuse it in multiple places. + +This technique makes maintaining your GraphQL queries much more doable. For example, if you want to request a new field for an entity type across many queries, you’re able to update it in one place if you’re leveraging fragments. + +## Search Query Best Practices + +### Deep Pagination: search* vs scroll* APIs + +`search*` APIs such as [`searchAcrossEntities`](/docs/GraphQL/queries/#searchacrossentities) are designed for minimal pagination (< ~50). They do not perform well for deep pagination requests. Use the equivalent `scroll*` APIs such as [`scrollAcrossEntities`](/docs/GraphQL/queries/#scrollacrossentities) when expecting the need to paginate deeply into the result set. + +:::note +It is impossible to use `search*` for paginating beyond 10k results. +::: + +:::caution +In order to `scroll*` through the entire result set it is required to use a stable sort order. This means using `_score` as +the first sort order cannot be used. Use the `urn` field as the sort order instead. +::: + +#### Examples + +In the following examples we demonstrate pagination for both `scroll*` and `search*` requests. This particular request is searching for two entities, Datasets and Charts, that +contain `pet` in the entities' name or title. The results will only include the URN for the entities. + + + +Page 1 Request: + +```graphql +{ + scrollAcrossEntities( + input: { + types: [DATASET, CHART] + count: 2 + query: "*" + orFilters: [ + { and: [{ field: "name", condition: CONTAIN, values: ["pet"] }] } + { and: [{ field: "title", condition: CONTAIN, values: ["pet"] }] } + ] + sortInput: { sortCriteria: [{ field: "urn", sortOrder: ASCENDING }] } + } + ) { + nextScrollId + searchResults { + entity { + ... on Dataset { + urn + } + ... on Chart { + urn + } + } + } + } +} +``` + +Page 1 Result: + +```json +{ + "data": { + "scrollAcrossEntities": { + "nextScrollId": "eyJzb3J0IjpbMi4wNzk2ODc2LCJ1cm46bGk6ZGF0YXNldDoodXJuOmxpOmRhdGFQbGF0Zm9ybTpzbm93Zmxha2UsbG9uZ190YWlsX2NvbXBhbmlvbnMuYWRvcHRpb24ucGV0X3Byb2ZpbGVzLFBST0QpIl0sInBpdElkIjpudWxsLCJleHBpcmF0aW9uVGltZSI6MH0=", + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)" + } + }, + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pet_profiles,PROD)" + } + } + ] + } + }, + "extensions": {} +} +``` + +Page 2 Request: + +```graphql +{ + scrollAcrossEntities( + input: { + scrollId: "eyJzb3J0IjpbMi4wNzk2ODc2LCJ1cm46bGk6ZGF0YXNldDoodXJuOmxpOmRhdGFQbGF0Zm9ybTpzbm93Zmxha2UsbG9uZ190YWlsX2NvbXBhbmlvbnMuYWRvcHRpb24ucGV0X3Byb2ZpbGVzLFBST0QpIl0sInBpdElkIjpudWxsLCJleHBpcmF0aW9uVGltZSI6MH0=" + types: [DATASET, CHART] + count: 2 + query: "*" + orFilters: [ + { and: [{ field: "name", condition: CONTAIN, values: ["pet"] }] } + { and: [{ field: "title", condition: CONTAIN, values: ["pet"] }] } + ] + sortInput: { sortCriteria: [{ field: "urn", sortOrder: ASCENDING }] } + } + ) { + nextScrollId + searchResults { + entity { + ... on Dataset { + urn + } + ... on Chart { + urn + } + } + } + } +} +``` + +Page 2 Result: + +```json +{ + "data": { + "scrollAcrossEntities": { + "nextScrollId": "eyJzb3J0IjpbMS43MTg3NSwidXJuOmxpOmRhdGFzZXQ6KHVybjpsaTpkYXRhUGxhdGZvcm06c25vd2ZsYWtlLGxvbmdfdGFpbF9jb21wYW5pb25zLmFkb3B0aW9uLnBldHMsUFJPRCkiXSwicGl0SWQiOm51bGwsImV4cGlyYXRpb25UaW1lIjowfQ==", + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_status_history,PROD)" + } + }, + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pets,PROD)" + } + } + ] + } + }, + "extensions": {} +} +``` + + + + +Page 1 Request: + +```graphql +{ + searchAcrossEntities( + input: { + types: [DATASET, CHART] + count: 2 + start: 0 + query: "*" + orFilters: [ + { and: [{ field: "name", condition: CONTAIN, values: ["pet"] }] } + { and: [{ field: "title", condition: CONTAIN, values: ["pet"] }] } + ] + } + ) { + searchResults { + entity { + ... on Dataset { + urn + } + ... on Chart { + urn + } + } + } + } +} +``` + +Page 1 Response: + +```json +{ + "data": { + "searchAcrossEntities": { + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)" + } + }, + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pet_profiles,PROD)" + } + } + ] + } + }, + "extensions": {} +} +``` + +Page 2 Request: + +```graphql +{ + searchAcrossEntities( + input: { + types: [DATASET, CHART] + count: 2 + start: 2 + query: "*" + orFilters: [ + { and: [{ field: "name", condition: CONTAIN, values: ["pet"] }] } + { and: [{ field: "title", condition: CONTAIN, values: ["pet"] }] } + ] + } + ) { + searchResults { + entity { + ... on Dataset { + urn + } + ... on Chart { + urn + } + } + } + } +} +``` + +Page 2 Response: + +```json +{ + "data": { + "searchAcrossEntities": { + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_status_history,PROD)" + } + }, + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.adoption.pets,PROD)" + } + } + ] + } + }, + "extensions": {} +} +``` + + + + +### SearchFlags: Highlighting and Aggregation + +When performing queries which accept [`searchFlags`](/docs/GraphQL/inputObjects#searchflags) and highlighting and aggregation is not needed, be sure to disable these flags. + +- skipHighlighting: true +- skipAggregates: true + +As a fallback, if only certain fields require highlighting use `customHighlightingFields` to limit highlighting to the specific fields. + + + + +Example for skipping highlighting and aggregates, typically used for scrolling search requests. + +```graphql +{ + scrollAcrossEntities( + input: { + types: [DATASET] + count: 2 + query: "pet" + searchFlags: { skipAggregates: true, skipHighlighting: true } + sortInput: { sortCriteria: [{ field: "urn", sortOrder: ASCENDING }] } + } + ) { + searchResults { + entity { + ... on Dataset { + urn + } + } + matchedFields { + name + value + } + } + facets { + displayName + aggregations { + value + count + } + } + } +} +``` + +Response: + +Note that a few `matchedFields` are still returned by default [`urn`, `customProperties`] + +```json +{ + "data": { + "scrollAcrossEntities": { + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)" + }, + "matchedFields": [ + { + "name": "urn", + "value": "" + }, + { + "name": "customProperties", + "value": "" + } + ] + }, + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.pet_details,PROD)" + }, + "matchedFields": [ + { + "name": "urn", + "value": "" + }, + { + "name": "customProperties", + "value": "" + } + ] + } + ], + "facets": [] + } + }, + "extensions": {} +} +``` + + + + + +Custom highlighting can be used for searchAcrossEntities when only a limited number of fields are useful for highlighting. In this example we specifically request highlighting for `description`. + +```graphql +{ + searchAcrossEntities( + input: { + types: [DATASET] + count: 2 + query: "pet" + searchFlags: { customHighlightingFields: ["description"] } + } + ) { + searchResults { + entity { + ... on Dataset { + urn + } + } + matchedFields { + name + value + } + } + } +} +``` + +Response: + +```json +{ + "data": { + "searchAcrossEntities": { + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,long_tail_companions.analytics.pet_details,PROD)" + }, + "matchedFields": [ + { + "name": "urn", + "value": "" + }, + { + "name": "customProperties", + "value": "" + }, + { + "name": "description", + "value": "Table with all pet-related details" + } + ] + }, + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.pet_details,PROD)" + }, + "matchedFields": [ + { + "name": "urn", + "value": "" + }, + { + "name": "customProperties", + "value": "" + } + ] + } + ] + } + }, + "extensions": {} +} +``` + + + + +### Aggregation + +When aggregation is required with `searchAcrossEntities`, it is possible to set the `count` to 0 to avoid fetching the top search hits, only returning the aggregations. Alternatively [aggregateAcrossEntities](/docs/GraphQL/queries#aggregateacrossentities) provides counts and can provide faster results from server-side caching. + +Request: + +```graphql +{ + searchAcrossEntities( + input: { + types: [DATASET] + count: 0 + query: "pet" + searchFlags: { skipHighlighting: true } + } + ) { + searchResults { + entity { + ... on Dataset { + urn + } + } + matchedFields { + name + value + } + } + facets { + displayName + aggregations { + value + count + } + } + } +} +``` + +Response: + +```json +{ + "data": { + "searchAcrossEntities": { + "searchResults": [], + "facets": [ + { + "displayName": "Container", + "aggregations": [ + { + "value": "urn:li:container:b41c14bc5cb3ccfbb0433c8cbdef2992", + "count": 4 + }, + { + "value": "urn:li:container:701919de0ec93cb338fe9bac0b35403c", + "count": 3 + } + ] + }, + { + "displayName": "Sub Type", + "aggregations": [ + { + "value": "table", + "count": 9 + }, + { + "value": "view", + "count": 6 + }, + { + "value": "explore", + "count": 5 + }, + { + "value": "source", + "count": 4 + }, + { + "value": "incremental", + "count": 1 + } + ] + }, + { + "displayName": "Type", + "aggregations": [ + { + "value": "DATASET", + "count": 24 + } + ] + }, + { + "displayName": "Environment", + "aggregations": [ + { + "value": "PROD", + "count": 24 + } + ] + }, + { + "displayName": "Glossary Term", + "aggregations": [ + { + "value": "urn:li:glossaryTerm:Adoption.DaysInStatus", + "count": 1 + }, + { + "value": "urn:li:glossaryTerm:Ecommerce.HighRisk", + "count": 1 + }, + { + "value": "urn:li:glossaryTerm:Classification.Confidential", + "count": 1 + } + ] + }, + { + "displayName": "Domain", + "aggregations": [ + { + "value": "urn:li:domain:094dc54b-0ebc-40a6-a4cf-e1b75e8b8089", + "count": 6 + }, + { + "value": "urn:li:domain:7d64d0fa-66c3-445c-83db-3a324723daf8", + "count": 2 + } + ] + }, + { + "displayName": "Owned By", + "aggregations": [ + { + "value": "urn:li:corpGroup:Adoption", + "count": 5 + }, + { + "value": "urn:li:corpuser:shannon@longtail.com", + "count": 4 + }, + { + "value": "urn:li:corpuser:admin", + "count": 2 + }, + { + "value": "urn:li:corpGroup:Analytics Engineering", + "count": 2 + }, + { + "value": "urn:li:corpuser:avigdor@longtail.com", + "count": 1 + }, + { + "value": "urn:li:corpuser:prentiss@longtail.com", + "count": 1 + }, + { + "value": "urn:li:corpuser:tasha@longtail.com", + "count": 1 + }, + { + "value": "urn:li:corpuser:ricca@longtail.com", + "count": 1 + }, + { + "value": "urn:li:corpuser:emilee@longtail.com", + "count": 1 + } + ] + }, + { + "displayName": "Platform", + "aggregations": [ + { + "value": "urn:li:dataPlatform:looker", + "count": 8 + }, + { + "value": "urn:li:dataPlatform:dbt", + "count": 7 + }, + { + "value": "urn:li:dataPlatform:snowflake", + "count": 7 + }, + { + "value": "urn:li:dataPlatform:s3", + "count": 1 + }, + { + "value": "urn:li:dataPlatform:mongodb", + "count": 1 + } + ] + }, + { + "displayName": "Tag", + "aggregations": [ + { + "value": "urn:li:tag:prod_model", + "count": 3 + }, + { + "value": "urn:li:tag:pii", + "count": 2 + }, + { + "value": "urn:li:tag:business critical", + "count": 2 + }, + { + "value": "urn:li:tag:business_critical", + "count": 2 + }, + { + "value": "urn:li:tag:Tier1", + "count": 1 + }, + { + "value": "urn:li:tag:prod", + "count": 1 + } + ] + }, + { + "displayName": "Type", + "aggregations": [ + { + "value": "DATASET", + "count": 24 + } + ] + } + ] + } + }, + "extensions": {} +} +``` + +### Limit Search Entity Types + +When querying for specific entities, enumerate only the entity types required using `types` , for example [`DATASET` , `CHART`] + +### Limit Results + +Limit search results based on the amount of information being requested. For example, a minimal number of attributes can fetch 1,000 - 2,000 results in a single page, however as the number of attributes increases (especially nested objects) the `count` should be lowered, 20-25 for very complex requests. + +## Lineage Query Best Practices + +There are two primary ways to query lineage: + +### Search Across Lineage + +`searchAcrossLineage` / `scrollAcrossLineage` root query: + +- Recommended for all lineage queries +- Only the shortest path is guaranteed to show up in `paths` +- Supports querying indirect lineage (depth > 1) + - Depending on the fanout of the lineage, 3+ hops may not return data, use 1-hop queries for the fastest response times. + - Specify using a filter with name `"degree"` and values `"1"` , `"2"`, and / or `"3+"` + +The following examples are demonstrated using sample data for `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. + +

+ +

+ + + + +The following example queries show UPSTREAM lineage with progressively higher degrees, first with degree `["1"]` and then `["1","2"]`. + +1-Hop Upstreams: + +Request: + +```graphql +{ + searchAcrossLineage( + input: { + urn: "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" + query: "*" + count: 10 + start: 0 + direction: UPSTREAM + orFilters: [ + { and: [{ field: "degree", condition: EQUAL, values: ["1"] }] } + ] + searchFlags: { skipAggregates: true, skipHighlighting: true } + } + ) { + start + count + total + searchResults { + entity { + urn + type + ... on Dataset { + name + } + } + paths { + path { + ... on Dataset { + urn + } + } + } + degree + } + } +} +``` + +Response: + +```json +{ + "data": { + "searchAcrossLineage": { + "start": 0, + "count": 10, + "total": 1, + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "type": "DATASET", + "name": "SampleHdfsDataset" + }, + "paths": [ + { + "path": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" + } + ] + } + ], + "degree": 1 + } + ] + } + }, + "extensions": {} +} +``` + + + + +1-Hop & 2-Hop Upstreams: + +Request: + +```graphql +{ + searchAcrossLineage( + input: { + urn: "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" + query: "*" + count: 10 + start: 0 + direction: UPSTREAM + orFilters: [ + { and: [{ field: "degree", condition: EQUAL, values: ["1", "2"] }] } + ] + searchFlags: { skipAggregates: true, skipHighlighting: true } + } + ) { + start + count + total + searchResults { + entity { + urn + type + ... on Dataset { + name + } + } + paths { + path { + ... on Dataset { + urn + } + } + } + degree + } + } +} +``` + +```json +{ + "data": { + "searchAcrossLineage": { + "start": 0, + "count": 10, + "total": 2, + "searchResults": [ + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "type": "DATASET", + "name": "SampleHdfsDataset" + }, + "paths": [ + { + "path": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" + } + ] + } + ], + "degree": 1 + }, + { + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", + "type": "DATASET", + "name": "SampleKafkaDataset" + }, + "paths": [ + { + "path": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)" + } + ] + } + ], + "degree": 2 + } + ] + } + }, + "extensions": {} +} +``` + + + + +### Lineage Subquery + +The previous query requires a root or starting node in the lineage graph. The following request offers a way to request lineage for multiple nodes at once with a few limitations. + +`lineage` query on `EntityWithRelationship` entities: + +- A more direct reflection of the graph index +- 1-hop lineage only +- Multiple URNs +- Should not be requested too many times in a single request. 20 is a tested limit + +The following examples are based on the sample lineage graph shown here: + +

+ +

+ +Example Request: + +```graphql +query getBulkEntityLineageV2( + $urns: [String!]! = [ + "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)" + "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)" + ] +) { + entities(urns: $urns) { + urn + type + ... on DataJob { + jobId + dataFlow { + flowId + } + properties { + name + } + upstream: lineage(input: { direction: UPSTREAM, start: 0, count: 10 }) { + total + relationships { + type + entity { + urn + type + } + } + } + downstream: lineage( + input: { direction: DOWNSTREAM, start: 0, count: 10 } + ) { + total + relationships { + type + entity { + urn + type + } + } + } + } + } +} +``` + +Example Response: + +```json +{ + "data": { + "entities": [ + { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "type": "DATA_JOB", + "jobId": "task_123", + "dataFlow": { + "flowId": "dag_abc" + }, + "properties": { + "name": "User Creations" + }, + "upstream": { + "total": 1, + "relationships": [ + { + "type": "Consumes", + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", + "type": "DATASET" + } + } + ] + }, + "downstream": { + "total": 2, + "relationships": [ + { + "type": "DownstreamOf", + "entity": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)", + "type": "DATA_JOB" + } + }, + { + "type": "Produces", + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "type": "DATASET" + } + } + ] + } + }, + { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_456)", + "type": "DATA_JOB", + "jobId": "task_456", + "dataFlow": { + "flowId": "dag_abc" + }, + "properties": { + "name": "User Deletions" + }, + "upstream": { + "total": 2, + "relationships": [ + { + "type": "DownstreamOf", + "entity": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(airflow,dag_abc,PROD),task_123)", + "type": "DATA_JOB" + } + }, + { + "type": "Consumes", + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)", + "type": "DATASET" + } + } + ] + }, + "downstream": { + "total": 1, + "relationships": [ + { + "type": "Produces", + "entity": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "type": "DATASET" + } + } + ] + } + } + ] + }, + "extensions": {} +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-endpoint-development.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-endpoint-development.md new file mode 100644 index 00000000..495fb4a4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/graphql-endpoint-development.md @@ -0,0 +1,65 @@ +--- +title: Creating a New GraphQL Endpoint in GMS +slug: /api/graphql/graphql-endpoint-development +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/graphql/graphql-endpoint-development.md +--- +# Creating a New GraphQL Endpoint in GMS + +This guide will walk you through how to add a new GraphQL endpoint in GMS. + +> **listOwnershipTypes example:** The `listOwnershipTypes` endpoint will be used as an example. This endpoint was added in [this commit](https://github.com/datahub-project/datahub/commit/ea92b86e6ab4cbb18742fb8db6bc11fae8970cdb#diff-df9c96427d45d7af6d92dd6caa23a349357dbc4bdb915768ab4ce000a4286964) which can be used as reference. + +## GraphQL API changes + +### Adding an endpoint definition + +GraphQL endpoint definitions for GMS are located in the `datahub-graphql-core/src/main/resources/` directory. New endpoints can be added to the relevant file, e.g. `entity.graphql` for entity management endpoints, `search.graphql` for search-related endpoints, etc. Or, for totally new features, new files can be added to this directory. + +> **listOwnershipTypes example:** The endpoint was added in the [`entity.graphql`](https://github.com/datahub-project/datahub/commit/ea92b86e6ab4cbb18742fb8db6bc11fae8970cdb#diff-df9c96427d45d7af6d92dd6caa23a349357dbc4bdb915768ab4ce000a4286964) file since ownership types are being added as an entity. + +#### Query or Mutation? + +Read-only functionality can go in the `Query` section, while mutations go in the `Mutation` section. The definition for new functionality can go in the appropriate section depending on the use case. + +> **listOwnershipTypes example:** The endpoint was added in the `type Query` section because it is read-only functionality. In the same commit, `createOwnershipType`, `updateOwnershipType`, and `deleteOwnershipType` were added in the `type Mutation` section as these are operations that perform writes. + +#### Input and Output Types + +If the new endpoint requires more than a few inputs or outputs, a struct can be created in the same file to collect these fields. + +> **listOwnershipTypes example:** Since this functionality takes and returns quite a few parameters, `input ListOwnershipTypesInput` and `type ListOwnershipTypesResult` were added to represent the input and output structs. In the same PR, no input and output structs were added for `deleteOwnershipType` since the inputs and output are primitive types. + +### Building your changes + +After adding the new endpoint, and new structs if necessary, building the project will generate the Java classes for the new code that can be used in making the server changes. Build the datahub project to make the new symbols available. + +> **listOwnershipTypes example:** The build step will make the new types `ListOwnershipTypesInput` and `ListOwnershipTypesResult` available in a Java IDE. + +## Java Server changes + +We turn now to developing the server-side functionality for the new endpoint. + +### Adding a resolver + +GraphQL queries are handled by `Resolver` classes located in the `datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/` directory. Resolvers are classes that implement the `DataFetcher` interface where `T` is `CompletableFuture`.This interface provides a `get` method that takes in a `DataFetchingEnvironment` and returns a `CompletableFuture` of the endpoint return type. The resolver can contain any services needed to resolve the endpoint, and use them to compute the result. + +> **listOwnershipTypes example:** The [`ListOwnershipTypesResolver`](https://github.com/datahub-project/datahub/commit/ea92b86e6ab4cbb18742fb8db6bc11fae8970cdb#diff-d2ad02d0ec286017d032640cfdb289fbdad554ef5f439355104766fa068513ac) class implements `DataFetcher>` since this is the return type of the endpoint. It contains an `EntityClient` instance variable to handle the ownership type fetching. + +Often the structure of the `Resolver` classes is to call a service to receive a response, then use a method to transform the result from the service into the GraphQL type returned. + +> **listOwnershipTypes example:** The [`ListOwnershipTypesResolver`](https://github.com/datahub-project/datahub/commit/ea92b86e6ab4cbb18742fb8db6bc11fae8970cdb#diff-d2ad02d0ec286017d032640cfdb289fbdad554ef5f439355104766fa068513ac) calls the `search` method in its `EntityClient` to get the ownership types, then calls the defined `mapUnresolvedOwnershipTypes` function to transform the response into a `ListOwnershipTypesResult`. + +Tip: Resolver classes can be tested with unit tests! + +> **listOwnershipTypes example:** The reference commit adds the [`ListOwnershipTypeResolverTest` class](https://github.com/datahub-project/datahub/commit/ea92b86e6ab4cbb18742fb8db6bc11fae8970cdb#diff-9443d70b221e36e9d47bfa9244673d1cd553a92ae496d03622932ad0a4832045). + +### Adding the resolver to the GMS server + +The main GMS server is located in [`GmsGraphQLEngine.java`](https://github.com/datahub-project/datahub/blob/master/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java). To hook up the resolver to handle the endpoint, find the relevant section based on if the new enpoint is a `Query` or a `Mutation` and add the resolver as the `dataFetcher` for the name of the endpoint. + +> **listOwnershipTypes example:** The following line of code is added in [`GmsGraphQLEngine`](https://github.com/datahub-project/datahub/commit/ea92b86e6ab4cbb18742fb8db6bc11fae8970cdb#diff-e04c9c2d80cbfd7aa7e3e0f867248464db0f6497684661132d6ead81ded21856): `.dataFetcher("listOwnershipTypes", new ListOwnershipTypesResolver(this.entityClient))`. This uses the `ListOwnershipTypes` resolver to handle queries for `listOwnershipTypes` endpoint. + +## Testing your change + +In addition to unit tests for your resolver mentioned above, GraphQL functionality in datahub can be tested using the built-in [GraphiQL](https://www.gatsbyjs.com/docs/how-to/querying-data/running-queries-with-graphiql/) endpoint. The endpoint is located at `localhost:8080/api/graphiql` on Quickstart and at the equivalent URL for a production instance. This provides fast debug-ability for querying GraphQL. See [How to Set Up GraphQL](./how-to-set-up-graphql.md#graphql-explorer-graphiql) for more information diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/how-to-set-up-graphql.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/how-to-set-up-graphql.md new file mode 100644 index 00000000..4d64f117 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/how-to-set-up-graphql.md @@ -0,0 +1,118 @@ +--- +title: How To Set Up GraphQL +slug: /api/graphql/how-to-set-up-graphql +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/graphql/how-to-set-up-graphql.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# How To Set Up GraphQL + +## Preparing Local DataHub Deployment + +The first thing you'll need to use the GraphQL API is a deployed instance of DataHub with some metadata ingested. +For more information, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Querying the GraphQL API + +DataHub's GraphQL endpoint is served at the path `/api/graphql`, e.g. `https://my-company.datahub.com/api/graphql`. +There are a few options when it comes to querying the GraphQL endpoint. + +For **Testing**: + +- GraphQL Explorer (GraphiQL) +- CURL +- Postman + +For **Production**: + +- GraphQL [Client SDK](https://graphql.org/code/) for the language of your choice +- Basic HTTP client + +> Notice: The DataHub GraphQL endpoint only supports POST requests at this time. + +### GraphQL Explorer (GraphiQL) + +DataHub provides a browser-based GraphQL Explorer Tool ([GraphiQL](https://github.com/graphql/graphiql)) for live interaction with the GraphQL API. This tool is available at the path `/api/graphiql` (e.g. `https://my-company.datahub.com/api/graphiql`) +This interface allows you to easily craft queries and mutations against real metadata stored in your live DataHub deployment. + +To experiment with GraphiQL before deploying it in your live DataHub deployment, you can access a demo site provided by DataHub at https://demo.datahub.com/api/graphiql. + +For instance, you can create a tag by posting the following query: + +```json +mutation createTag { + createTag(input: + { + name: "Deprecated", + description: "Having this tag means this column or table is deprecated." + }) +} +``` + +For a detailed usage guide, check out [How to use GraphiQL](https://www.gatsbyjs.com/docs/how-to/querying-data/running-queries-with-graphiql/). + +To navigate to `GraphiQL` on the demo site or your local instance, select `GraphiQL` from the user profile drop-down menu as +shown below. + + + +

+ +

+ + +

+ +

+ + + +This link will then display the following interface for exploring GraphQL queries. + +

+ +

+ +### CURL + +CURL is a command-line tool used for transferring data using various protocols including HTTP, HTTPS, and others. +To query the DataHub GraphQL API using CURL, you can send a `POST` request to the `/api/graphql` endpoint with the GraphQL query in the request body. +Here is an example CURL command to create a tag via GraphQL API: + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation createTag { createTag(input: { name: \"Deprecated\", description: \"Having this tag means this column or table is deprecated.\" }) }", "variables":{}}' +``` + +### Postman + +Postman is a popular API client that provides a graphical user interface for sending requests and viewing responses. +Within Postman, you can create a `POST` request and set the request URL to the `/api/graphql` endpoint. +In the request body, select the `GraphQL` option and enter your GraphQL query in the request body. + +

+ +

+ +Please refer to [Querying with GraphQL](https://learning.postman.com/docs/sending-requests/graphql/graphql-overview/) in the Postman documentation for more information. + +### Authentication + Authorization + +In general, you'll need to provide an [Access Token](../../authentication/personal-access-tokens.md) when querying the GraphQL by +providing an `Authorization` header containing a `Bearer` token. The header should take the following format: + +```bash +Authorization: Bearer +``` + +Authorization for actions exposed by the GraphQL endpoint will be performed based on the actor making the request. +For Personal Access Tokens, the token will carry the user's privileges. Please refer to [Access Token Management](/docs/api/graphql/token-management.md) for more information. + +## What's Next? + +Now that you are ready with GraphQL, how about browsing through some use cases? +Please refer to [Getting Started With GraphQL](/docs/api/graphql/getting-started.md) for more information. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/overview.md new file mode 100644 index 00000000..521b5436 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/overview.md @@ -0,0 +1,45 @@ +--- +slug: /api/graphql/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/graphql/overview.md +--- +# DataHub GraphQL API + +DataHub provides a rich [`graphql`](https://graphql.org/) API for programmatically interacting with the Entities & Relationships comprising your organization's Metadata Graph. + +## Getting Started + +To begin using the DataHub `graphql` API, please consult the [Getting Started](/docs/api/graphql/getting-started.md). + +For detailed guidance on using `graphql` for specific use cases, please refer to [DataHub API Comparison](/docs/api/datahub-apis.md#datahub-api-comparison). + +> **Pro Tip!** Throughout our API guides, we have examples of using GraphQL API. +> Lookout for the `| GraphQL |` tab within our tutorials. + +## About GraphQL + +[`graphql`](https://graphql.org/) provides a data query language and API with the following characteristics: + +- A **validated specification**: The `graphql` spec verifies a _schema_ on the API server. The server in turn is responsible + for validating incoming queries from the clients against that schema. +- **Strongly typed**: A GraphQL schema declares the universe of types and relationships composing the interface. +- **Document-oriented & hierarchical**: GraphQL makes it eay to ask for related entities using a familiar JSON document + structure. This minimizes the number of round-trip API requests a client must make to answer a particular question. +- **Flexible & efficient**: GraphQL provides a way to ask for only the data you want, and that's it. Ignore all + the rest. It allows you to replace multiple REST calls with one GraphQL call. +- **Large Open Source Ecosystem**: Open source GraphQL projects have been developed for [virtually every programming language](https://graphql.org/code/). With a thriving + community, it offers a sturdy foundation to build upon. + +For these reasons among others DataHub provides a GraphQL API on top of the Metadata Graph, +permitting easy exploration of the Entities & Relationships composing it. + +For more information about the GraphQL specification, check out [Introduction to GraphQL](https://graphql.org/learn/). + +## GraphQL Schema Reference + +The Reference docs in the sidebar are generated from the DataHub GraphQL schema. Each call to the `/api/graphql` endpoint is +validated against this schema. You can use these docs to understand data that is available for retrieval and operations +that may be performed using the API. + +- Available Operations: [Queries](/graphql/queries.md) (Reads) & [Mutations](/graphql/mutations.md) (Writes) +- Schema Types: [Objects](/graphql/objects.md), [Input Objects](/graphql/inputObjects.md), [Interfaces](/graphql/interfaces.md), [Unions](/graphql/unions.md), [Enums](/graphql/enums.md), [Scalars](/graphql/scalars.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/token-management.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/token-management.md new file mode 100644 index 00000000..19cda7f8 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/graphql/token-management.md @@ -0,0 +1,139 @@ +--- +title: Access Token Management +slug: /api/graphql/token-management +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/graphql/token-management.md +--- +# Access Token Management + +DataHub provides the following `graphql` endpoints for managing Access Tokens. In this page you will see examples as well +as explanations as to how to administrate access tokens within the project whether for yourself or others, depending on the caller's privileges. + +_Note_: This API makes use of DataHub Policies to safeguard against improper use. By default, a user will not be able to interact with it at all unless they have at least `Generate Personal Access Tokens` privileges. + +### Generating Access Tokens + +To generate an access token, simply use the `createAccessToken(input: GetAccessTokenInput!)` `graphql` Query. +This endpoint will return an `AccessToken` object, containing the access token string itself alongside with metadata +which will allow you to identify said access token later on. + +For example, to generate an access token for the `datahub` corp user, you can issue the following `graphql` Query: + +_As GraphQL_ + +```graphql +mutation { + createAccessToken( + input: { + type: PERSONAL + actorUrn: "urn:li:corpuser:datahub" + duration: ONE_HOUR + name: "my personal token" + } + ) { + accessToken + metadata { + id + name + description + } + } +} +``` + +_As CURL_ + +```curl +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'X-DataHub-Actor: urn:li:corpuser:datahub' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query":"mutation { createAccessToken(input: { type: PERSONAL, actorUrn: \"urn:li:corpuser:datahub\", duration: ONE_HOUR, name: \"my personal token\" } ) { accessToken metadata { id name description} } }", "variables":{}}' +``` + +### Listing Access Tokens + +Listing tokens is a powerful endpoint that allows you to list the tokens owned by a particular user (ie. YOU). +To list all tokens that you own, you must specify a filter with: `{field: "actorUrn", value: ""}` configuration. + +_As GraphQL_ + +```graphql +{ + listAccessTokens( + input: { + start: 0 + count: 100 + filters: [{ field: "ownerUrn", value: "urn:li:corpuser:datahub" }] + } + ) { + start + count + total + tokens { + urn + id + actorUrn + } + } +} +``` + +_As CURL_ + +```curl +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'X-DataHub-Actor: urn:li:corpuser:datahub' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query":"{ listAccessTokens(input: {start: 0, count: 100, filters: [{field: \"ownerUrn\", value: \"urn:li:corpuser:datahub\"}]}) { start count total tokens {urn id actorUrn} } }", "variables":{}}' +``` + +Admin users can also list tokens owned by other users of the platform. To list tokens belonging to other users, you must have the `Manage All Access Tokens` Platform privilege. + +_As GraphQL_ + +```graphql +{ + listAccessTokens(input: { start: 0, count: 100, filters: [] }) { + start + count + total + tokens { + urn + id + actorUrn + } + } +} +``` + +_As CURL_ + +```curl +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'X-DataHub-Actor: urn:li:corpuser:datahub' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query":"{ listAccessTokens(input: {start: 0, count: 100, filters: []}) { start count total tokens {urn id actorUrn} } }", "variables":{}}' +``` + +Other filters besides `actorUrn=` are possible. You can filter by property in the `DataHubAccessTokenInfo` aspect which you can find in the Entities documentation. + +### Revoking Access Tokens + +To revoke an existing access token, you can use the `revokeAccessToken` mutation. + +_As GraphQL_ + +```graphql +mutation { + revokeAccessToken(tokenId: "HnMJylxuowJ1FKN74BbGogLvXCS4w+fsd3MZdI35+8A=") +} +``` + +```curl +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'X-DataHub-Actor: urn:li:corpuser:datahub' \ +--header 'Content-Type: application/json' \ +--data-raw '{"query":"mutation {revokeAccessToken(tokenId: \"HnMJylxuowJ1FKN74BbGogLvXCS4w+fsd3MZdI35+8A=\")}","variables":{}}}' +``` + +This endpoint will return a boolean detailing whether the operation was successful. In case of failure, an error message will appear explaining what went wrong. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/openapi/openapi-usage-guide.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/openapi/openapi-usage-guide.md new file mode 100644 index 00000000..452b2ed4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/openapi/openapi-usage-guide.md @@ -0,0 +1,922 @@ +--- +title: DataHub OpenAPI Guide +sidebar_label: OpenAPI Guide +slug: /api/openapi/openapi-usage-guide +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/openapi/openapi-usage-guide.md +--- +# DataHub OpenAPI Guide + +## Why OpenAPI + +The OpenAPI standard is a widely used documentation and design approach for REST-ful APIs. +To make it easier to integrate with DataHub, we are publishing an OpenAPI based set of endpoints. + +Read [the DataHub API overview](../datahub-apis.md) to understand the rationale behind the different API-s and when to use each one. + +## Locating the OpenAPI endpoints + +Currently, the OpenAPI endpoints are isolated to a servlet on GMS and are automatically deployed with a GMS server. +The servlet includes auto-generation of an OpenAPI UI, also known as Swagger, which is available at **GMS_SERVER_HOST:GMS_PORT/openapi/swagger-ui/index.html**. For example, the Quickstart running locally exposes this at http://localhost:8080/openapi/swagger-ui/index.html. + +This is also exposed through DataHub frontend as a proxy with the same endpoint, but GMS host and port replaced with DataHub frontend's url ([Local Quickstart link](http://localhost:9002/openapi/swagger-ui/index.html)) and is available in the top right dropdown under the user profile picture as a link. +![image](https://github.com/datahub-project/static-assets/blob/main/imgs/api/openapi/openapi_dropdown.png?raw=true) + +Note that it is possible to get the raw JSON or YAML formats of the OpenAPI spec by navigating to [**BASE_URL/openapi/v3/api-docs**](http://localhost:9002/openapi/v3/api-docs) or [**BASE_URL/openapi/v3/api-docs.yaml**](http://localhost:9002/openapi/v3/api-docs.yaml). +The raw forms can be fed into codegen systems to generate client side code in the language of your choice that support the OpenAPI format. We have noticed varying degrees of maturity with different languages in these codegen systems so some may require customizations to be fully compatible. + +The OpenAPI UI includes explorable schemas for request and response objects that are fully documented. The models used +in the OpenAPI UI are all autogenerated at build time from the PDL models to JSON Schema compatible Java Models. + +## Understanding the OpenAPI endpoints + +While the full OpenAPI spec is always available at [**GMS_SERVER_HOST:GMS_PORT/openapi/swagger-ui/index.html**](http://localhost:8080/openapi/swagger-ui/index.html), here's a quick overview of the main OpenAPI endpoints and their purpose. + +### Entities (/entities) + +The entities endpoints are intended for reads and writes to the metadata graph. The entire DataHub metadata model is available for you to write to (as entity, aspect pairs) or to read an individual entity's metadata from. See [examples](#entities-entities-endpoint) below. + +### Relationships (/relationships) + +The relationships endpoints are intended for you to query the graph, to navigate relationships from one entity to others. See [examples](#relationships-relationships-endpoint) below. + +### Timeline (/timeline) + +The timeline endpoints are intended for querying the versioned history of a given entity over time. For example, you can query a dataset for all schema changes that have happened to it over time, or all documentation changes that have happened to it. See [this](../../dev-guides/timeline.md) guide for more details. + +### Platform (/platform) + +Even lower-level API-s that allow you to write metadata events into the DataHub platform using a standard format. + +### Example Requests + +#### Entities (/entities) endpoint + +##### POST (UPSERT) + +A post without any additional URL parameters performs an UPSERT of entity's aspects. The entity will be +created if it doesn't exist or updated if it does. + +```shell +curl --location --request POST 'localhost:8080/openapi/entities/v1/' \ +--header 'Content-Type: application/json' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer ' \ +--data-raw '[ + { + "aspect": { + "__type": "SchemaMetadata", + "schemaName": "SampleHdfsSchema", + "platform": "urn:li:dataPlatform:platform", + "platformSchema": { + "__type": "MySqlDDL", + "tableSchema": "schema" + }, + "version": 0, + "created": { + "time": 1621882982738, + "actor": "urn:li:corpuser:etl", + "impersonator": "urn:li:corpuser:jdoe" + }, + "lastModified": { + "time": 1621882982738, + "actor": "urn:li:corpuser:etl", + "impersonator": "urn:li:corpuser:jdoe" + }, + "hash": "", + "fields": [ + { + "fieldPath": "county_fips_codefg", + "jsonPath": "null", + "nullable": true, + "description": "null", + "type": { + "type": { + "__type": "StringType" + } + }, + "nativeDataType": "String()", + "recursive": false + }, + { + "fieldPath": "county_name", + "jsonPath": "null", + "nullable": true, + "description": "null", + "type": { + "type": { + "__type": "StringType" + } + }, + "nativeDataType": "String()", + "recursive": false + } + ] + }, + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)" + } +]' +``` + +##### POST (CREATE) + +The second POST example will write the update ONLY if the entity doesn't exist. If the entity does exist the +command will return an error instead of overwriting the entity. + +In this example we've added a URL parameter `createEntityIfNotExists=true` + +```shell +curl --location --request POST 'localhost:8080/openapi/entities/v1/?createEntityIfNotExists=true' \ +--header 'Content-Type: application/json' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer ' \ +--data-raw '' +``` + +If the entity doesn't exist the response will be identical to the previous example. In the case where the entity already exists, +the following error will occur. + +> 422 ValidationExceptionCollection{EntityAspect:(`urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD),schemaMetadata)` Exceptions: [com.linkedin.metadata.aspect.plugins.validation.AspectValidationException: Cannot perform CREATE if not exists since the entity key already exists.]} + +##### GET + +```shell +curl --location --request GET 'localhost:8080/openapi/entities/v1/latest?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&aspectNames=schemaMetadata' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer ' +``` + +##### DELETE + +```shell +curl --location --request DELETE 'localhost:8080/openapi/entities/v1/?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&soft=true' \ +--header 'Accept: application/json' \ +--header 'Authorization: Bearer ' +``` + +#### Postman Collection + +Collection includes a POST, GET, and DELETE for a single entity with a SchemaMetadata aspect + +```json +{ + "info": { + "_postman_id": "87b7401c-a5dc-47e4-90b4-90fe876d6c28", + "name": "DataHub OpenAPI", + "description": "A description", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "entities/v1", + "item": [ + { + "name": "post Entities 1", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Accept", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "[\n {\n \"aspect\": {\n \"__type\": \"SchemaMetadata\",\n \"schemaName\": \"SampleHdfsSchema\",\n \"platform\": \"urn:li:dataPlatform:platform\",\n \"platformSchema\": {\n \"__type\": \"MySqlDDL\",\n \"tableSchema\": \"schema\"\n },\n \"version\": 0,\n \"created\": {\n \"time\": 1621882982738,\n \"actor\": \"urn:li:corpuser:etl\",\n \"impersonator\": \"urn:li:corpuser:jdoe\"\n },\n \"lastModified\": {\n \"time\": 1621882982738,\n \"actor\": \"urn:li:corpuser:etl\",\n \"impersonator\": \"urn:li:corpuser:jdoe\"\n },\n \"hash\": \"\",\n \"fields\": [\n {\n \"fieldPath\": \"county_fips_codefg\",\n \"jsonPath\": \"null\",\n \"nullable\": true,\n \"description\": \"null\",\n \"type\": {\n \"type\": {\n \"__type\": \"StringType\"\n }\n },\n \"nativeDataType\": \"String()\",\n \"recursive\": false\n },\n {\n \"fieldPath\": \"county_name\",\n \"jsonPath\": \"null\",\n \"nullable\": true,\n \"description\": \"null\",\n \"type\": {\n \"type\": {\n \"__type\": \"StringType\"\n }\n },\n \"nativeDataType\": \"String()\",\n \"recursive\": false\n }\n ]\n },\n \"aspectName\": \"schemaMetadata\",\n \"entityType\": \"dataset\",\n \"entityUrn\": \"urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/openapi/entities/v1/", + "host": ["{{baseUrl}}"], + "path": ["openapi", "entities", "v1", ""] + } + }, + "response": [ + { + "name": "OK", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"aspect\": {\n \"value\": \"\"\n },\n \"aspectName\": \"aliquip ipsum tempor\",\n \"entityType\": \"ut est\",\n \"entityUrn\": \"enim in nulla\",\n \"entityKeyAspect\": {\n \"value\": \"\"\n }\n },\n {\n \"aspect\": {\n \"value\": \"\"\n },\n \"aspectName\": \"ipsum id\",\n \"entityType\": \"deser\",\n \"entityUrn\": \"aliqua sit\",\n \"entityKeyAspect\": {\n \"value\": \"\"\n }\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{baseUrl}}/entities/v1/", + "host": ["{{baseUrl}}"], + "path": ["entities", "v1", ""] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "cookie": [], + "body": "[\n \"c\",\n \"labore dolor exercitation in\"\n]" + } + ] + }, + { + "name": "delete Entities", + "request": { + "method": "DELETE", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/openapi/entities/v1/?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&soft=true", + "host": ["{{baseUrl}}"], + "path": ["openapi", "entities", "v1", ""], + "query": [ + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request." + }, + { + "key": "urns", + "value": "labore dolor exercitation in", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request.", + "disabled": true + }, + { + "key": "soft", + "value": "true", + "description": "Determines whether the delete will be soft or hard, defaults to true for soft delete" + } + ] + } + }, + "response": [ + { + "name": "OK", + "originalRequest": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{baseUrl}}/entities/v1/?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&soft=true", + "host": ["{{baseUrl}}"], + "path": ["entities", "v1", ""], + "query": [ + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)" + }, + { + "key": "urns", + "value": "officia occaecat elit dolor", + "disabled": true + }, + { + "key": "soft", + "value": "true" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "cookie": [], + "body": "[\n {\n \"rowsRolledBack\": [\n {\n \"urn\": \"urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)\"\n }\n ],\n \"rowsDeletedFromEntityDeletion\": 1\n }\n]" + } + ] + }, + { + "name": "get Entities", + "protocolProfileBehavior": { + "disableUrlEncoding": false + }, + "request": { + "method": "GET", + "header": [ + { + "key": "Accept", + "value": "application/json" + } + ], + "url": { + "raw": "{{baseUrl}}/openapi/entities/v1/latest?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&aspectNames=schemaMetadata", + "host": ["{{baseUrl}}"], + "path": ["openapi", "entities", "v1", "latest"], + "query": [ + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request." + }, + { + "key": "urns", + "value": "labore dolor exercitation in", + "description": "(Required) A list of raw urn strings, only supports a single entity type per request.", + "disabled": true + }, + { + "key": "aspectNames", + "value": "schemaMetadata", + "description": "The list of aspect names to retrieve" + }, + { + "key": "aspectNames", + "value": "labore dolor exercitation in", + "description": "The list of aspect names to retrieve", + "disabled": true + } + ] + } + }, + "response": [ + { + "name": "OK", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "{{baseUrl}}/entities/v1/latest?urns=urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)&aspectNames=schemaMetadata", + "host": ["{{baseUrl}}"], + "path": ["entities", "v1", "latest"], + "query": [ + { + "key": "urns", + "value": "non exercitation occaecat", + "disabled": true + }, + { + "key": "urns", + "value": "urn:li:dataset:(urn:li:dataPlatform:platform,testSchemaIngest,PROD)" + }, + { + "key": "aspectNames", + "value": "non exercitation occaecat", + "disabled": true + }, + { + "key": "aspectNames", + "value": "schemaMetadata" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "cookie": [], + "body": "{\n \"responses\": {\n \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\": {\n \"entityName\": \"dataset\",\n \"urn\": \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\",\n \"aspects\": {\n \"datasetKey\": {\n \"name\": \"datasetKey\",\n \"type\": \"VERSIONED\",\n \"version\": 0,\n \"value\": {\n \"__type\": \"DatasetKey\",\n \"platform\": \"urn:li:dataPlatform:hive\",\n \"name\": \"SampleHiveDataset\",\n \"origin\": \"PROD\"\n },\n \"created\": {\n \"time\": 1650657843351,\n \"actor\": \"urn:li:corpuser:__datahub_system\"\n }\n },\n \"schemaMetadata\": {\n \"name\": \"schemaMetadata\",\n \"type\": \"VERSIONED\",\n \"version\": 0,\n \"value\": {\n \"__type\": \"SchemaMetadata\",\n \"schemaName\": \"SampleHiveSchema\",\n \"platform\": \"urn:li:dataPlatform:hive\",\n \"version\": 0,\n \"created\": {\n \"time\": 1581407189000,\n \"actor\": \"urn:li:corpuser:jdoe\"\n },\n \"lastModified\": {\n \"time\": 1581407189000,\n \"actor\": \"urn:li:corpuser:jdoe\"\n },\n \"hash\": \"\",\n \"platformSchema\": {\n \"__type\": \"KafkaSchema\",\n \"documentSchema\": \"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"SampleHiveSchema\\\",\\\"namespace\\\":\\\"com.linkedin.dataset\\\",\\\"doc\\\":\\\"Sample Hive dataset\\\",\\\"fields\\\":[{\\\"name\\\":\\\"field_foo\\\",\\\"type\\\":[\\\"string\\\"]},{\\\"name\\\":\\\"field_bar\\\",\\\"type\\\":[\\\"boolean\\\"]}]}\"\n },\n \"fields\": [\n {\n \"fieldPath\": \"field_foo\",\n \"nullable\": false,\n \"description\": \"Foo field description\",\n \"type\": {\n \"type\": {\n \"__type\": \"BooleanType\"\n }\n },\n \"nativeDataType\": \"varchar(100)\",\n \"recursive\": false,\n \"isPartOfKey\": true\n },\n {\n \"fieldPath\": \"field_bar\",\n \"nullable\": false,\n \"description\": \"Bar field description\",\n \"type\": {\n \"type\": {\n \"__type\": \"BooleanType\"\n }\n },\n \"nativeDataType\": \"boolean\",\n \"recursive\": false,\n \"isPartOfKey\": false\n }\n ]\n },\n \"created\": {\n \"time\": 1650610810000,\n \"actor\": \"urn:li:corpuser:UNKNOWN\"\n }\n }\n }\n }\n }\n}" + } + ] + } + ], + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{token}}", + "type": "string" + } + ] + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [""] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [""] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [""] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [""] + } + } + ], + "variable": [ + { + "key": "baseUrl", + "value": "localhost:8080", + "type": "string" + }, + { + "key": "token", + "value": "eyJhbGciOiJIUzI1NiJ9.eyJhY3RvclR5cGUiOiJVU0VSIiwiYWN0b3JJZCI6ImRhdGFodWIiLCJ0eXBlIjoiUEVSU09OQUwiLCJ2ZXJzaW9uIjoiMSIsImV4cCI6MTY1MDY2MDY1NSwianRpIjoiM2E4ZDY3ZTItOTM5Yi00NTY3LWE0MjYtZDdlMDA1ZGU3NjJjIiwic3ViIjoiZGF0YWh1YiIsImlzcyI6ImRhdGFodWItbWV0YWRhdGEtc2VydmljZSJ9.pp_vW2u1tiiTT7U0nDF2EQdcayOMB8jatiOA8Je4JJA", + "type": "default" + } + ] +} +``` + +#### Relationships (/relationships) endpoint + +##### GET + +**Sample Request** + +```shell +curl -X 'GET' \ + 'http://localhost:8080/openapi/relationships/v1/?urn=urn%3Ali%3Acorpuser%3Adatahub&relationshipTypes=IsPartOf&direction=INCOMING&start=0&count=200' \ + -H 'accept: application/json' +``` + +**Sample Response** + +```json +{ + "start": 0, + "count": 2, + "total": 2, + "entities": [ + { + "relationshipType": "IsPartOf", + "urn": "urn:li:corpGroup:bfoo" + }, + { + "relationshipType": "IsPartOf", + "urn": "urn:li:corpGroup:jdoe" + } + ] +} +``` + +## Programmatic Usage + +Programmatic usage of the models can be done through the Java Rest Emitter which includes the generated models. A minimal +Java project for emitting to the OpenAPI endpoints would need the following dependencies (gradle format): + +```groovy +dependencies { + implementation 'io.acryl:datahub-client:' + implementation 'org.apache.httpcomponents:httpclient:' + implementation 'org.apache.httpcomponents:httpasyncclient:' +} +``` + +### Writing metadata events to the /platform endpoints + +The following code emits metadata events through OpenAPI by constructing a list of `UpsertAspectRequest`s. Behind the scenes, this is using the **/platform/entities/v1** endpoint to send metadata to GMS. + +```java +import io.datahubproject.openapi.generated.DatasetProperties; +import datahub.client.rest.RestEmitter; +import datahub.event.UpsertAspectRequest; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; + + +public class Main { + public static void main(String[] args) throws IOException, ExecutionException, InterruptedException { + RestEmitter emitter = RestEmitter.createWithDefaults(); + + List requests = new ArrayList<>(); + UpsertAspectRequest upsertAspectRequest = UpsertAspectRequest.builder() + .entityType("dataset") + .entityUrn("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.my-other-dataset.user-table,PROD)") + .aspect(new DatasetProperties().description("This is the canonical User profile dataset")) + .build(); + UpsertAspectRequest upsertAspectRequest2 = UpsertAspectRequest.builder() + .entityType("dataset") + .entityUrn("urn:li:dataset:(urn:li:dataPlatform:bigquery,my-project.another-dataset.user-table,PROD)") + .aspect(new DatasetProperties().description("This is the canonical User profile dataset 2")) + .build(); + requests.add(upsertAspectRequest); + requests.add(upsertAspectRequest2); + System.out.println(emitter.emit(requests, null).get()); + System.exit(0); + } +} +``` + +## OpenAPI v3 Features + +### Conditional Writes + +All the create/POST endpoints for aspects support `headers` in the POST body to support batch APIs. See the docs in the +[MetadataChangeProposal](../../advanced/mcp-mcl.md) section for the use of these headers to support conditional writes semantics. + +### Batch Get + +Batch get endpoints in the form of `/v3/entity/{entityName}/batchGet` exist for all entities. This endpoint allows +fetching entity and aspects in bulk. In combination with the `If-Version-Match` header it can also retrieve +a specific version of the aspects, however it defaults to the latest aspect version. Currently, this interface is limited +to returning a single version for each entity/aspect however different versions can be specified across entities. + +A few example queries are as follows: + +Example Request: + +Fetch the latest aspects for the given URNs with the url parameter `systemMetadata=true` in order to view the current +versions of the aspects. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "globalTags": {}, + "datasetProperties": {} + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "globalTags": {}, + "datasetProperties": {} + } +] +``` + +Example Response: + +Notice that `systemMetadata` contains `"version": "1"` for each of the aspects that exist in the system. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users deleted on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548776, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + } + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users created on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548773, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + }, + "globalTags": { + "value": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "systemMetadata": { + "properties": { + "appSource": "ui" + }, + "version": "1", + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "runId": "no-run-id-provided" + } + } + } +] +``` + +Next let's mutate `globalTags` for the second URN by adding a new tag. This will increment the version of +the `globalTags` aspect. The response will then look at like the following, notice the incremented +`"version": "2"` in `systemMetadata` for the `globalTags` aspect. Also notice that there are now 2 tags present, unlike +previously where only `urn:li:tag:NeedsDocumentation` was present. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users deleted on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548776, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + } + }, + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "datasetProperties": { + "value": { + "description": "table containing all the users created on a single day", + "customProperties": { + "encoding": "utf-8" + }, + "tags": [] + }, + "systemMetadata": { + "properties": { + "clientVersion": "1!0.0.0.dev0", + "clientId": "acryl-datahub" + }, + "version": "1", + "lastObserved": 1720781548773, + "lastRunId": "file-2024_07_12-05_52_28", + "runId": "file-2024_07_12-05_52_28" + } + }, + "globalTags": { + "value": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + }, + { + "tag": "urn:li:tag:Legacy" + } + ] + }, + "systemMetadata": { + "properties": { + "appSource": "ui" + }, + "version": "2", + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "runId": "no-run-id-provided" + } + } + } +] +``` + +Next, we'll retrieve the previous version of the `globalTags` for the one aspect with a version 2 with the following query. +We can do this by populating the `headers` map with `If-Version-Match` to retrieve the previous version 1. + +Example Request: + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "globalTags": { + "headers": { + "If-Version-Match": "1" + } + } + } +] +``` + +Example Response: + +The previous version `1` of the `globalTags` aspect is returned as expected with only the single tag. + +```json +[ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "globalTags": { + "value": { + "tags": [ + { + "tag": "urn:li:tag:NeedsDocumentation" + } + ] + }, + "systemMetadata": { + "properties": { + "appSource": "ui" + }, + "version": "1", + "lastObserved": 0, + "lastRunId": "no-run-id-provided", + "runId": "no-run-id-provided" + } + } + } +] +``` + +### Generic Patching + +The OpenAPI v3 PATCH endpoints offer advantages over previous patch support by removing the need for specific backend +code to handle patching, see [Template Classes](/docs/advanced/patch.md#implementation-details). This technique leverages +the natural JSON structure of aspects and extends a generic patching mechanism based on the JSON Patch standard (RFC 6902) +with significant enhancements for array operations. + +Note that the traditional patching templates are used by default in order to maintain backwards compatibility. The generic +patching is activated if `arrayPrimaryKeys` is non-empty or `forceGenericPatch` is set to `true`. + +#### Advanced JSON Patch for Arrays + +Standard JSON Patch Limitations: + +The JSON Patch standard allows for array modifications primarily through index-based operations: + +- `add/[index]`: Insert at specific position +- `remove/[index]`: Remove element at position +- `replace/[index]`: Replace element at position + +This approach becomes problematic when: + +- Array ordering is unpredictable or may change +- Multiple clients are modifying the same resource concurrently +- The client doesn't have knowledge of current array indexes + +Key Concept: Array Primary Keys: + +DataHub extends the JSON Patch standard with the `arrayPrimaryKeys` field that transforms array operations into map-like +operations: + +- Arrays are conceptually treated as maps where each element can be addressed by its primary key +- Primary keys can be composite (multiple fields combined) +- Path expressions use these keys instead of numeric indexes +- Backend handles the conversion between the map-like operations and actual array modifications + +This approach allows for: + +- Idempotent operations regardless of array order +- Targeted modifications without needing to know current array state +- Concurrent updates without conflicts (when modifying different array elements) +- More intuitive and maintainable API usage + +#### Primary Key Definition and Path Construction + +In this section lets take a look at a common patch use-case, adding/removing global tags while considering an +attribution source for the tag. The following examples are specifically modifying the `globalTags` aspect. + +Defining Primary Keys: + +The `arrayPrimaryKeys` property specifies which fields uniquely identify each array element: + +```json +{ + "arrayPrimaryKeys": { + "tags": ["attribution␟source", "tag"] + }, + "patch": [ + { + "op": "add", + "path": "/tags/urn:li:platformResource:source1/urn:li:tag:tag1", + "value": { + "tag": "urn:li:tag:tag1", + "attribution": { + "source": "urn:li:platformResource:source1", + "actor": "urn:li:corpuser:user", + "time": 0 + } + } + } + ] +} +``` + +In this example: + +- tags is the array field being patched +- The primary key is a composite of attribution.source and tag +- The `␟`, "Unit Separator" (U+241F), delimiter indicates a nested path in the first key component + +Path Construction: + +When the backend processes a patch operation, it: + +- Converts the array to a map using the specified primary key fields +- Processes the operation against this map representation +- Converts the map back to an array for storage + +For example, with the path: + +```text +/tags/urn:li:platformResource:source1/urn:li:tag:tag1 +``` + +The system: + +- Identifies tags as the target array +- Uses `urn:li:platformResource:source1` as the value for `attribution.source` +- Uses `urn:li:tag:tag1` as the value for the tag +- Finds the matching array element(s) with these key values + +Supported Operations: + +The implementation supports standard JSON Patch operations: + +| Operation | Description | +| --------- | -------------------------------------- | +| add | Add a new element or replace if exists | +| remove | Remove an element matching keys | + +#### Patch Operation Examples + +Adding Tagged Elements with Attribution: + +```json +{ + "op": "add", + "path": "/tags/urn:li:platformResource:source1/urn:li:tag:tag1", + "value": { + "tag": "urn:li:tag:tag1", + "attribution": { + "source": "urn:li:platformResource:source1", + "actor": "urn:li:corpuser:user", + "time": 0 + } + } +} +``` + +This operation: + +- Checks if an element with matching keys exists in the array +- If not found, adds the new element +- If found, replaces the existing element + +Selective Removal: + +```json +{ + "op": "remove", + "path": "/tags/urn:li:platformResource:source1/urn:li:tag:tag1" +} +``` + +This operation: + +- Finds elements matching the composite key +- Removes only those elements +- Preserves other elements, even with partially matching keys diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/evaluate-tests.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/evaluate-tests.md new file mode 100644 index 00000000..ee859c5b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/evaluate-tests.md @@ -0,0 +1,27 @@ +--- +title: Evaluate Tests Endpoint +slug: /api/restli/evaluate-tests +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/restli/evaluate-tests.md +--- +# Evaluate Tests Endpoint + + + +You can do a HTTP POST request to `/gms/test?action=evaluate` endpoint with the `urn` as part of JSON Payload to run metadata tests for the particular URN. + +``` +curl --location --request POST 'https://DOMAIN.acryl.io/gms/test?action=evaluate' \ +--header 'Authorization: Bearer TOKEN' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "urn": "YOUR_URN" +}' +``` + +w +The supported parameters are + +- `urn` - Required URN string +- `push` - Optional Boolean - whether or not to push the results to persist them. Default `false`. +- `testUrns` - Optional List of string - If you wish to get specific test URNs evaluated diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-elastic-task-status.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-elastic-task-status.md new file mode 100644 index 00000000..4597e848 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-elastic-task-status.md @@ -0,0 +1,58 @@ +--- +title: Get ElasticSearch Task Status Endpoint +slug: /api/restli/get-elastic-task-status +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/restli/get-elastic-task-status.md +--- +# Get ElasticSearch Task Status Endpoint + +You can do a HTTP POST request to `/gms/operations?action=getEsTaskStatus` endpoint to see the status of the input task running in ElasticSearch. For example, the task ID given by the [`truncateTimeseriesAspect` endpoint](./truncate-time-series-aspect.md). The task ID can be passed in as a string with node name and task ID separated by a colon (as is output by the previous API), or the node name and task ID parameters separately. + +``` +curl --location --request POST 'https://demo.datahub.com/api/gms/operations?action=getEsTaskStatus' \ +--header 'Authorization: Bearer TOKEN' +--header 'Content-Type: application/json' \ +--data-raw '{ + "task": "aB1cdEf2GHIJKLMnoPQr3S:123456" +}' + +curl --location --request POST http://localhost:8080/operations\?action\=getEsTaskStatus \ +--header 'Authorization: Bearer TOKEN' +--header 'Content-Type: application/json' \ +--data-raw '{ + "nodeId": "aB1cdEf2GHIJKLMnoPQr3S", + taskId: 12345 +}' +``` + +The output will be a string representing a JSON object with the task status. + +``` +{ + "value": "{\"error\":\"Could not get task status for XIAMx5WySACgg9XxBgaKmw:12587\"}" +} +``` + +``` +"{ + "completed": true, + "taskId": "qhxGdzytQS-pQek8CwBCZg:54654", + "runTimeNanos": 1179458, + "status": "{ + "total": 0, + "updated": 0, + "created": 0, + "deleted": 0, + "batches": 0, + "version_conflicts": 0, + "noops": 0, + "retries": { + "bulk": 0, + "search": 0 + }, + "throttled_millis": 0, + "requests_per_second": -1.0, + "throttled_until_millis": 0 + } +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-index-sizes.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-index-sizes.md new file mode 100644 index 00000000..321701c3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/get-index-sizes.md @@ -0,0 +1,25 @@ +--- +title: Get Index Sizes Endpoint +slug: /api/restli/get-index-sizes +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/restli/get-index-sizes.md +--- +# Get Index Sizes Endpoint + +You can do a HTTP POST request to `/gms/operations?action=getIndexSizes` endpoint with no parameters to see the size of indices in ElasticSearch. For now, only timeseries indices are supported, as they can grow indefinitely, and the `truncateTimeseriesAspect` endpoint is provided to clean up old entries. This endpoint can be used in conjunction with the cleanup endpoint to see which indices are the largest before truncation. + +``` +curl --location --request POST 'https://demo.datahub.com/api/gms/operations?action=getIndexSizes' \ +--header 'Authorization: Bearer TOKEN' +``` + +The endpoint takes no parameters, and the output will be a string representing a JSON object containing the following information about each index: + +``` + { + "aspectName": "datasetusagestatistics", + "sizeMb": 0.208, + "indexName": "dataset_datasetusagestatisticsaspect_v1", + "entityName": "dataset" + } +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restli-overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restli-overview.md new file mode 100644 index 00000000..1e011aec --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restli-overview.md @@ -0,0 +1,1442 @@ +--- +title: Rest.li API +slug: /api/restli/restli-overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/restli/restli-overview.md +--- +# Rest.li API + +You can access basic documentation on the API endpoints by opening the `/restli/docs` endpoint in the browser. + +``` +python -c "import webbrowser; webbrowser.open('http://localhost:8080/restli/docs', new=2)" +``` + +\*Please note that because DataHub is in a period of rapid development, the APIs below are subject to change. + +#### Sample API Calls + +#### Ingesting Aspects + +To ingest individual aspects into DataHub, you can use the following CURL: + +```shell +curl --location --request POST 'http://localhost:8080/aspects?action=ingestProposal' \ +--header 'X-RestLi-Protocol-Version: 2.0.0' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "proposal" : { + "entityType": "dataset", + "entityUrn" : "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType" : "UPSERT", + "aspectName" : "datasetUsageStatistics", + "aspect" : { + "value" : "{ \"timestampMillis\":1629840771000,\"uniqueUserCount\" : 10, \"totalSqlQueries\": 20, \"fieldCounts\": [ {\"fieldPath\": \"col1\", \"count\": 20}, {\"fieldPath\" : \"col2\", \"count\": 5} ]}", + "contentType": "application/json" + } + } +}' +``` + +Notice that you need to provide the target entity urn, the entity type, a change type (`UPSERT` + `DELETE` supported), +the aspect name, and a JSON-serialized aspect, which corresponds to the PDL schema defined for the aspect. + +For more examples of serialized aspect payloads, see [bootstrap_mce.json](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/mce_files/bootstrap_mce.json). + +#### Ingesting Entities (Legacy) + +> Note - we are deprecating support for ingesting Entities via Snapshots. Please see **Ingesting Aspects** above for the latest +> guidance around ingesting metadata into DataHub without defining or changing the legacy snapshot models. (e.g. using ConfigEntityRegistry) + +The Entity Snapshot Ingest endpoints allow you to ingest multiple aspects about a particular entity at the same time. + +##### Create a user + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.CorpUserSnapshot":{ + "urn":"urn:li:corpuser:footbarusername", + "aspects":[ + { + "com.linkedin.identity.CorpUserInfo":{ + "active":true, + "displayName":"Foo Bar", + "fullName":"Foo Bar", + "email":"fbar@linkedin.com" + } + } + ] + } + } + } +}' +``` + +##### Create a group + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.CorpGroupSnapshot":{ + "urn":"urn:li:corpGroup:dev", + "aspects":[ + { + "com.linkedin.identity.CorpGroupInfo":{ + "email":"dev@linkedin.com", + "admins":[ + "urn:li:corpUser:jdoe" + ], + "members":[ + "urn:li:corpUser:datahub", + "urn:li:corpUser:jdoe" + ], + "groups":[ + + ] + } + } + ] + } + } + } +}' +``` + +##### Create a dataset + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.DatasetSnapshot":{ + "urn":"urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)", + "aspects":[ + { + "com.linkedin.common.Ownership":{ + "owners":[ + { + "owner":"urn:li:corpuser:fbar", + "type":"DATAOWNER" + } + ], + "lastModified":{ + "time":0, + "actor":"urn:li:corpuser:fbar" + } + } + }, + { + "com.linkedin.common.InstitutionalMemory":{ + "elements":[ + { + "url":"https://www.linkedin.com", + "description":"Sample doc", + "createStamp":{ + "time":0, + "actor":"urn:li:corpuser:fbar" + } + } + ] + } + }, + { + "com.linkedin.schema.SchemaMetadata":{ + "schemaName":"FooEvent", + "platform":"urn:li:dataPlatform:foo", + "version":0, + "created":{ + "time":0, + "actor":"urn:li:corpuser:fbar" + }, + "lastModified":{ + "time":0, + "actor":"urn:li:corpuser:fbar" + }, + "hash":"", + "platformSchema":{ + "com.linkedin.schema.KafkaSchema":{ + "documentSchema":"{\"type\":\"record\",\"name\":\"MetadataChangeEvent\",\"namespace\":\"com.linkedin.mxe\",\"doc\":\"Kafka event for proposing a metadata change for an entity.\",\"fields\":[{\"name\":\"auditHeader\",\"type\":{\"type\":\"record\",\"name\":\"KafkaAuditHeader\",\"namespace\":\"com.linkedin.avro2pegasus.events\",\"doc\":\"Header\"}}]}" + } + }, + "fields":[ + { + "fieldPath":"foo", + "description":"Bar", + "nativeDataType":"string", + "type":{ + "type":{ + "com.linkedin.schema.StringType":{ + + } + } + } + } + ] + } + } + ] + } + } + } +}' +``` + +##### Create a chart + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.ChartSnapshot":{ + "urn":"urn:li:chart:(looker,baz1)", + "aspects":[ + { + "com.linkedin.chart.ChartInfo":{ + "title":"Baz Chart 1", + "description":"Baz Chart 1", + "inputs":[ + { + "string":"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" + } + ], + "lastModified":{ + "created":{ + "time":0, + "actor":"urn:li:corpuser:jdoe" + }, + "lastModified":{ + "time":0, + "actor":"urn:li:corpuser:datahub" + } + } + } + } + ] + } + } + } +}' +``` + +##### Create a dashboard + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.DashboardSnapshot":{ + "urn":"urn:li:dashboard:(looker,baz)", + "aspects":[ + { + "com.linkedin.dashboard.DashboardInfo":{ + "title":"Baz Dashboard", + "description":"Baz Dashboard", + "charts":[ + "urn:li:chart:(looker,baz1)", + "urn:li:chart:(looker,baz2)" + ], + "lastModified":{ + "created":{ + "time":0, + "actor":"urn:li:corpuser:jdoe" + }, + "lastModified":{ + "time":0, + "actor":"urn:li:corpuser:datahub" + } + } + } + } + ] + } + } + } +}' +``` + +##### Create Tags + +To create a new tag called "Engineering", we can use the following curl. + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.TagSnapshot":{ + "urn":"urn:li:tag:Engineering", + "aspects":[ + { + "com.linkedin.tag.TagProperties":{ + "name":"Engineering", + "description":"The tag will be assigned to all assets owned by the Eng org." + } + } + ] + } + } + } +}' +``` + +This tag can subsequently be associated with a Data Asset using the "Global Tags" aspect associated with each. For example, +to add a tag to a Dataset, you can use the following CURL: + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.DatasetSnapshot":{ + "urn":"urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)", + "aspects":[ + { + "com.linkedin.common.GlobalTags":{ + "tags":[ + { + "tag":"urn:li:tag:Engineering" + } + ] + } + } + ] + } + } + } +}' +``` + +And to add the tag to a field in a particular Dataset's schema, you can use a CURL to update the EditableSchemaMetadata Aspect: + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.DatasetSnapshot":{ + "urn":"urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)", + "aspects":[ + { + "com.linkedin.schema.EditableSchemaMetadata": { + "editableSchemaFieldInfo":[ + { + "fieldPath":"myFieldName", + "globalTags": { + "tags":[ + { + "tag":"urn:li:tag:Engineering" + } + ] + } + } + ] + } + } + ] + } + } + } +}' +``` + +##### Soft Deleting an Entity + +DataHub uses a special "Status" aspect associated with each entity to represent the lifecycle state of an Entity. +To soft delete an entire Entity, you can use the special "Status" aspect. Note that soft deletion means that +an entity will not be discoverable via Search or Browse, but its entity page will still be viewable. + +For example, to delete a particular chart: + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.ChartSnapshot":{ + "aspects":[ + { + "com.linkedin.common.Status":{ + "removed": true + } + } + ], + "urn":"urn:li:chart:(looker,baz1)" + } + } + } +}' +``` + +To re-enable the Entity, you can make a similar request: + +``` +curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{ + "entity":{ + "value":{ + "com.linkedin.metadata.snapshot.ChartSnapshot":{ + "aspects":[ + { + "com.linkedin.common.Status":{ + "removed": false + } + } + ], + "urn":"urn:li:chart:(looker,baz1)" + } + } + } +}' +``` + +To issue a hard delete or soft-delete, or undo a particular ingestion run, you can use the [DataHub CLI](docs/how/delete-metadata.md). + +#### Retrieving Entity Aspects + +Simply curl the `entitiesV2` endpoint of GMS: + +``` +curl 'http://localhost:8080/entitiesV2/' +``` + +For example, to retrieve the latest aspects associated with the "SampleHdfsDataset" `Dataset`: + +``` +curl --header 'X-RestLi-Protocol-Version: 2.0.0' 'http://localhost:8080/entitiesV2/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahdfs%2CSampleHdfsDataset%2CPROD%29' +``` + +**Example Response** + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "aspects": { + "editableSchemaMetadata": { + "name": "editableSchemaMetadata", + "version": 0, + "value": { + "created": { + "actor": "urn:li:corpuser:jdoe", + "time": 1581407189000 + }, + "editableSchemaFieldInfo": [ + { + "fieldPath": "shipment_info", + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Legacy" + } + ] + } + } + ], + "lastModified": { + "actor": "urn:li:corpuser:jdoe", + "time": 1581407189000 + } + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + }, + "browsePaths": { + "name": "browsePaths", + "version": 0, + "value": { + "paths": ["/prod/hdfs/SampleHdfsDataset"] + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + }, + "datasetKey": { + "name": "datasetKey", + "version": 0, + "value": { + "name": "SampleHdfsDataset", + "platform": "urn:li:dataPlatform:hdfs", + "origin": "PROD" + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + }, + "ownership": { + "name": "ownership", + "version": 0, + "value": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER" + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER" + } + ], + "lastModified": { + "actor": "urn:li:corpuser:jdoe", + "time": 1581407189000 + } + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + }, + "dataPlatformInstance": { + "name": "dataPlatformInstance", + "version": 0, + "value": { + "platform": "urn:li:dataPlatform:hdfs" + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + }, + "institutionalMemory": { + "name": "institutionalMemory", + "version": 0, + "value": { + "elements": [ + { + "createStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1581407189000 + }, + "description": "Sample doc", + "url": "https://www.linkedin.com" + } + ] + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + }, + "schemaMetadata": { + "name": "schemaMetadata", + "version": 0, + "value": { + "created": { + "actor": "urn:li:corpuser:jdoe", + "time": 1581407189000 + }, + "platformSchema": { + "com.linkedin.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"SampleHdfsSchema\",\"namespace\":\"com.linkedin.dataset\",\"doc\":\"Sample HDFS dataset\",\"fields\":[{\"name\":\"field_foo\",\"type\":[\"string\"]},{\"name\":\"field_bar\",\"type\":[\"boolean\"]}]}" + } + }, + "lastModified": { + "actor": "urn:li:corpuser:jdoe", + "time": 1581407189000 + }, + "schemaName": "SampleHdfsSchema", + "fields": [ + { + "nullable": false, + "fieldPath": "shipment_info", + "description": "Shipment info description", + "isPartOfKey": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "nullable": false, + "fieldPath": "shipment_info.date", + "description": "Shipment info date description", + "isPartOfKey": false, + "type": { + "type": { + "com.linkedin.schema.DateType": {} + } + }, + "nativeDataType": "Date", + "recursive": false + }, + { + "nullable": false, + "fieldPath": "shipment_info.target", + "description": "Shipment info target description", + "isPartOfKey": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false + }, + { + "nullable": false, + "fieldPath": "shipment_info.destination", + "description": "Shipment info destination description", + "isPartOfKey": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "nullable": false, + "fieldPath": "shipment_info.geo_info", + "description": "Shipment info geo_info description", + "isPartOfKey": false, + "type": { + "type": { + "com.linkedin.schema.RecordType": {} + } + }, + "nativeDataType": "varchar(100)", + "recursive": false + }, + { + "nullable": false, + "fieldPath": "shipment_info.geo_info.lat", + "description": "Shipment info geo_info lat", + "isPartOfKey": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + }, + { + "nullable": false, + "fieldPath": "shipment_info.geo_info.lng", + "description": "Shipment info geo_info lng", + "isPartOfKey": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "float", + "recursive": false + } + ], + "version": 0, + "hash": "", + "platform": "urn:li:dataPlatform:hdfs" + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + }, + "upstreamLineage": { + "name": "upstreamLineage", + "version": 0, + "value": { + "upstreams": [ + { + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1581407189000 + }, + "type": "TRANSFORMED", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)" + } + ] + }, + "created": { + "actor": "urn:li:corpuser:UNKNOWN", + "time": 1646245614843 + } + } + }, + "entityName": "dataset" +} +``` + +You can also optionally limit to specific aspects using the `aspects` query parameter: + +``` +curl 'http://localhost:8080/entitiesV2/?aspects=List(upstreamLineage)' +``` + +#### Retrieving Entities (Legacy) + +> Note that this method of retrieving entities is deprecated, as it uses the legacy Snapshot models. Please refer to the **Retriving Entity Aspects** section above for the +> latest guidance. + +The Entity Snapshot Get APIs allow to retrieve the latest version of each aspect associated with an Entity. + +In general, when reading entities by primary key (urn), you will use the general-purpose `entities` endpoints. To fetch by primary key (urn), you'll +issue a query of the following form: + +``` +curl 'http://localhost:8080/entities/' +``` + +##### Get a CorpUser + +``` +curl 'http://localhost:8080/entities/urn%3Ali%3Acorpuser%3Afbar' + +{ + "value":{ + "com.linkedin.metadata.snapshot.CorpUserSnapshot":{ + "urn":"urn:li:corpuser:fbar", + "aspects":[ + { + "com.linkedin.metadata.key.CorpUserKey":{ + "username":"fbar" + } + }, + { + "com.linkedin.identity.CorpUserInfo":{ + "active":true, + "fullName":"Foo Bar", + "displayName":"Foo Bar", + "email":"fbar@linkedin.com" + } + }, + { + "com.linkedin.identity.CorpUserEditableInfo":{ + + } + } + ] + } + } +} +``` + +##### Get a CorpGroup + +``` +curl 'http://localhost:8080/entities/urn%3Ali%3AcorpGroup%3Adev' + +{ + "value":{ + "com.linkedin.metadata.snapshot.CorpGroupSnapshot":{ + "urn":"urn:li:corpGroup:dev", + "aspects":[ + { + "com.linkedin.metadata.key.CorpGroupKey":{ + "name":"dev" + } + }, + { + "com.linkedin.identity.CorpGroupInfo":{ + "groups":[ + + ], + "email":"dev@linkedin.com", + "admins":[ + "urn:li:corpUser:jdoe" + ], + "members":[ + "urn:li:corpUser:datahub", + "urn:li:corpUser:jdoe" + ] + } + } + ] + } + } +} +``` + +##### Get a Dataset + +``` +curl 'http://localhost:8080/entities/urn%3Ali%3Adataset%3A(urn%3Ali%3AdataPlatform%3Afoo,bar,PROD)' + +{ + "value":{ + "com.linkedin.metadata.snapshot.DatasetSnapshot":{ + "urn":"urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)", + "aspects":[ + { + "com.linkedin.metadata.key.DatasetKey":{ + "origin":"PROD", + "name":"bar", + "platform":"urn:li:dataPlatform:foo" + } + }, + { + "com.linkedin.common.InstitutionalMemory":{ + "elements":[ + { + "createStamp":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "description":"Sample doc", + "url":"https://www.linkedin.com" + } + ] + } + }, + { + "com.linkedin.common.Ownership":{ + "owners":[ + { + "owner":"urn:li:corpuser:fbar", + "type":"DATAOWNER" + } + ], + "lastModified":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + } + } + }, + { + "com.linkedin.schema.SchemaMetadata":{ + "created":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "platformSchema":{ + "com.linkedin.schema.KafkaSchema":{ + "documentSchema":"{\"type\":\"record\",\"name\":\"MetadataChangeEvent\",\"namespace\":\"com.linkedin.mxe\",\"doc\":\"Kafka event for proposing a metadata change for an entity.\",\"fields\":[{\"name\":\"auditHeader\",\"type\":{\"type\":\"record\",\"name\":\"KafkaAuditHeader\",\"namespace\":\"com.linkedin.avro2pegasus.events\",\"doc\":\"Header\"}}]}" + } + }, + "lastModified":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "schemaName":"FooEvent", + "fields":[ + { + "fieldPath":"foo", + "description":"Bar", + "type":{ + "type":{ + "com.linkedin.schema.StringType":{ + + } + } + }, + "nativeDataType":"string" + } + ], + "version":0, + "hash":"", + "platform":"urn:li:dataPlatform:foo" + } + }, + { + "com.linkedin.common.BrowsePaths":{ + "paths":[ + "/prod/foo/bar" + ] + } + }, + { + "com.linkedin.dataset.UpstreamLineage":{ + "upstreams":[ + { + "auditStamp":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "type":"TRANSFORMED", + "dataset":"urn:li:dataset:(urn:li:dataPlatform:foo,barUp,PROD)" + } + ] + } + } + ] + } + } +} +``` + +##### Get a Chart + +``` +curl 'http://localhost:8080/entities/urn%3Ali%3Achart%3A(looker,baz1)' + +{ + "value":{ + "com.linkedin.metadata.snapshot.ChartSnapshot":{ + "urn":"urn:li:chart:(looker,baz1)", + "aspects":[ + { + "com.linkedin.metadata.key.ChartKey":{ + "chartId":"baz1", + "dashboardTool":"looker" + } + }, + { + "com.linkedin.common.BrowsePaths":{ + "paths":[ + "/looker/baz1" + ] + } + }, + { + "com.linkedin.chart.ChartInfo":{ + "description":"Baz Chart 1", + "lastModified":{ + "created":{ + "actor":"urn:li:corpuser:jdoe", + "time":0 + }, + "lastModified":{ + "actor":"urn:li:corpuser:datahub", + "time":0 + } + }, + "title":"Baz Chart 1", + "inputs":[ + { + "string":"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" + } + ] + } + } + ] + } + } +} +``` + +##### Get a Dashboard + +``` +curl 'http://localhost:8080/entities/urn%3Ali%3Adashboard%3A(looker,foo)' + +{ + "value":{ + "com.linkedin.metadata.snapshot.DashboardSnapshot":{ + "urn":"urn:li:dashboard:(looker,foo)", + "aspects":[ + { + "com.linkedin.metadata.key.DashboardKey":{ + "dashboardId":"foo", + "dashboardTool":"looker" + } + } + ] + } + } +} +``` + +##### Get a GlossaryTerm + +``` +curl 'http://localhost:8080/entities/urn%3Ali%3AglossaryTerm%3A(instruments,instruments.FinancialInstrument_v1)' +{ + "value":{ + "com.linkedin.metadata.snapshot.GlossaryTermSnapshot":{ + "urn":"urn:li:glossaryTerm:instruments.FinancialInstrument_v1", + "ownership":{ + "owners":[ + { + "owner":"urn:li:corpuser:jdoe", + "type":"DATAOWNER" + } + ], + "lastModified":{ + "actor":"urn:li:corpuser:jdoe", + "time":1581407189000 + } + }, + "glossaryTermInfo":{ + "definition":"written contract that gives rise to both a financial asset of one entity and a financial liability of another entity", + "customProperties":{ + "FQDN":"full" + }, + "sourceRef":"FIBO", + "sourceUrl":"https://spec.edmcouncil.org/fibo/ontology/FBC/FinancialInstruments/FinancialInstruments/FinancialInstrument", + "termSource":"EXTERNAL" + } + } + } +} +``` + +##### Browse an Entity + +To browse (explore) for an Entity of a particular type (e.g. dataset, chart, etc), you can use the following query format: + +``` +curl -X POST 'http://localhost:8080/entities?action=browse' \ +--data '{ + "path": "", + "entity": "", + "start": 0, + "limit": 10 +}' +``` + +For example, to browse the "charts" entity, you could use the following query: + +``` +curl -X POST 'http://localhost:8080/entities?action=browse' \ +--data '{ + "path": "/looker", + "entity": "chart", + "start": 0, + "limit": 10 +}' + +{ + "value":{ + "numEntities":1, + "pageSize":1, + "metadata":{ + "totalNumEntities":1, + "groups":[ + + ], + "path":"/looker" + }, + "from":0, + "entities":[ + { + "name":"baz1", + "urn":"urn:li:chart:(looker,baz1)" + } + ] + } +} +``` + +##### Search an Entity + +To search for an Entity of a particular type (e.g. dataset, chart, etc), you can use the following query format: + +``` +curl -X POST 'http://localhost:8080/entities?action=search' \ +--data '{ + "input": "", + "entity": "", + "start": 0, + "count": 10 +}' +``` + +The API will return a list of URNs that matched your search query. + +For example, to search the "charts" entity, you could use the following query: + +``` +curl -X POST 'http://localhost:8080/entities?action=search' \ +--data '{ + "input": "looker", + "entity": "chart", + "start": 0, + "count": 10 +}' + +{ + "value":{ + "numEntities":1, + "pageSize":10, + "metadata":{ + "urns":[ + "urn:li:chart:(looker,baz1)" + ], + "matches":[ + { + "matchedFields":[ + { + "name":"tool", + "value":"looker" + } + ] + } + ], + "searchResultMetadatas":[ + { + "name":"tool", + "aggregations":{ + "looker":1 + } + } + ] + }, + "from":0, + "entities":[ + "urn:li:chart:(looker,baz1)" + ] + } +} +``` + +###### Exact Match Search + +You can use colon search for exact match searching on particular @Searchable fields of an Entity. + +###### Example: Find assets by Tag + +For example, to fetch all Datasets having a particular tag (Engineering), we can use the following query: + +``` +curl -X POST 'http://localhost:8080/entities?action=search' \ +--data '{ + "input": "tags:Engineering", + "entity": "dataset", + "start": 0, + "count": 10 +}' + +{ + "value":{ + "numEntities":1, + "pageSize":10, + "metadata":{ + "urns":[ + "urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)" + ], + "matches":[ + { + "matchedFields":[ + { + "name":"tags", + "value":"urn:li:tag:Engineering" + } + ] + } + ], + "searchResultMetadatas":[ + { + "name":"platform", + "aggregations":{ + "foo":1 + } + }, + { + "name":"origin", + "aggregations":{ + "PROD":1 + } + } + ] + }, + "from":0, + "entities":[ + "urn:li:dataset:(urn:li:dataPlatform:foo,bar,PROD)" + ] + } +} +``` + +###### Filtering + +In addition to performing full-text search, you can also filter explicitly against fields marked as @Searchable in the corresponding aspect PDLs. + +For example, to perform filtering for a chart with title "Baz Chart 1", you could issue the following query: + +``` +curl -X POST 'http://localhost:8080/entities?action=search' \ +--data '{ + "input": "looker", + "entity": "chart", + "start": 0, + "count": 10, + "filter": { + "or": [{ + "and": [ + { + "field": "title", + "values": ["Baz Chart 1"], + "condition": "EQUAL" + } + ] + }] + } +}' + +{ + "value":{ + "numEntities":1, + "pageSize":10, + "metadata":{ + "urns":[ + "urn:li:chart:(looker,baz1)" + ], + "matches":[ + { + "matchedFields":[ + { + "name":"tool", + "value":"looker" + } + ] + } + ], + "searchResultMetadatas":[ + { + "name":"tool", + "aggregations":{ + "looker":1 + } + } + ] + }, + "from":0, + "entities":[ + "urn:li:chart:(looker,baz1)" + ] + } +} +``` + +where valid conditions include - CONTAIN - END_WITH - EQUAL - IEQUAL (Supports case insensitive equals) - GREATER_THAN - GREATER_THAN_OR_EQUAL_TO - LESS_THAN - LESS_THAN_OR_EQUAL_TO - START_WITH + +\*Note that the search API only includes data corresponding to the latest snapshots of a particular Entity. + +##### Autocomplete against fields of an entity + +To autocomplete a query for a particular entity type, you can use a query of the following form: + +``` +curl -X POST 'http://localhost:8080/entities?action=autocomplete' \ +--data '{ + "query": "", + "entity": "", + "limit": 10 +}' +``` + +For example, to autocomplete a query against all Dataset entities, you could issue the following: + +``` +curl -X POST 'http://localhost:8080/entities?action=autocomplete' \ +--data '{ + "query": "Baz Ch", + "entity": "chart", + "start": 0, + "limit": 10 +}' + +{ + "value":{ + "suggestions":[ + "Baz Chart 1" + ], + "query":"Baz Ch" + } +} +``` + +Note that you can also provide a `Filter` to the autocomplete endpoint: + +``` +curl -X POST 'http://localhost:8080/entities?action=autocomplete' \ +--data '{ + "query": "Baz C", + "entity": "chart", + "start": 0, + "limit": 10, + "filter": { + "or": [{ + "and": [ + { + "field": "tool", + "values": ["looker"], + "condition": "EQUAL" + } + ] + }] + } +}' + +{ + "value":{ + "suggestions":[ + "Baz Chart 1" + ], + "query":"Baz Ch" + } +} +``` + +\*Note that the autocomplete API only includes data corresponding to the latest snapshots of a particular Entity. + +##### Get a Versioned Aspect + +In addition to fetching the set of latest Snapshot aspects for an entity, we also support doing a point lookup of an entity at a particular version. + +To do so, you can use the following query template: + +``` +curl 'http://localhost:8080/aspects/?aspect=&version= +``` + +Which will return a VersionedAspect, which is a record containing a version and an aspect inside a Rest.li Union, wherein the fully-qualified record name of the +aspect is the key for the union. + +For example, to fetch the latest version of a Dataset's "schemaMetadata" aspect, you could issue the following query: + +``` +curl 'http://localhost:8080/aspects/urn%3Ali%3Adataset%3A(urn%3Ali%3AdataPlatform%3Afoo%2Cbar%2CPROD)?aspect=schemaMetadata&version=0' + +{ + "version":0, + "aspect":{ + "com.linkedin.schema.SchemaMetadata":{ + "created":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "platformSchema":{ + "com.linkedin.schema.KafkaSchema":{ + "documentSchema":"{\"type\":\"record\",\"name\":\"MetadataChangeEvent\",\"namespace\":\"com.linkedin.mxe\",\"doc\":\"Kafka event for proposing a metadata change for an entity.\",\"fields\":[{\"name\":\"auditHeader\",\"type\":{\"type\":\"record\",\"name\":\"KafkaAuditHeader\",\"namespace\":\"com.linkedin.avro2pegasus.events\",\"doc\":\"Header\"}}]}" + } + }, + "lastModified":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "schemaName":"FooEvent", + "fields":[ + { + "fieldPath":"foo", + "description":"Bar", + "type":{ + "type":{ + "com.linkedin.schema.StringType":{ + + } + } + }, + "nativeDataType":"string" + } + ], + "version":0, + "hash":"", + "platform":"urn:li:dataPlatform:foo" + } + } +} +``` + +Keep in mind that versions increase monotonically _after_ version 0, which represents the latest. + +Note that this API will soon be deprecated and replaced by the V2 Aspect API, discussed below. + +##### Get a range of Versioned Aspects + +_Coming Soon_! + +##### Get a range of Timeseries Aspects + +With the introduction of Timeseries Aspects, we've introduced a new API for fetching a series of aspects falling into a particular time range. For this, you'll +use the `/aspects` endpoint. The V2 APIs are unique in that they return a new type of payload: an "Enveloped Aspect". This is essentially a serialized aspect along with +some system metadata. The serialized aspect can be in any form, though we currently default to escaped Rest.li-compatible JSON. + +Callers of the V2 Aspect APIs will be expected to deserialize the aspect payload in the way they see fit. For example, they may bind the deserialized JSON object +into a strongly typed Rest.li RecordTemplate class (which is what datahub-frontend does). The benefit of doing it this way is thaet we remove the necessity to +use Rest.li Unions to represent an object which can take on multiple payload forms. It also makes adding and removing aspects from the model easier, a process +which could theoretically be done at runtime as opposed to at deploy time. + +To fetch a set of Timeseries Aspects that fall into a particular time range, you can use the following query template: + +``` +curl -X POST 'http://localhost:8080/aspects?action=getTimeseriesAspectValues' \ +--data '{ + "urn": "", + "entity": "", + "aspect": "", + "startTimeMillis": "", + "endTimeMillis": "" +}' +``` + +For example, to fetch "datasetProfile" timeseries aspects for a dataset with urn `urn:li:dataset:(urn:li:dataPlatform:foo,barUp,PROD)` +that were reported after July 26, 2021 and before July 28, 2021, you could issue the following query: + +``` +curl -X POST 'http://localhost:8080/aspects?action=getTimeseriesAspectValues' \ +--data '{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:redshift,global_dev.larxynx_carcinoma_data_2020,PROD)", + "entity": "dataset", + "aspect": "datasetProfile", + "startTimeMillis": 1625122800000, + "endTimeMillis": 1627455600000 +}' + +{ + "value":{ + "limit":2000, + "aspectName":"datasetProfile", + "endTimeMillis":1627455600000, + "startTimeMillis":1625122800000, + "entityName":"dataset", + "values":[ + { + "aspect":{ + "value":"{\"timestampMillis\":1626912000000,\"fieldProfiles\":[{\"uniqueProportion\":1.0,\"sampleValues\":[\"123MMKK12\",\"13KDFMKML\",\"123NNJJJL\"],\"fieldPath\":\"id\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":3742},{\"uniqueProportion\":1.0,\"min\":\"1524406400000\",\"max\":\"1624406400000\",\"sampleValues\":[\"1640023230002\",\"1640343012207\",\"16303412330117\"],\"mean\":\"1555406400000\",\"fieldPath\":\"date\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":3742},{\"uniqueProportion\":0.037,\"min\":\"21\",\"median\":\"68\",\"max\":\"92\",\"sampleValues\":[\"45\",\"65\",\"81\"],\"mean\":\"65\",\"distinctValueFrequencies\":[{\"value\":\"12\",\"frequency\":103},{\"value\":\"54\",\"frequency\":12}],\"fieldPath\":\"patient_age\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":79},{\"uniqueProportion\":0.00820873786407767,\"sampleValues\":[\"male\",\"female\"],\"fieldPath\":\"patient_gender\",\"nullCount\":120,\"nullProportion\":0.03,\"uniqueCount\":2}],\"rowCount\":3742,\"columnCount\":4}", + "contentType":"application/json" + } + }, + ] + } +} +``` + +You'll notice that in this API (V2), we return a generic serialized aspect string as opposed to an inlined Rest.li-serialized Snapshot Model. + +This is part of an initiative to move from MCE + MAE to MetadataChangeProposal and MetadataChangeLog. For more information, see [this doc](docs/advanced/mcp-mcl.md). + +##### Get Relationships (Edges) + +To get relationships between entities, you can use the `/relationships` API. Do do so, you must provide the following inputs: + +1. Urn of the source node +2. Direction of the edge (INCOMING, OUTGOING) +3. The name of the Relationship (This can be found in Aspect PDLs within the @Relationship annotation) + +For example, to get all entities owned by `urn:li:corpuser:fbar`, we could issue the following query: + +``` +curl 'http://localhost:8080/relationships?direction=INCOMING&urn=urn%3Ali%3Acorpuser%3Auser1&types=OwnedBy' +``` + +which will return a list of urns, representing entities on the other side of the relationship: + +``` +{ + "entities":[ + urn:li:dataset:(urn:li:dataPlatform:foo,barUp,PROD) + ] +} +``` + +## FAQ + +_1. How do I find the valid set of Entity names?_ + +Entities are named inside of PDL schemas. Each entity will be annotated with the @Entity annotation, which will include a "name" field inside. +This represents the "common name" for the entity which can be used in browsing, searching, and more. By default, DataHub ships with the following entities: + +By convention, all entity PDLs live under `metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot` + +_2. How do I find the valid set of Aspect names?_ + +Aspects are named inside of PDL schemas. Each aspect will be annotated with the @Aspect annotation, which will include a "name" field inside. +This represents the "common name" for the entity which can be used in browsing, searching, and more. + +By convention, all entity PDLs live under `metadata-models/src/main/pegasus/com/linkedin/metadata/common` or `metadata-models/src/main/pegasus/com/linkedin/metadata/`. For example, +the dataset-specific aspects are located under `metadata-models/src/main/pegasus/com/linkedin/metadata/dataset`. + +_3. How do I find the valid set of Relationship names?_ + +All relationships are defined on foreign-key fields inside Aspect PDLs. They are reflected by fields bearing the @Relationship annotation. Inside this annotation +is a "name" field that defines the standardized name of the Relationship to be used when querying. + +By convention, all entity PDLs live under `metadata-models/src/main/pegasus/com/linkedin/metadata/common` or `metadata-models/src/main/pegasus/com/linkedin/metadata/`. For example, +the dataset-specific aspects are located under `metadata-models/src/main/pegasus/com/linkedin/metadata/dataset`. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restore-indices.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restore-indices.md new file mode 100644 index 00000000..2cc44ec5 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/restore-indices.md @@ -0,0 +1,35 @@ +--- +title: Restore Indices Endpoint +slug: /api/restli/restore-indices +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/restli/restore-indices.md +--- +# Restore Indices Endpoint + +This is reference material for the REST.li `restoreIndices` endpoint. For general information on reindexing and restoring indices, see [Restore Indices](../../how/restore-indices.md). + +You can do a HTTP POST request to `/gms/operations?action=restoreIndices` endpoint with the `urn` as part of JSON Payload to restore indices for the particular URN, or with the `urnLike` regex to restore for `batchSize` URNs matching the pattern starting from `start`. + +``` +curl --location --request POST 'https://demo.datahub.com/api/gms/operations?action=restoreIndices' \ +--header 'Authorization: Bearer TOKEN' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "urn": "YOUR_URN" +}' + +curl --location --request POST 'https://demo.datahub.com/api/gms/operations?action=restoreIndices' \ +--header 'Authorization: Bearer TOKEN' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "urnLike": "urn:dataPlatform:%" +}' +``` + +The supported parameters are + +- `urn` - Optional URN string +- `aspect` - Optional Aspect string +- `urnLike` - Optional string regex to match URNs +- `start` - Optional integer to decide which rows number of sql store to restore. Default: 0 +- `batchSize` - Optional integer to decide how many rows to restore. Default: 10 diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/truncate-time-series-aspect.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/truncate-time-series-aspect.md new file mode 100644 index 00000000..193e5cce --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/restli/truncate-time-series-aspect.md @@ -0,0 +1,54 @@ +--- +title: Truncate Timeseries Index Endpoint +slug: /api/restli/truncate-time-series-aspect +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/restli/truncate-time-series-aspect.md +--- +# Truncate Timeseries Index Endpoint + +You can do a HTTP POST request to `/gms/operations?action=truncateTimeseriesAspect` endpoint to manage the size of a time series index by removing entries older than a certain timestamp, thereby truncating the table to only the entries needed, to save space. The `getIndexSizes` endpoint can be used to identify the largest indices. The output includes the index parameters needed for this function. + +``` +curl --location --request POST 'https://demo.datahub.com/api/gms/operations?action=truncateTimeseriesAspect' \ +--header 'Authorization: Bearer TOKEN' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "entityType": "YOUR_ENTITY_TYPE", + "aspect": "YOUR_ASPECT_NAME", + "endTimeMillis": 1000000000000 +}' + +curl --location --request POST 'https://demo.datahub.com/api/gms/operations?action=truncateTimeseriesAspect' \ +--header 'Authorization: Bearer TOKEN' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "entityType": "YOUR_ENTITY_TYPE", + "aspect": "YOUR_ASPECT_NAME", + "endTimeMillis": 1000000000000, + "dryRun": false, + "batchSize": 100, + "timeoutSeconds": 3600 +}' +``` + +The supported parameters are + +- `entityType` - Required type of the entity to truncate the index of, for example, `dataset`. +- `aspect` - Required name of the aspect to truncate the index of, for example, `datasetusagestatistics`. A call to `getIndexSizes` shows the `entityType` and `aspect` parameters for each index along with its size. +- `endTimeMillis` - Required timestamp to truncate the index to. Entities with timestamps older than this time will be deleted. +- `dryRun` - Optional boolean to enable/disable dry run functionality. Default: true. In a dry run, the following information will be printed: + +``` +{"value":"Delete 0 out of 201 rows (0.00%). Reindexing the aspect without the deleted records. This was a dry run. Run with dryRun = false to execute."} +``` + +- `batchSize` - Optional integer to control the batch size for the deletion. Default: 10000 +- `timeoutSeconds` - Optional integer to set a timeout for the delete operation. Default: No timeout set + +The output to the call will be information about how many rows would be deleted and how to proceed for a dry run: + +``` +{"value":"Delete 0 out of 201 rows (0.00%). Reindexing the aspect without the deleted records. This was a dry run. Run with dryRun = false to execute."} +``` + +For a non-dry-run, the output will be the Task ID of the asynchronous delete operation. This task ID can be used to monitor the status of the operation. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/applications.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/applications.md new file mode 100644 index 00000000..d900720f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/applications.md @@ -0,0 +1,356 @@ +--- +title: Applications +slug: /api/tutorials/applications +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/applications.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Applications + +## Why Would You Use Applications? + +Applications are groupings of assets based on a particular purpose, similar to domains and data products. +For more information on what an Application is, and how it differs from other concepts, refer to [About DataHub Applications](/docs/features/feature-guides/applications.md). + +### Goal Of This Guide + +This guide will show you how to + +- Create an application. +- Read the application attached to a dataset. +- Add a dataset to an application +- Remove the application from a dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create Application + + + + +```json +mutation createApplication { + createApplication( + input: { + properties: { + name: "My New Application" + description: "An optional description" + } + } + ) +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "createApplication": "" + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/application_create_full.py +#!/usr/bin/env python3 + +import os + +from datahub.emitter.mce_builder import make_dataset_urn, make_user_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.metadata.schema_classes import ( + ApplicationPropertiesClass, + ApplicationsClass, + DomainsClass, + GlobalTagsClass, + OwnerClass, + OwnershipClass, + OwnershipTypeClass, + TagAssociationClass, +) + + +# Utility function for creating application URNs (not yet in SDK) +def make_application_urn(application_id: str) -> str: + """Create a DataHub application URN.""" + return f"urn:li:application:{application_id}" + + +def create_banking_application(): + """Complete example of creating an application and associating entities.""" + + # Initialize emitter + gms_server = os.getenv("DATAHUB_GMS_URL", "http://localhost:8080") + token = os.getenv("DATAHUB_GMS_TOKEN") + emitter = DatahubRestEmitter(gms_server=gms_server, token=token) + + try: + # Create application + application_urn = make_application_urn("banking-app") + + # 1. Application Properties + application_properties = ApplicationPropertiesClass( + name="Banking Application", + description="Core banking application handling customer accounts and transactions.", + ) + + emitter.emit( + MetadataChangeProposalWrapper( + entityUrn=application_urn, aspect=application_properties + ) + ) + + # 2. Ownership + ownership = OwnershipClass( + owners=[ + OwnerClass( + owner=make_user_urn("john.smith@company.com"), + type=OwnershipTypeClass.TECHNICAL_OWNER, + ), + OwnerClass( + owner=make_user_urn("jane.doe@company.com"), + type=OwnershipTypeClass.BUSINESS_OWNER, + ), + ] + ) + emitter.emit( + MetadataChangeProposalWrapper(entityUrn=application_urn, aspect=ownership) + ) + + # 3. Tags and Domain + tags = GlobalTagsClass(tags=[TagAssociationClass(tag="urn:li:tag:Production")]) + emitter.emit( + MetadataChangeProposalWrapper(entityUrn=application_urn, aspect=tags) + ) + + domains = DomainsClass(domains=["urn:li:domain:finance-domain"]) + emitter.emit( + MetadataChangeProposalWrapper(entityUrn=application_urn, aspect=domains) + ) + + # 4. Associate datasets + datasets = ["customer_accounts", "transaction_history", "account_balances"] + + for dataset_name in datasets: + dataset_urn = make_dataset_urn("hive", dataset_name, "PROD") + applications_aspect = ApplicationsClass(applications=[application_urn]) + + emitter.emit( + MetadataChangeProposalWrapper( + entityUrn=dataset_urn, aspect=applications_aspect + ) + ) + + print(f"Successfully created application: {application_urn}") + print(f"Associated {len(datasets)} datasets with the application") + + finally: + emitter.close() + + +if __name__ == "__main__": + create_banking_application() + +``` + + + + +### Expected Outcomes of Creating Applications + +You can now see the applications under `Applications` sidebar section. + +

+ +

+ +## Read Applications + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)") { + application { + application { + urn + properties { + name + description + } + } + } + } +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "dataset": { + "application": { + "application": { + "urn": "urn:li:application:71b3bf7b-2e3f-4686-bfe1-93172c8c4e10", + "properties": { + "name": "Cancellation Processing" + } + } + } + } + }, + "extensions": {} +} +``` + + + + +## Add Application + + + + +```json +mutation batchSetApplication { + batchSetApplication( + input: { + resourceUrns: [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,banking.public.customer,PROD)" + ] + applicationUrn: "urn:li:application:new-customer-signup" + } + ) +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "batchSetApplication": true + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/application_add.py +#!/usr/bin/env python3 +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.metadata.schema_classes import ApplicationsClass + + +# Utility function for creating application URNs (not yet in SDK) +def make_application_urn(application_id: str) -> str: + """Create a DataHub application URN.""" + return f"urn:li:application:{application_id}" + + +def add_application_aspect(): + emitter = DatahubRestEmitter(gms_server="http://localhost:8080", token="") + dataset_urn = make_dataset_urn("snowflake", "database.schema.table", "PROD") + + application_urn = make_application_urn("my_application") + applications_aspect = ApplicationsClass(applications=[application_urn]) + + emitter.emit( + MetadataChangeProposalWrapper(entityUrn=dataset_urn, aspect=applications_aspect) + ) + + print(f"Successfully added application: {application_urn}") + + +if __name__ == "__main__": + add_application_aspect() + +``` + + + + +### Expected Outcomes of Adding Application + +You can now see the application has been added to the dataset. + +## Remove Applications + + + + +```json +mutation batchSetApplication { + batchSetApplication( + input: { + resourceUrns: [ + "urn:li:dataset:(urn:li:dataPlatform:bigquery,banking.public.customer,PROD)" + ], + applicationUrn: null + } + ) +} +``` + +Expected Response: + +```python +{ + "data": { + "batchSetApplication": true + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/application_remove.py +#!/usr/bin/env python3 + +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.metadata.schema_classes import ApplicationsClass + + +def remove_application_aspect(): + emitter = DatahubRestEmitter(gms_server="http://localhost:8080", token="") + dataset_urn = make_dataset_urn("snowflake", "database.schema.table", "PROD") + + applications_aspect = ApplicationsClass(applications=[]) + + emitter.emit( + MetadataChangeProposalWrapper(entityUrn=dataset_urn, aspect=applications_aspect) + ) + + print("Successfully removed application") + + +if __name__ == "__main__": + remove_application_aspect() + +``` + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/assertions.md new file mode 100644 index 00000000..e6ca45b1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/assertions.md @@ -0,0 +1,1820 @@ +--- +title: Assertions +slug: /api/tutorials/assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/assertions.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Assertions + + + +This guide specifically covers how to use the Assertion APIs for **DataHub Cloud** native assertions, including: + +- [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md) +- [Volume Assertions](/docs/managed-datahub/observe/volume-assertions.md) +- [Column Assertions](/docs/managed-datahub/observe/column-assertions.md) +- [Schema Assertions](/docs/managed-datahub/observe/schema-assertions.md) +- [Custom SQL Assertions](/docs/managed-datahub/observe/custom-sql-assertions.md) + +## Why Would You Use Assertions APIs? + +The Assertions APIs allow you to create, schedule, run, and delete Assertions with DataHub Cloud. Additionally, you can manage subscriptions to receive notifications when assertions change state or when other entity changes occur. + +### Goal Of This Guide + +This guide will show you how to create, schedule, run and delete Assertions for a Table. + +## Prerequisites + +The actor making API calls must have the `Edit Assertions` and `Edit Monitors` privileges for the Tables at hand. + +If you are using the Python examples in this guide, install the DataHub Cloud SDK extension: + +```bash +pip install acryl-datahub-cloud +``` + +## Create Assertions + +You can create new dataset Assertions to DataHub using the following APIs. + +### Freshness Assertion + + + + +To create a new freshness assertion, use the `upsertDatasetFreshnessAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + input: { + entityUrn: "" + schedule: { + type: FIXED_INTERVAL + fixedInterval: { unit: HOUR, multiple: 8 } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetFreshnessAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + + + + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +# Initialize the client +client = DataHubClient(server="", token="") + +# Create smart freshness assertion (AI-powered anomaly detection) +dataset_urn = DatasetUrn.from_string("urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)") + +smart_freshness_assertion = client.assertions.sync_smart_freshness_assertion( + dataset_urn=dataset_urn, + display_name="Smart Freshness Anomaly Monitor", + # Detection mechanism - information_schema is recommended + detection_mechanism="information_schema", + # Smart sensitivity setting + sensitivity="medium", # options: "low", "medium", "high" + # Tags for grouping + tags=["automated", "freshness", "data_quality"], + # Enable the assertion + enabled=True +) + +print(f"Created smart freshness assertion: {smart_freshness_assertion.urn}") + +# Create traditional freshness assertion (fixed interval) +freshness_assertion = client.assertions.sync_freshness_assertion( + dataset_urn=dataset_urn, + display_name="Fixed Interval Freshness Check", + # Fixed interval check - table should be updated within lookback window + freshness_schedule_check_type="fixed_interval", + # Lookback window - table should be updated within 8 hours + lookback_window={"unit": "HOUR", "multiple": 8}, + # Detection mechanism + detection_mechanism="information_schema", + # Evaluation schedule - how often to check + schedule="0 */2 * * *", # Check every 2 hours + # Tags + tags=["automated", "freshness", "fixed_interval"], + enabled=True +) + +print(f"Created freshness assertion: {freshness_assertion.urn}") + +# Create since-last-check freshness assertion +since_last_check_assertion = client.assertions.sync_freshness_assertion( + dataset_urn=dataset_urn, + display_name="Since Last Check Freshness", + # Since last check - table should be updated since the last evaluation + freshness_schedule_check_type="since_the_last_check", + # Detection mechanism with last modified column + detection_mechanism={ + "type": "last_modified_column", + "column_name": "updated_at", + "additional_filter": "status = 'active'" + }, + # Evaluation schedule - how often to check + schedule="0 */6 * * *", # Check every 6 hours + # Tags + tags=["automated", "freshness", "since_last_check"], + enabled=True +) + +print(f"Created since last check assertion: {since_last_check_assertion.urn}") + +# Create freshness assertion with high watermark column +watermark_freshness_assertion = client.assertions.sync_freshness_assertion( + dataset_urn=dataset_urn, + display_name="High Watermark Freshness Check", + # Fixed interval check with specific lookback window + freshness_schedule_check_type="fixed_interval", + # Lookback window - check for updates in the last 24 hours + lookback_window={"unit": "DAY", "multiple": 1}, + # Detection mechanism using high watermark column (e.g., auto-incrementing ID) + detection_mechanism={ + "type": "high_watermark_column", + "column_name": "id", + "additional_filter": "status != 'deleted'" + }, + # Evaluation schedule + schedule="0 8 * * *", # Check daily at 8 AM + # Tags + tags=["automated", "freshness", "high_watermark"], + enabled=True +) + +print(f"Created watermark freshness assertion: {watermark_freshness_assertion.urn}") +``` + + + + +For more details, see the [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md) guide. + +### Volume Assertions + + + + +To create a new volume assertion, use the `upsertDatasetVolumeAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetVolumeAssertionMonitor { + upsertDatasetVolumeAssertionMonitor( + input: { + entityUrn: "" + type: ROW_COUNT_TOTAL + rowCountTotal: { + operator: BETWEEN + parameters: { + minValue: { value: "10", type: NUMBER } + maxValue: { value: "20", type: NUMBER } + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetVolumeAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + + + + + + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +# Initialize the client +client = DataHubClient(server="", token="") + +# Create smart volume assertion (AI-powered anomaly detection) +dataset_urn = DatasetUrn.from_string("urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)") + +smart_volume_assertion = client.assertions.sync_smart_volume_assertion( + dataset_urn=dataset_urn, + display_name="Smart Volume Check", + # Detection mechanism options + detection_mechanism="information_schema", + # Smart sensitivity setting + sensitivity="medium", # options: "low", "medium", "high" + # Tags for grouping + tags=["automated", "volume", "data_quality"], + # Optional: partition data into daily/weekly time buckets + time_bucketing_strategy={ + "timestamp_field_path": "created_at", + "bucket_interval": {"unit": "DAY", "multiple": 1}, + "timezone": "America/Los_Angeles", + }, + # Optional: backfill historical data for smarter predictions + backfill_config={"backfill_start_date_ms": 1704067200000}, # 2024-01-01 + # Enable the assertion + enabled=True +) + +print(f"Created smart volume assertion: {smart_volume_assertion.urn}") + +# Create traditional volume assertion (fixed threshold range) +volume_assertion = client.assertions.sync_volume_assertion( + dataset_urn=dataset_urn, + display_name="Row Count Range Check", + criteria_condition="ROW_COUNT_IS_WITHIN_A_RANGE", + criteria_parameters=(1000, 10000), # Between 1000 and 10000 rows + # Detection mechanism + detection_mechanism="information_schema", + # Evaluation schedule + schedule="0 */4 * * *", # Every 4 hours + # Tags + tags=["automated", "volume", "threshold_check"], + enabled=True +) + +print(f"Created volume assertion: {volume_assertion.urn}") + +# Example with single threshold +min_volume_assertion = client.assertions.sync_volume_assertion( + dataset_urn=dataset_urn, + display_name="Minimum Row Count Check", + criteria_condition="ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO", + criteria_parameters=500, # At least 500 rows + detection_mechanism="information_schema", + schedule="0 */2 * * *", # Every 2 hours + tags=["automated", "volume", "minimum_check"], + enabled=True +) + +print(f"Created minimum volume assertion: {min_volume_assertion.urn}") + +# Example with growth-based assertion +growth_volume_assertion = client.assertions.sync_volume_assertion( + dataset_urn=dataset_urn, + display_name="Daily Growth Check", + criteria_condition="ROW_COUNT_GROWS_BY_AT_MOST_ABSOLUTE", + criteria_parameters=1000, # Grows by at most 1000 rows between checks + detection_mechanism="information_schema", + schedule="0 6 * * *", # Daily at 6 AM + tags=["automated", "volume", "growth_check"], + enabled=True +) + +print(f"Created growth volume assertion: {growth_volume_assertion.urn}") +``` + + + + +For more details, see the [Volume Assertions](/docs/managed-datahub/observe/volume-assertions.md) guide. + +### Column Assertions + + + + +To create a new column assertion, use the `upsertDatasetFieldAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetFieldAssertionMonitor { + upsertDatasetFieldAssertionMonitor( + input: { + entityUrn: "" + type: FIELD_VALUES + fieldValuesAssertion: { + field: { + path: "" + type: "NUMBER" + nativeType: "NUMBER(38,0)" + } + operator: GREATER_THAN + parameters: { value: { type: NUMBER, value: "10" } } + failThreshold: { type: COUNT, value: 0 } + excludeNulls: true + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: ALL_ROWS_QUERY } + mode: ACTIVE + } + ) { + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetFieldAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + + + + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +# Initialize the client +client = DataHubClient(server="", token="") + +# Create smart column metric assertion (AI-powered anomaly detection). +# Note: Smart Assertions currently only support the following column metrics: +# null_count, unique_count, empty_count, zero_count, negative_count. +# For other metrics (e.g. min, max, mean), use a regular column metric assertion +# with a fixed threshold (see below). +dataset_urn = DatasetUrn.from_string("urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)") + +smart_column_assertion = client.assertions.sync_smart_column_metric_assertion( + dataset_urn=dataset_urn, + column_name="user_id", + metric_type="null_count", + display_name="Smart Null Count Check - user_id", + # Detection mechanism for column metrics + detection_mechanism="all_rows_query_datahub_dataset_profile", + # Smart sensitivity setting + sensitivity="medium", # options: "low", "medium", "high" + # Optional: partition data into daily time buckets + time_bucketing_strategy={ + "timestamp_field_path": "created_at", + "bucket_interval": {"unit": "DAY", "multiple": 1}, + "timezone": "UTC", + }, + # Optional: backfill historical data for smarter predictions + backfill_config={"backfill_start_date_ms": 1704067200000}, # 2024-01-01 + # Tags + tags=["automated", "column_quality", "null_checks"], + enabled=True +) + +print(f"Created smart column assertion: {smart_column_assertion.urn}") + +# Create regular column metric assertion (fixed threshold on aggregated metric). +# Use this for metrics not supported by Smart Assertions (e.g. min, max, mean, median, stddev). +column_metric_assertion = client.assertions.sync_column_metric_assertion( + dataset_urn=dataset_urn, + column_name="price", + metric_type="min", + operator="greater_than_or_equal_to", + criteria_parameters=0, + display_name="Price Minimum Check", + # Evaluation schedule + schedule="0 */4 * * *", # Every 4 hours + # Tags + tags=["automated", "column_quality", "price_validation"], + enabled=True +) + +print(f"Created column metric assertion: {column_metric_assertion.urn}") + +# ---------------------------- +# Column value assertions (row-level checks) +# ---------------------------- + +# Example 1: Simple email validation with regex pattern +# Validates that all email values match a valid email format +email_regex_assertion = client.assertions.sync_column_value_assertion( + dataset_urn=dataset_urn, + column_name="email", + operator="regex_match", + criteria_parameters=r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$" +) + +print(f"Created email regex assertion: {email_regex_assertion.urn}") + +# Example 2: Detailed column value assertion with all parameters +# Validates individual row values in a column against semantic constraints +column_value_assertion = client.assertions.sync_column_value_assertion( + dataset_urn=dataset_urn, + column_name="quantity", + display_name="Quantity Positive Check", + # Operator applied to each row's value (e.g., "greater_than", "between", "regex_match", "not_null") + operator="greater_than", + criteria_parameters=0, # Each quantity must be > 0 + # Optional: Apply a transform before validation (currently supports "length" for strings) + # transform="length", + # Fail threshold configuration + fail_threshold_type="count", # How to count failures: "count" (absolute number) or "percentage" + fail_threshold_value=0, # Assertion fails if this many rows fail (0 = zero tolerance) + # Whether to exclude null values from validation + exclude_nulls=True, + # Evaluation schedule - how often to check + schedule="0 */4 * * *", # Every 4 hours (cron format) + # Tags for grouping and categorization + tags=["automated", "column_quality", "value_validation"], + # Enable the assertion + enabled=True +) + +print(f"Created column value assertion: {column_value_assertion.urn}") +``` + + + + +For more details, see the [Column Assertions](/docs/managed-datahub/observe/column-assertions.md) guide. + +### Custom SQL Assertions + + + + +To create a new custom SQL assertion, use the `upsertDatasetSqlAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetSqlAssertionMonitor { + upsertDatasetSqlAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: METRIC + description: "" + statement: "" + operator: GREATER_THAN_OR_EQUAL_TO + parameters: { value: { value: "100", type: NUMBER } } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetSqlAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + +--- + +To create a new **smart SQL** assertion (AI anomaly detection), use the same mutation with `inferWithAI: true`. + +```graphql +mutation upsertDatasetSqlAssertionMonitor { + upsertDatasetSqlAssertionMonitor( + input: { + entityUrn: "" + type: METRIC + description: "" + statement: "" + inferWithAI: true + inferenceSettings: { sensitivity: { level: 5 } } + # Placeholder operator and parameters (AI will infer actual thresholds) + operator: GREATER_THAN_OR_EQUAL_TO + parameters: { value: { value: "0", type: NUMBER } } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + mode: ACTIVE + } + ) { + urn + } +} +``` + + + + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +# Initialize the client +client = DataHubClient(server="", token="") + +# Create custom SQL assertion +dataset_urn = DatasetUrn.from_string("urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)") + +sql_assertion = client.assertions.sync_sql_assertion( + dataset_urn=dataset_urn, + display_name="Revenue Quality Check", + statement="SELECT SUM(revenue) FROM database.schema.table WHERE date >= CURRENT_DATE - INTERVAL '1 day'", + criteria_condition="IS_GREATER_THAN_OR_EQUAL_TO", + criteria_parameters=1000, + # Evaluation schedule + schedule="0 6 * * *", # Daily at 6 AM + # Tags + tags=["automated", "revenue", "data_quality"], + enabled=True +) + +print(f"Created SQL assertion: {sql_assertion.urn}") + +# Example with range check +range_sql_assertion = client.assertions.sync_sql_assertion( + dataset_urn=dataset_urn, + display_name="Daily Order Count Range Check", + statement="SELECT COUNT(*) FROM database.schema.orders WHERE DATE(created_at) = CURRENT_DATE", + criteria_condition="IS_WITHIN_A_RANGE", + criteria_parameters=(50, 500), # Between 50 and 500 orders per day + schedule="0 */6 * * *", # Every 6 hours + tags=["automated", "orders", "volume_check"], + enabled=True +) + +print(f"Created range SQL assertion: {range_sql_assertion.urn}") + +# ---------------------------- +# Smart SQL assertions (AI anomaly detection) +# ---------------------------- + +smart_sql_assertion = client.assertions.sync_smart_sql_assertion( + dataset_urn=dataset_urn, + display_name="Smart Revenue Monitor", + # The SQL statement to evaluate - should return a single numeric value + statement="SELECT SUM(revenue) FROM database.schema.table WHERE date >= CURRENT_DATE - INTERVAL '1 day'", + # AI sensitivity setting + sensitivity="medium", # options: "low", "medium", "high" + # Evaluation schedule + schedule="0 */6 * * *", # Every 6 hours + # Optional: training data lookback + training_data_lookback_days=60, + # Tags + tags=["automated", "revenue", "smart_sql"], + enabled=True +) + +print(f"Created smart SQL assertion: {smart_sql_assertion.urn}") +``` + + + + +For more details, see the [Custom SQL Assertions](/docs/managed-datahub/observe/custom-sql-assertions.md) guide. + +### Schema Assertions + + + + +To create a new schema assertion, use the `upsertDatasetSchemaAssertionMonitor` GraphQL Mutation. + +```graphql +mutation upsertDatasetSchemaAssertionMonitor { + upsertDatasetSchemaAssertionMonitor( + assertionUrn: "urn:li:assertion:existing-assertion-id" + input: { + entityUrn: "" + assertion: { + compatibility: EXACT_MATCH + fields: [ + { path: "id", type: STRING } + { path: "count", type: NUMBER } + { path: "struct", type: STRUCT } + { path: "struct.nestedBooleanField", type: BOOLEAN } + ] + } + description: "" + mode: ACTIVE + } + ) +} +``` + +This API will return a unique identifier (URN) for the new assertion if you were successful: + +```json +{ + "data": { + "upsertDatasetSchemaAssertionMonitor": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + + + + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +# Initialize the client +client = DataHubClient(server="", token="") + +# Create schema assertion with exact match compatibility +dataset_urn = DatasetUrn.from_string("urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)") + +schema_assertion = client.assertions.sync_schema_assertion( + dataset_urn=dataset_urn, + display_name="Expected Schema Check", + # Compatibility mode - how strictly to match the schema + compatibility="EXACT_MATCH", # options: "EXACT_MATCH", "SUPERSET", "SUBSET" + # Expected schema fields + fields=[ + {"path": "id", "type": "STRING"}, + {"path": "count", "type": "NUMBER"}, + {"path": "created_at", "type": "TIME"}, + {"path": "is_active", "type": "BOOLEAN"}, + ], + # Tags for grouping + tags=["automated", "schema", "data_quality"], + # Enable the assertion + enabled=True +) + +print(f"Created schema assertion: {schema_assertion.urn}") + +# Create schema assertion with superset compatibility +# (actual schema must contain at least these fields, but can have more) +superset_assertion = client.assertions.sync_schema_assertion( + dataset_urn=dataset_urn, + display_name="Required Fields Check", + compatibility="SUPERSET", + fields=[ + {"path": "id", "type": "STRING"}, + {"path": "name", "type": "STRING"}, + ], + # Evaluation schedule + schedule="0 */6 * * *", # Every 6 hours + tags=["automated", "schema", "required_fields"], + enabled=True +) + +print(f"Created superset schema assertion: {superset_assertion.urn}") + +# Create schema assertion with native type specification +detailed_schema_assertion = client.assertions.sync_schema_assertion( + dataset_urn=dataset_urn, + display_name="Detailed Schema Validation", + compatibility="EXACT_MATCH", + fields=[ + {"path": "id", "type": "STRING", "native_type": "VARCHAR(255)"}, + {"path": "amount", "type": "NUMBER", "native_type": "DECIMAL(10,2)"}, + {"path": "metadata", "type": "STRUCT"}, + {"path": "metadata.key", "type": "STRING"}, + {"path": "tags", "type": "ARRAY"}, + ], + enabled=True +) + +print(f"Created detailed schema assertion: {detailed_schema_assertion.urn}") +``` + + + + +For more details, see the [Schema Assertions](/docs/managed-datahub/observe/schema-assertions.md) guide. + +## Run Assertions + +You can use the following APIs to trigger the assertions you've created to run on-demand. This is +particularly useful for running assertions on a custom schedule, for example from your production +data pipelines. + +> **Long-Running Assertions**: The timeout for synchronously running an assertion is currently limited to a maximum of 30 seconds. +> Each of the following APIs support an `async` parameter, which can be set to `true` to run the assertion asynchronously. +> When set to `true`, the API will kick off the assertion run and return null immediately. To view the result of the assertion, +> simply fetching the runEvents field of the `assertion(urn: String!)` GraphQL query. + + + + +### Run Assertion + +```graphql +mutation runAssertion { + runAssertion(urn: "urn:li:assertion:your-assertion-id", saveResult: true) { + type + nativeResults { + key + value + } + } +} +``` + +Where **type** will contain the Result of the assertion run, either `SUCCESS`, `FAILURE`, or `ERROR`. + +The `saveResult` argument determines whether the result of the assertion will be saved to DataHub's backend, +and available to view through the DataHub UI. If this is set to false, the result will NOT be stored in DataHub's +backend. **Default: `true`** (results are saved when not specified). + +The `async` argument controls whether the assertion runs asynchronously. When set to `true`, the API will kick off +the assertion run and return immediately. When set to `false` or omitted, the assertion runs synchronously with a +30-second timeout. **Default: `false`** (synchronous execution when not specified). + +If the assertion is external (not natively executed by DataHub), this API will return an error. + +If running the assertion is successful, the result will be returned as follows: + +```json +{ + "data": { + "runAssertion": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + "extensions": {} +} +``` + +### Run Group of Assertions + +```graphql +mutation runAssertions { + runAssertions( + urns: [ + "urn:li:assertion:your-assertion-id-1" + "urn:li:assertion:your-assertion-id-2" + ] + saveResults: true + ) { + passingCount + failingCount + errorCount + results { + urn + result { + type + nativeResults { + key + value + } + } + } + } +} +``` + +Where **type** will contain the Result of the assertion run, either `SUCCESS`, `FAILURE`, or `ERROR`. + +The `saveResults` argument determines whether the result of the assertion will be saved to DataHub's backend, +and available to view through the DataHub UI. If this is set to false, the result will NOT be stored in DataHub's +backend. **Default: `true`** (results are saved when not specified). + +The `async` argument controls whether the assertions run asynchronously. When set to `true`, the API will kick off +the assertion runs and return immediately. When set to `false` or omitted, the assertions run synchronously with a +30-second timeout per assertion. **Default: `false`** (synchronous execution when not specified). + +If any of the assertion are external (not natively executed by DataHub), they will simply be omitted from the result set. + +If running the assertions is successful, the results will be returned as follows: + +```json +{ + "data": { + "runAssertions": { + "passingCount": 2, + "failingCount": 0, + "errorCount": 0, + "results": [ + { + "urn": "urn:li:assertion:your-assertion-id-1", + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + { + "urn": "urn:li:assertion:your-assertion-id-2", + "result": { + "type": "FAILURE", + "nativeResults": [ + { + "key": "Value", + "value": "12323" + } + ] + } + } + ] + } + }, + "extensions": {} +} +``` + +Where you should see one result object for each assertion. + +### Run All Assertions for Table + +You can also run all assertions for a specific data asset using the `runAssertionsForAsset` mutation. + +```graphql +mutation runAssertionsForAsset { + runAssertionsForAsset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchase_events,PROD)" + saveResults: true + ) { + passingCount + failingCount + errorCount + results { + urn + result { + type + nativeResults { + key + value + } + } + } + } +} +``` + +Where `type` will contain the Result of the assertion run, either `SUCCESS`, `FAILURE`, or `ERROR`. + +The `saveResults` argument determines whether the result of the assertion will be saved to DataHub's backend, +and available to view through the DataHub UI. If this is set to false, the result will NOT be stored in DataHub's +backend. **Default: `true`** (results are saved when not specified). + +The `async` argument controls whether the assertions run asynchronously. When set to `true`, the API will kick off +the assertion runs and return immediately. When set to `false` or omitted, the assertions run synchronously with a +30-second timeout per assertion. **Default: `false`** (synchronous execution when not specified). + +If any of the assertion are external (not natively executed by DataHub), they will simply be omitted from the result +set. + +If running the assertions is successful, the results will be returned as follows: + +```json +{ + "data": { + "runAssertionsForAsset": { + "passingCount": 2, + "failingCount": 0, + "errorCount": 0, + "results": [ + { + "urn": "urn:li:assertion:your-assertion-id-1", + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + { + "urn": "urn:li:assertion:your-assertion-id-2", + "result": { + "type": "FAILURE", + "nativeResults": [ + { + "key": "Value", + "value": "12323" + } + ] + } + } + ] + } + }, + "extensions": {} +} +``` + +Where you should see one result object for each assertion. + +### Run Group of Assertions for Table + +If you don't always want to run _all_ assertions for a given table, you can also opt to run a subset of the +table's assertions using _Assertion Tags_. First, you'll add tags to your assertions to group and categorize them, +then you'll call the `runAssertionsForAsset` mutation with the `tagUrns` argument to filter for assertions having those tags. + +#### Step 1: Adding Tag to an Assertion + +Currently, you can add tags to an assertion only via the DataHub GraphQL API. You can do this using the following mutation: + +```graphql +mutation addTags { + addTag( + input: { + resourceUrn: "urn:li:assertion:your-assertion" + tagUrn: "urn:li:tag:my-important-tag" + } + ) +} +``` + +#### Step 2: Run All Assertions for a Table with Tags + +Now, you can run all assertions for a table with a specific tag(s) using the `runAssertionsForAsset` mutation with the +`tagUrns` input parameter: + +```graphql +mutation runAssertionsForAsset { + runAssertionsForAsset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchase_events,PROD)" + tagUrns: ["urn:li:tag:my-important-tag"] + ) { + passingCount + failingCount + errorCount + results { + urn + result { + type + nativeResults { + key + value + } + } + } + } +} +``` + +**Coming Soon**: Support for adding tags to assertions through the DataHub UI. + + + + + +### Run Assertion + +```python +# Inlined from /metadata-ingestion/examples/library/run_assertion.py +import logging + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +assertion_urn = "urn:li:assertion:6e3f9e09-1483-40f9-b9cd-30e5f182694a" + +# Run the assertion +assertion_result = graph.run_assertion(urn=assertion_urn, save_result=True) + +log.info( + f"Assertion result (SUCCESS / FAILURE / ERROR): {assertion_result.get('type')}" +) + +``` + +### Run Group of Assertions + +```python +# Inlined from /metadata-ingestion/examples/library/run_assertions.py +import logging + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +assertion_urns = [ + "urn:li:assertion:6e3f9e09-1483-40f9-b9cd-30e5f182694a", + "urn:li:assertion:9e3f9e09-1483-40f9-b9cd-30e5f182694g", +] + +# Run the assertions +assertion_results = graph.run_assertions(urns=assertion_urns, save_result=True).get( + "results" +) + +if assertion_results is not None: + assertion_result_1 = assertion_results.get( + "urn:li:assertion:6e3f9e09-1483-40f9-b9cd-30e5f182694a" + ) + assertion_result_2 = assertion_results.get( + "urn:li:assertion:9e3f9e09-1483-40f9-b9cd-30e5f182694g" + ) + + log.info(f"Assertion results: {assertion_results}") + log.info( + f"Assertion result 1 (SUCCESS / FAILURE / ERROR): {assertion_result_1.get('type')}" + ) + log.info( + f"Assertion result 2 (SUCCESS / FAILURE / ERROR): {assertion_result_2.get('type')}" + ) + +``` + +### Run All Assertions for Table + +```python +# Inlined from /metadata-ingestion/examples/library/run_assertions_for_asset.py +import logging + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_snowflake_table,PROD)" + +# Run all native assertions for the dataset +assertion_results = graph.run_assertions_for_asset(urn=dataset_urn).get("results") + +if assertion_results is not None: + assertion_result_1 = assertion_results.get( + "urn:li:assertion:6e3f9e09-1483-40f9-b9cd-30e5f182694a" + ) + assertion_result_2 = assertion_results.get( + "urn:li:assertion:9e3f9e09-1483-40f9-b9cd-30e5f182694g" + ) + + log.info(f"Assertion results: {assertion_results}") + log.info( + f"Assertion result 1 (SUCCESS / FAILURE / ERROR): {assertion_result_1.get('type')}" + ) + log.info( + f"Assertion result 2 (SUCCESS / FAILURE / ERROR): {assertion_result_2.get('type')}" + ) + +# Run a subset of native assertions having a specific tag +important_assertion_tag = "urn:li:tag:my-important-assertion-tag" +assertion_results = graph.run_assertions_for_asset( + urn=dataset_urn, tag_urns=[important_assertion_tag] +).get("results") + +``` + + + + + +### Providing Dynamic Parameters to Assertions + +You can provide **dynamic parameters** to your assertions to customize their behavior. This is particularly useful for +assertions that require dynamic parameters, such as a threshold value that changes based on the time of day. + +Dynamic parameters can be injected into the SQL fragment portion of any Assertion. For example, it can appear +in any part of the SQL statement in a [Custom SQL](/docs/managed-datahub/observe/custom-sql-assertions.md) Assertion, +or it can appear in the **Advanced > Filter** section of a [Column](/docs/managed-datahub/observe/column-assertions.md), +[Volume](/docs/managed-datahub/observe/volume-assertions.md), or [Freshness](/docs/managed-datahub/observe/freshness-assertions.md) Assertion. + +To do so, you'll first need to edit the SQL fragment to include the dynamic parameter. Dynamic parameters appear +as `${parameterName}` in the SQL fragment. + +Next, you'll call the `runAssertion`, `runAssertions`, or `runAssertionsForAsset` mutations with the `parameters` input argument. +This argument is a list of key-value tuples, where the key is the parameter name and the value is the parameter value: + + + + +```graphql +mutation runAssertion { + runAssertion( + urn: "urn:li:assertion:your-assertion-id" + parameters: [{ key: "parameterName", value: "parameterValue" }] + ) { + type + nativeResults { + key + value + } + } +} +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/run_assertion_with_parameters.py +import logging + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +assertion_urn = "urn:li:assertion:6e3f9e09-1483-40f9-b9cd-30e5f182694a" + +# Define dynamic parameters to inject into the assertion's SQL fragment. +# These parameters will replace ${parameterName} placeholders in the SQL. +parameters = { + "min_threshold": "100", + "max_threshold": "1000", +} + +# Run the assertion with dynamic parameters +assertion_result = graph.run_assertion( + urn=assertion_urn, + save_result=True, + parameters=parameters, +) + +log.info( + f"Assertion result (SUCCESS / FAILURE / ERROR): {assertion_result.get('type')}" +) + +``` + + + + + +At runtime, the `${parameterName}` placeholder in the SQL fragment will be replaced with the provided `parameterValue` before the query +is sent to the database for execution. + +## Get Assertion Details + +You can use the following APIs to + +1. Fetch existing assertion definitions + run history +2. Fetch the assertions associated with a given table + their run history. + + + + +### Get Assertions for Table + +To retrieve all the assertions for a table, you can use the following GraphQL Query. + +```graphql +query dataset { + dataset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)" + ) { + assertions(start: 0, count: 1000) { + start + count + total + assertions { + urn + # Fetch the last run of each associated assertion. + runEvents(status: COMPLETE, limit: 1) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type + description + lastUpdated { + time + actor + } + datasetAssertion { + datasetUrn + scope + aggregation + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + fields { + urn + path + } + nativeType + nativeParameters { + key + value + } + logic + } + freshnessAssertion { + type + entityUrn + schedule { + type + cron { + cron + timezone + } + fixedInterval { + unit + multiple + } + } + filter { + type + sql + } + } + sqlAssertion { + type + entityUrn + statement + changeType + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + fieldAssertion { + type + entityUrn + filter { + type + sql + } + fieldValuesAssertion { + field { + path + type + nativeType + } + transform { + type + } + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + failThreshold { + type + value + } + excludeNulls + } + fieldMetricAssertion { + field { + path + type + nativeType + } + metric + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + volumeAssertion { + type + entityUrn + filter { + type + sql + } + rowCountTotal { + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + rowCountChange { + type + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + schemaAssertion { + entityUrn + compatibility + fields { + path + type + nativeType + } + schema { + fields { + fieldPath + type + nativeDataType + } + } + } + source { + type + created { + time + actor + } + } + } + } + } + } +} +``` + +### Get Assertion Details + +You can use the following GraphQL query to fetch the details for an assertion along with its evaluation history by URN. + +```graphql +query getAssertion { + assertion(urn: "urn:li:assertion:assertion-id") { + urn + # Fetch the last 10 runs for the assertion. + runEvents(status: COMPLETE, limit: 10) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type + description + lastUpdated { + time + actor + } + datasetAssertion { + datasetUrn + scope + aggregation + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + fields { + urn + path + } + nativeType + nativeParameters { + key + value + } + logic + } + freshnessAssertion { + type + entityUrn + schedule { + type + cron { + cron + timezone + } + fixedInterval { + unit + multiple + } + } + filter { + type + sql + } + } + sqlAssertion { + type + entityUrn + statement + changeType + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + fieldAssertion { + type + entityUrn + filter { + type + sql + } + fieldValuesAssertion { + field { + path + type + nativeType + } + transform { + type + } + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + failThreshold { + type + value + } + excludeNulls + } + fieldMetricAssertion { + field { + path + type + nativeType + } + metric + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + volumeAssertion { + type + entityUrn + filter { + type + sql + } + rowCountTotal { + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + rowCountChange { + type + operator + parameters { + value { + value + type + } + minValue { + value + type + } + maxValue { + value + type + } + } + } + } + schemaAssertion { + entityUrn + compatibility + fields { + path + type + nativeType + } + schema { + fields { + fieldPath + type + nativeDataType + } + } + } + source { + type + created { + time + actor + } + } + } + } +} +``` + + + + + +```python +Python support coming soon! +``` + + + + +## Add Tag to Assertion + +You can add tags to individual assertions to group and categorize them, for example by its priority or severity. +Note that the tag should already exist in DataHub, or the operation will fail. + + + + +```graphql +mutation addTags { + addTag( + input: { + resourceUrn: "urn:li:assertion:your-assertion" + tagUrn: "urn:li:tag:my-important-tag" + } + ) +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "addTag": true + }, + "extensions": {} +} +``` + +You can create new tags using the `createTag` mutation or via the UI. + + + + +## Delete Assertions + +You can use delete dataset operations to DataHub using the following APIs. + + + + +```graphql +mutation deleteAssertion { + deleteAssertion(urn: "urn:li:assertion:test") +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "deleteAssertion": true + }, + "extensions": {} +} +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/assertion_delete.py +import logging + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +assertion_urn = "urn:li:assertion:my-assertion" + +# Delete the Assertion +graph.delete_entity(urn=assertion_urn, hard=True) + +log.info(f"Deleted assertion {assertion_urn}") + +``` + + + + +## (Advanced) Create and Report Results for Custom Assertions + +If you'd like to create and report results for your own custom assertions, e.g. those which are run and +evaluated outside of DataHub Cloud, you need to generate 2 important Assertion Entity aspects, and give the assertion a unique +URN of the following format: + +1. Generate a unique URN for your assertion + +```plaintext +urn:li:assertion: +``` + +2. Generate the [**AssertionInfo**](/docs/generated/metamodel/entities/assertion.md#assertion-info) aspect for the assertion. You can do this using the Python SDK. Give your assertion a `type` and a `source` + with type `EXTERNAL` to mark it as an external assertion, not run by DataHub itself. + +3. Generate the [**AssertionRunEvent**](/docs/generated/metamodel/entities/assertion.md#assertionrunevent-timeseries) timeseries aspect using the Python SDK. This aspect should contain the result of the assertion + run at a given timestamp and will be shown on the results graph in DataHub's UI. + +## Create and Remove Subscriptions + +Reference the [Subscriptions SDK](/docs/api/tutorials/subscriptions.md) for more information on how to create and remove subscriptions on Datasets or Assertions. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/container.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/container.md new file mode 100644 index 00000000..db5b81f0 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/container.md @@ -0,0 +1,57 @@ +--- +title: Container +slug: /api/tutorials/container +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/container.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Container + +## Why Would You Use Containers? + +The Container entity represents a logical grouping of entities, such as datasets, data processing instances, or even other containers. It helps users organize and manage metadata in a hierarchical structure, making it easier to navigate and understand relationships between different entities. + +#### How is a Container related to other entities? + +1. **Parent-Child Relationship** : Containers can contain other entities such as datasets, charts, dashboards, data jobs, ML models, and even other containers (nested containers). For example, a dataset can have a container aspect that links it to the schema or folder (container) it belongs to. + +2. **Hierarchical Organization** : Containers can be nested, forming a hierarchy (e.g., a database container contains schema containers, which contain table containers). This enables a folder-like browsing experience in the DataHub UI. + +3. **Relationships in the Metadata Model** : Many entities (datasets, data jobs, ML models, etc.) have a container aspect that links them to their parent container. Containers themselves can have a parentContainer aspect for nesting. + +### Goal Of This Guide + +This guide will show you how to + +- Create a container + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create Container + + + + +```python +# Inlined from /metadata-ingestion/examples/library/container_create.py +from datahub.emitter.mcp_builder import DatabaseKey +from datahub.sdk import Container, DataHubClient + +client = DataHubClient.from_env() + +container = Container( + container_key=DatabaseKey(platform="snowflake", database="my_database"), + display_name="MY_DATABASE", +) + +client.entities.upsert(container) + +``` + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-assertions.md new file mode 100644 index 00000000..64e7eb6c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-assertions.md @@ -0,0 +1,360 @@ +--- +title: Custom Assertions +slug: /api/tutorials/custom-assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/custom-assertions.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Custom Assertions + +This guide specifically covers how to create and report results for custom assertions in DataHub. +Custom Assertions are those not natively run or directly modeled by DataHub, and managed by a 3rd party framework or tool. + +To create _native_ assertions using the API (e.g. for DataHub to manage), please refer to the [Assertions API](./assertions.md). + +This guide may be used as reference for partners seeking to integrate their own monitoring tools with DataHub. + +## Goal Of This Guide + +In this guide, you will learn how to + +1. Create and update custom assertions via GraphQL and Python APIs +2. Report results for custom assertions via GraphQL and Python APIs +3. Retrieve results for custom assertions via GraphQL and Python APIs +4. Delete custom assertions via GraphQL and Python APIs + +## Prerequisites + +The actor making API calls must have the `Edit Assertions` and `Edit Monitors` privileges for the Tables being monitored. + +## Create And Update Custom Assertions + +You may create custom assertions using the following APIs for a Dataset in DataHub. + + + + +To create a new assertion, use the `upsertCustomAssertion` GraphQL Mutation. This mutation both allows you to +create and update a given assertion. + +```graphql +mutation upsertCustomAssertion { + upsertCustomAssertion( + urn: "urn:li:assertion:my-custom-assertion-id" # Optional: if you want to provide a custom id. If not, one will be generated for you. + input: { + entityUrn: "" + type: "My Custom Category" # This is how your assertion will appear categorized in DataHub. + description: "The description of my external assertion for my dataset" + platform: { + urn: "urn:li:dataPlatform:great-expectations" # OR you can provide name: "My Custom Platform" if you do not have an URN for the platform. + } + fieldPath: "field_foo" # Optional: if you want to associated with a specific field, + externalUrl: "https://my-monitoring-tool.com/result-for-this-assertion" # Optional: if you want to provide a link to the monitoring tool + # Optional: If you want to provide a custom SQL query for the assertion. This will be rendered as a query in the UI. + # logic: "SELECT * FROM X WHERE Y" + } + ) { + urn + } +} +``` + +Note that you can either provide a unique `urn` for the assertion, which will be used to generate the corresponding assertion urn in the following format: + +`urn:li:assertion:` + +or a random urn will be created and returned for you. This id should be stable over time and unique for each assertion. + +The upsert API will return the unique identifier (URN) for the the assertion if you were successful: + +```json +{ + "data": { + "upsertExternalAssertion": { + "urn": "urn:li:assertion:your-new-assertion-id" + } + }, + "extensions": {} +} +``` + + + + + +To upsert an assertion in Python, simply use the `upsert_external_assertion` method on the DataHub Client object. + +```python +# Inlined from /metadata-ingestion/examples/library/upsert_custom_assertion.py +import logging + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +new_assertion_urn = "urn:li:assertion:my-unique-assertion-id" + +# Upsert the assertion +res = graph.upsert_custom_assertion( + urn=new_assertion_urn, # If the assertion already exists, provide the URN + entity_urn="", + type="My Custom Category", # This categorizes your assertion in DataHub + description="The description of my external assertion for my dataset", + platform_urn="urn:li:dataPlatform:great-expectations", # OR you can provide 'platformName="My Custom Platform"' + field_path="field_foo", # Optional: if you want to associate it with a specific field + external_url="https://my-monitoring-tool.com/result-for-this-assertion", # Optional: link to monitoring tool + logic="SELECT * FROM X WHERE Y", # Optional: custom SQL for the assertion, rendered in the UI +) + +if res is not None: + log.info(f"Upserted assertion with urn: {new_assertion_urn}") + +``` + + + + + +## Report Results For Custom Assertions + +When an assertion is evaluated against a Dataset, or a new result is available, you can report the result to DataHub +using the following APIs. + +Once reported, these will appear in the evaluation history of the assertion and will be used to determine whether the assertion is +displayed as passing or failing in the DataHub UI. + + + + +To report results for a custom, use the `reportAssertionResult` GraphQL Mutation. This mutation both allows you to +create and update a given assertion. + +```graphql +mutation reportAssertionResult { + reportAssertionResult( + urn: "urn:li:assertion:" + result: { + timestampMillis: 1620000000000 # Unix timestamp in millis. If not provided, the current time will be used. + type: SUCCESS # or FAILURE or ERROR or INIT + properties: [{ key: "my_custom_key", value: "my_custom_value" }] + externalUrl: "https://my-great-expectations.com/results/1234" # Optional: URL to the results in the external tool + # Optional: If the type is ERROR, you can provide additional context. See full list of error types below. + # error: { + # type: UNKNOWN_ERROR, + # message: "The assertion failed due to an unknown error" + # } + } + ) +} +``` + +The `type` field is used to communicate the latest health status of the assertion. + +The `properties` field is used to provide additional key-value pair context that will be displayed alongside the result +in DataHub's UI. + +The full list of supported error types include: + +- SOURCE_CONNECTION_ERROR +- SOURCE_QUERY_FAILED +- INSUFFICIENT_DATA +- INVALID_PARAMETERS +- INVALID_SOURCE_TYPE +- UNSUPPORTED_PLATFORM +- CUSTOM_SQL_ERROR +- FIELD_ASSERTION_ERROR +- UNKNOWN_ERROR + +```json +{ + "data": { + "reportAssertionResult": true + }, + "extensions": {} +} +``` + +If the result is `true`, the result was successfully reported. + + + + + +To report an assertion result in Python, simply use the `report_assertion_result` method on the DataHub Client object. + +```python +# Inlined from /metadata-ingestion/examples/library/report_assertion_result.py +import logging +import time + +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +log = logging.getLogger(__name__) + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +existing_assertion_urn = "urn:li:assertion:my-unique-assertion-id" + +# Report result for assertion +res = graph.report_assertion_result( + urn="urn:li:assertion:", # Replace with your actual assertion URN + timestamp_millis=int(time.time() * 1000), # Current Unix timestamp in milliseconds + type="SUCCESS", # Can be 'SUCCESS', 'FAILURE', 'ERROR', or 'INIT' + properties=[{"key": "my_custom_key", "value": "my_custom_value"}], + external_url="https://my-great-expectations.com/results/1234", # Optional: URL to the results in the external tool + # Uncomment the following section and use if type is 'ERROR' + # error_type="UNKNOWN_ERROR", + # error_message="The assertion failed due to an unknown error" +) + +if res: + log.info("Successfully reported Assertion Result!") + +``` + + + + + +## Retrieve Results For Custom Assertions + +After an assertion has been created and run, it will appear in the set of assertions associated with a given dataset urn. +You can retrieve the results of these assertions using the following APIs. + + + + +### Get Assertions for Dataset + +To retrieve all the assertions for a table / dataset, you can use the following GraphQL Query. + +```graphql +query dataset { + dataset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)" + ) { + assertions(start: 0, count: 1000) { + start + count + total + assertions { + urn + # Fetch the last run of each associated assertion. + runEvents(status: COMPLETE, limit: 1) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type # Will be CUSTOM + customType # Will be your custom type. + description + lastUpdated { + time + actor + } + customAssertion { + entityUrn + fieldPath + externalUrl + logic + } + source { + type + created { + time + actor + } + } + } + } + } + } +} +``` + +### Get Assertion Details + +You can use the following GraphQL query to fetch the details for an assertion along with its evaluation history by URN. + +```graphql +query getAssertion { + assertion(urn: "urn:li:assertion:my-custom-assertion-id") { + urn + # Fetch the last 10 runs for the assertion. + runEvents(status: COMPLETE, limit: 10) { + total + failed + succeeded + runEvents { + timestampMillis + status + result { + type + nativeResults { + key + value + } + } + } + } + info { + type # Will be CUSTOM + customType # Will be your custom type. + description + lastUpdated { + time + actor + } + customAssertion { + entityUrn + fieldPath + externalUrl + logic + } + source { + type + created { + time + actor + } + } + } + # Fetch what entities have the assertion attached to it + relationships(input: { types: ["Asserts"], direction: OUTGOING }) { + total + relationships { + entity { + urn + } + } + } + } +} +``` + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-properties.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-properties.md new file mode 100644 index 00000000..37a1d439 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/custom-properties.md @@ -0,0 +1,452 @@ +--- +title: Custom Properties +slug: /api/tutorials/custom-properties +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/custom-properties.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Custom Properties + +## Why Would You Use Custom Properties on Datasets? + +Custom properties to datasets can help to provide additional information about the data that is not readily available in the standard metadata fields. Custom properties can be used to describe specific attributes of the data, such as the units of measurement used, the date range covered, or the geographical region the data pertains to. This can be particularly helpful when working with large and complex datasets, where additional context can help to ensure that the data is being used correctly and effectively. + +DataHub models custom properties of a Dataset as a map of key-value pairs of strings. + +Custom properties can also be used to enable advanced search and discovery capabilities, by allowing users to filter and sort datasets based on specific attributes. This can help users to quickly find and access the data they need, without having to manually review large numbers of datasets. + +### Goal Of This Guide + +This guide will show you how to add, remove or replace custom properties on a dataset `fct_users_deleted`. Here's what each operation means: + +- Add: Add custom properties to a dataset without affecting existing properties +- Remove: Removing specific properties from the dataset without affecting other properties +- Replace: Completely replacing the entire property map without affecting other fields that are located in the same aspect. e.g. `DatasetProperties` aspect contains `customProperties` as well as other fields like `name` and `description`. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed information, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +Before adding custom properties, you need to ensure the target dataset is already present in your DataHub instance. +If you attempt to manipulate entities that do not exist, your operation will fail. +In this guide, we will be using data from sample ingestion. +::: + +In this example, we will add some custom properties `cluster_name` and `retention_time` to the dataset `fct_users_deleted`. + +After you have ingested sample data, the dataset `fct_users_deleted` should have a custom properties section with `encoding` set to `utf-8`. + +

+ +

+ +```shell +datahub get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" --aspect datasetProperties +{ + "datasetProperties": { + "customProperties": { + "encoding": "utf-8" + }, + "description": "table containing all the users deleted on a single day", + "tags": [] + } +} +``` + +## Add Custom Properties programmatically + +The following code adds custom properties `cluster_name` and `retention_time` to a dataset named `fct_users_deleted` without affecting existing properties. + + + + +> 🚫 Adding Custom Properties on Dataset via GraphQL is currently not supported. +> Please check out [API feature comparison table](/docs/api/datahub-apis.md#datahub-api-comparison) for more information, + + + + +```java +# Inlined from /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAdd.java +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class DatasetCustomPropertiesAdd { + + private DatasetCustomPropertiesAdd() {} + + /** + * Adds properties to an existing custom properties aspect without affecting any existing + * properties + * + * @param args + * @throws IOException + * @throws ExecutionException + * @throws InterruptedException + */ + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + MetadataChangeProposal datasetPropertiesProposal = + new DatasetPropertiesPatchBuilder() + .urn(UrnUtils.toDatasetUrn("hive", "fct_users_deleted", "PROD")) + .addCustomProperty("cluster_name", "datahubproject.acryl.io") + .addCustomProperty("retention_time", "2 years") + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + try { + Future response = emitter.emit(datasetPropertiesProposal); + + System.out.println(response.get().getResponseContent()); + } catch (Exception e) { + log.error("Failed to emit metadata to DataHub", e); + throw e; + } finally { + emitter.close(); + } + } +} + +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_custom_properties_patch.py +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + +# Create Dataset Patch to Add Custom Properties +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.add_custom_property("cluster_name", "datahubproject.acryl.io") +patch_builder.add_custom_property("retention_time", "2 years") +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + +### Expected Outcome of Adding Custom Properties + +You can now see the two new properties are added to `fct_users_deleted` and the previous property `encoding` is unchanged. + +

+ +

+ +We can also verify this operation by programmatically checking the `datasetProperties` aspect after running this code using the `datahub` cli. + +```shell +datahub get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" --aspect datasetProperties +{ + "datasetProperties": { + "customProperties": { + "encoding": "utf-8", + "cluster_name": "datahubproject.acryl.io", + "retention_time": "2 years" + }, + "description": "table containing all the users deleted on a single day", + "tags": [] + } +} +``` + +## Add and Remove Custom Properties programmatically + +The following code shows you how can add and remove custom properties in the same call. In the following code, we add custom property `cluster_name` and remove property `retention_time` from a dataset named `fct_users_deleted` without affecting existing properties. + + + + +> 🚫 Adding and Removing Custom Properties on Dataset via GraphQL is currently not supported. +> Please check out [API feature comparison table](/docs/api/datahub-apis.md#datahub-api-comparison) for more information, + + + + +```java +# Inlined from /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesAddRemove.java +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class DatasetCustomPropertiesAddRemove { + + private DatasetCustomPropertiesAddRemove() {} + + /** + * Applies Add and Remove property operations on an existing custom properties aspect without + * affecting any other properties + * + * @param args + * @throws IOException + * @throws ExecutionException + * @throws InterruptedException + */ + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + MetadataChangeProposal datasetPropertiesProposal = + new DatasetPropertiesPatchBuilder() + .urn(UrnUtils.toDatasetUrn("hive", "fct_users_deleted", "PROD")) + .addCustomProperty("cluster_name", "datahubproject.acryl.io") + .removeCustomProperty("retention_time") + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + try { + Future response = emitter.emit(datasetPropertiesProposal); + + System.out.println(response.get().getResponseContent()); + } catch (Exception e) { + log.error("Failed to emit metadata to DataHub", e); + throw e; + } finally { + emitter.close(); + } + } +} + +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_remove_custom_properties_patch.py +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.specific.dataset import DatasetPatchBuilder + +# Create DataHub Client +datahub_client = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +# Create Dataset URN +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + +# Create Dataset Patch to Add + Remove Custom Properties +patch_builder = DatasetPatchBuilder(dataset_urn) +patch_builder.add_custom_property("cluster_name", "datahubproject.acryl.io") +patch_builder.remove_custom_property("retention_time") +patch_mcps = patch_builder.build() + +# Emit Dataset Patch +for patch_mcp in patch_mcps: + datahub_client.emit(patch_mcp) + +``` + + + + +### Expected Outcome of Add and Remove Operations on Custom Properties + +You can now see the `cluster_name` property is added to `fct_users_deleted` and the `retention_time` property is removed. + +

+ +

+ +We can also verify this operation programmatically by checking the `datasetProperties` aspect using the `datahub` cli. + +```shell +datahub get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" --aspect datasetProperties +{ + "datasetProperties": { + "customProperties": { + "encoding": "utf-8", + "cluster_name": "datahubproject.acryl.io" + }, + "description": "table containing all the users deleted on a single day", + "tags": [] + } +} +``` + +## Replace Custom Properties programmatically + +The following code replaces the current custom properties with a new properties map that includes only the properties `cluster_name` and `retention_time`. After running this code, the previous `encoding` property will be removed. + + + + +> 🚫 Replacing Custom Properties on Dataset via GraphQL is currently not supported. +> Please check out [API feature comparison table](/docs/api/datahub-apis.md#datahub-api-comparison) for more information, + + + + +```java +# Inlined from /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetCustomPropertiesReplace.java +package io.datahubproject.examples; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.metadata.aspect.patch.builder.DatasetPropertiesPatchBuilder; +import com.linkedin.mxe.MetadataChangeProposal; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Future; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +class DatasetCustomPropertiesReplace { + + private DatasetCustomPropertiesReplace() {} + + /** + * Replaces the existing custom properties map with a new map. Fields like dataset name, + * description etc remain unchanged. + * + * @param args + * @throws IOException + */ + public static void main(String[] args) throws IOException { + Map customPropsMap = new HashMap<>(); + customPropsMap.put("cluster_name", "datahubproject.acryl.io"); + customPropsMap.put("retention_time", "2 years"); + MetadataChangeProposal datasetPropertiesProposal = + new DatasetPropertiesPatchBuilder() + .urn(UrnUtils.toDatasetUrn("hive", "fct_users_deleted", "PROD")) + .setCustomProperties(customPropsMap) + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + + try { + Future response = emitter.emit(datasetPropertiesProposal); + System.out.println(response.get().getResponseContent()); + } catch (Exception e) { + log.error("Failed to emit metadata to DataHub", e); + } finally { + emitter.close(); + } + } +} + +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_replace_properties.py +import logging +from typing import Union + +from datahub.configuration.kafka import KafkaProducerConnectionConfig +from datahub.emitter.kafka_emitter import DatahubKafkaEmitter, KafkaEmitterConfig +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.emitter.rest_emitter import DataHubRestEmitter +from datahub.specific.dataset import DatasetPatchBuilder + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +# Get an emitter, either REST or Kafka, this example shows you both +def get_emitter() -> Union[DataHubRestEmitter, DatahubKafkaEmitter]: + USE_REST_EMITTER = True + if USE_REST_EMITTER: + gms_endpoint = "http://localhost:8080" + return DataHubRestEmitter(gms_server=gms_endpoint) + else: + kafka_server = "localhost:9092" + schema_registry_url = "http://localhost:8081" + return DatahubKafkaEmitter( + config=KafkaEmitterConfig( + connection=KafkaProducerConnectionConfig( + bootstrap=kafka_server, schema_registry_url=schema_registry_url + ) + ) + ) + + +dataset_urn = make_dataset_urn(platform="hive", name="fct_users_created", env="PROD") + +property_map_to_set = { + "cluster_name": "datahubproject.acryl.io", + "retention_time": "2 years", +} + +with get_emitter() as emitter: + for patch_mcp in ( + DatasetPatchBuilder(dataset_urn) + .set_custom_properties(property_map_to_set) + .build() + ): + emitter.emit(patch_mcp) + + +log.info( + f"Replaced custom properties on dataset {dataset_urn} as {property_map_to_set}" +) + +``` + + + + +### Expected Outcome of Replacing Custom Properties + +You can now see the `cluster_name` and `retention_time` properties are added to `fct_users_deleted` but the previous `encoding` property is no longer present. + +

+ +

+ +We can also verify this operation programmatically by checking the `datasetProperties` aspect using the `datahub` cli. + +```shell +datahub get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" --aspect datasetProperties +{ + "datasetProperties": { + "customProperties": { + "cluster_name": "datahubproject.acryl.io", + "retention_time": "2 years" + }, + "description": "table containing all the users deleted on a single day", + "tags": [] + } +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dashboard-chart.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dashboard-chart.md new file mode 100644 index 00000000..0177bc40 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dashboard-chart.md @@ -0,0 +1,216 @@ +--- +title: Dashboard & Chart +slug: /api/tutorials/dashboard-chart +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/dashboard-chart.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Dashboard & Chart + +## Why Would You Use Dashboards and Charts? + +The dashboard and chart entities are used to represent visualizations of data, typically in the context of business intelligence or analytics platforms. They allow users to create, manage, and share visual representations of data insights. + +### Goal Of This Guide + +This guide will show you how to + +- Create a dashboard and a chart. +- Link the dashboard to the chart or another dashboard. +- Read dashboard and chart entities. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create Chart + +```python +# Inlined from /metadata-ingestion/examples/library/chart_create_simple.py +from datahub.metadata.urns import TagUrn +from datahub.sdk import Chart, DataHubClient + +client = DataHubClient.from_env() + +chart = Chart( + name="example_chart", + platform="looker", + description="looker chart for production", + tags=[TagUrn(name="production"), TagUrn(name="data_engineering")], +) + +client.entities.upsert(chart) + +``` + +### Link Chart with Datasets + +You can associate datasets with the chart by providing the dataset URN in the `input_datasets` parameter. This will create lineage between the chart and the datasets, so you can track the data sources used by the chart. + +```python +# Inlined from /metadata-ingestion/examples/library/chart_create_complex.py +from datahub.metadata.urns import TagUrn +from datahub.sdk import Chart, DataHubClient, Dataset + +client = DataHubClient.from_env() + +input_datasets = [ + Dataset( + name="example_dataset", + platform="snowflake", + description="looker dataset for production", + schema=[("id", "string"), ("name", "string")], + ), + Dataset( + name="example_dataset_2", + platform="snowflake", + description="looker dataset for production", + schema=[("id", "string"), ("name", "string")], + ), + Dataset( + name="example_dataset_3", + platform="snowflake", + description="looker dataset for production", + schema=[("id", "string"), ("name", "string")], + ), +] + +# create a chart with two input datasets +chart = Chart( + name="example_chart", + platform="looker", + description="looker chart for production", + tags=[TagUrn(name="production"), TagUrn(name="data_engineering")], + input_datasets=[input_datasets[0], input_datasets[1]], +) + +for dataset in input_datasets: + client.entities.upsert(dataset) + +# add a new dataset to the chart +chart.add_input_dataset(input_datasets[2]) +client.entities.upsert(chart) + +``` + +## Create Dashboard + +```python +# Inlined from /metadata-ingestion/examples/library/dashboard_create_simple.py +from datahub.metadata.urns import TagUrn +from datahub.sdk import Dashboard, DataHubClient + +client = DataHubClient.from_env() + +dashboard = Dashboard( + name="example_dashboard", + platform="looker", + description="looker dashboard for production", + tags=[TagUrn(name="production"), TagUrn(name="data_engineering")], +) + +client.entities.upsert(dashboard) + +``` + +### Link Dashboard with Charts, Dashboards, and Datasets + +You can associate charts, dashboards, and datasets with the dashboard by providing their URNs in the `charts`, `dashboards`, and `input_datasets` parameters, respectively. This will create lineage between the dashboard and the associated entities. + +```python +# Inlined from /metadata-ingestion/examples/library/dashboard_create_complex.py +from datahub.metadata.urns import TagUrn +from datahub.sdk import Chart, Dashboard, DataHubClient, Dataset + +client = DataHubClient.from_env() +dashboard1 = Dashboard( + name="example_dashboard_2", + platform="looker", + description="looker dashboard for production", +) +chart = Chart( + name="example_chart", + platform="looker", + description="looker chart for production", +) + +input_dataset = Dataset( + name="example_dataset5", + platform="snowflake", + description="snowflake dataset for production", +) + + +dashboard2 = Dashboard( + name="example_dashboard", + platform="looker", + description="looker dashboard for production", + tags=[TagUrn(name="production"), TagUrn(name="data_engineering")], + input_datasets=[input_dataset.urn], + charts=[chart.urn], + dashboards=[dashboard1.urn], +) + + +client.entities.upsert(dashboard1) +client.entities.upsert(chart) +client.entities.upsert(input_dataset) + +client.entities.upsert(dashboard2) + +``` + +## Read Chart + +```python +# Inlined from /metadata-ingestion/examples/library/chart_read.py +from datahub.sdk import ChartUrn, DataHubClient + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use ChartUrn.from_string(...) +chart_urn = ChartUrn("looker", "example_chart_id") + +chart_entity = client.entities.get(chart_urn) +print("Chart name:", chart_entity.name) +print("Chart platform:", chart_entity.platform) +print("Chart description:", chart_entity.description) + +``` + +#### Expected Output + +```python +>> Chart name: example_chart +>> Chart platform: urn:li:dataPlatform:looker +>> Chart description: looker chart for production +``` + +## Read Dashboard + +```python +# Inlined from /metadata-ingestion/examples/library/dashboard_read.py +from datahub.sdk import DashboardUrn, DataHubClient + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use DashboardUrn.from_string(...) +dashboard_urn = DashboardUrn("looker", "example_dashboard_id") + +dashboard_entity = client.entities.get(dashboard_urn) +print("Dashboard name:", dashboard_entity.name) +print("Dashboard platform:", dashboard_entity.platform) +print("Dashboard description:", dashboard_entity.description) + +``` + +#### Expected Output + +```python +>> Dashboard name: example_dashboard +>> Dashboard platform: urn:li:dataPlatform:looker +>> Dashboard description: looker dashboard for production +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/data-contracts.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/data-contracts.md new file mode 100644 index 00000000..d840001d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/data-contracts.md @@ -0,0 +1,224 @@ +--- +title: Data Contracts +slug: /api/tutorials/data-contracts +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/data-contracts.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Data Contracts + + + +This guide specifically covers how to use the Data Contract APIs with **DataHub Cloud**. + +## Why Would You Use Data Contract APIs? + +The Assertions APIs allow you to create, update, and evaluate Data Contracts programmatically. This is particularly +useful to automate the monitoring of data quality and schema compliance for your data. + +### Goal Of This Guide + +This guide will show you how to create, update, and check the status of aData Contract. + +## Prerequisites + +### Privileges Required + +The actor making API calls must have the `Edit Data Contract` privileges for the Tables at hand. + +### Assertions + +Before creating a Data Contract, you should have already created the Assertions that you want to associate with the Data Contract. +Check out the [Assertions](/docs/api/tutorials/assertions.md) guide for details on how to create DataHub Assertions. + +## Create & Update Data Contract + +You can create a new Data Contract, which is simply bundle of "important" assertions, using the following APIs. + + + + +To create or update a Data Contract, simply use the `upsertDataContract` GraphQL Mutation. + +```graphql +mutation upsertDataContract { + upsertDataContract( + input: { + entityUrn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)", # Table to Create Contract for + freshness: [ + { + assertionUrn: "urn:li:assertion:your-freshness-assertion-id", + } + ], + schema: [ + { + assertionUrn: "urn:li:assertion:your-schema-assertion-id", + } + ], + dataQuality: [ + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-1", + }, + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-2", + } + ] + }) { + urn + } + ) +} +``` + +This API will return a unique identifier (URN) for the Data Contract if you were successful: + +```json +{ + "data": { + "upsertDataContract": { + "urn": "urn:li:dataContract:your-new-contract-id" + } + }, + "extensions": {} +} +``` + +If you want to update an existing Data Contract, you can use the same API, but also passing the `urn` parameter in the +`upsertDataContract` mutation. + +```graphql +mutation upsertDataContract { + upsertDataContract( + urn: "urn:li:dataContract:your-existing-contract-id", + input: { + freshness: [ + { + assertionUrn: "urn:li:assertion:your-freshness-assertion-id", + } + ], + schema: [ + { + assertionUrn: "urn:li:assertion:your-schema-assertion-id", + } + ], + dataQuality: [ + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-1", + }, + { + assertionUrn: "urn:li:assertion:your-column-assertion-id-2", + } + ] + }) { + urn + } + ) +} +``` + + + + +## Check Contract Status + +You can use the following APIs to check whether a Data Contract is passing or failing, which is determined +by the last status of the assertions associated with the contract. + + + + + +### Check Contract Status for Table + +```graphql +query getTableContractStatus { + dataset(urn: "urn:li:dataset(urn:li:dataPlatform:snowflake,purchases,PROD") { + contract { + result { + type # Passing or Failing. + assertionResults { + # Results of each contract assertion. + assertion { + urn + } + result { + type + nativeResults { + key + value + } + } + } + } + } + } +} +``` + +You can also _force refresh_ all of the Contract Assertions by evaluating them on-demand by providing the `refresh` argument +in your query. + +```graphql +query getTableContractStatus { + dataset(urn: "urn:li:dataset(urn:li:dataPlatform:snowflake,purchases,PROD") { + contract(refresh: true) { + ...same + } + } +} +``` + +This will run any native DataHub Cloud assertions comprising the Data Contract. Be careful! This can take a while depending on how many native assertions are part of the contract. + +If you're successful, you'll get the latest status for the Table Contract: + +```json +{ + "data": { + "dataset": { + "contract": { + "result": { + "type": "PASSING", + "assertionResults": [ + { + "assertion": { + "urn": "urn:li:assertion:your-freshness-assertion-id" + }, + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "1382" + } + ] + } + }, + { + "assertion": { + "urn": "urn:li:assertion:your-volume-assertion-id" + }, + "result": { + "type": "SUCCESS", + "nativeResults": [ + { + "key": "Value", + "value": "12323" + } + ] + } + } + ] + } + } + } + }, + "extensions": {} +} +``` + + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dataflow-datajob.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dataflow-datajob.md new file mode 100644 index 00000000..a197bc7a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/dataflow-datajob.md @@ -0,0 +1,179 @@ +--- +title: DataFlow & DataJob +slug: /api/tutorials/dataflow-datajob +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/dataflow-datajob.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataFlow & DataJob + +## Why Would You Use DataFlow and DataJob? + +The DataFlow and DataJob entities are used to represent data processing pipelines and jobs within a data ecosystem. They allow users to define, manage, and monitor the flow of data through various stages of processing, from ingestion to transformation and storage. + +### Goal Of This Guide + +This guide will show you how to + +- Create a DataFlow. +- Create a Datajob with a DataFlow. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create DataFlow + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataflow_create.py +from datahub.metadata.urns import TagUrn +from datahub.sdk import DataFlow, DataHubClient + +client = DataHubClient.from_env() + +dataflow = DataFlow( + name="example_dataflow", + platform="airflow", + description="airflow pipeline for production", + tags=[TagUrn(name="production"), TagUrn(name="data_engineering")], +) + +client.entities.upsert(dataflow) + +``` + + + + +## Create DataJob + +DataJob must be associated with a DataFlow. You can create a DataJob by providing the DataFlow object or the DataFlow URN and its platform instance. + + + + +```python +# Inlined from /metadata-ingestion/examples/library/datajob_create_full.py +from datahub.metadata.urns import DatasetUrn, TagUrn +from datahub.sdk import DataFlow, DataHubClient, DataJob + +client = DataHubClient.from_env() + +# datajob will inherit the platform and platform instance from the flow + +dataflow = DataFlow( + platform="airflow", + name="example_dag", + platform_instance="PROD", + description="example dataflow", + tags=[TagUrn(name="tag1"), TagUrn(name="tag2")], +) + +datajob = DataJob( + name="example_datajob", + flow=dataflow, + inlets=[ + DatasetUrn(platform="hdfs", name="dataset1", env="PROD"), + ], + outlets=[ + DatasetUrn(platform="hdfs", name="dataset2", env="PROD"), + ], +) + +client.entities.upsert(dataflow) +client.entities.upsert(datajob) + +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/datajob_create_with_flow_urn.py +from datahub.metadata.urns import DataFlowUrn, DatasetUrn +from datahub.sdk import DataHubClient, DataJob + +client = DataHubClient.from_env() + +# datajob will inherit the platform and platform instance from the flow + +datajob = DataJob( + name="example_datajob", + flow_urn=DataFlowUrn( + orchestrator="airflow", + flow_id="example_dag", + cluster="PROD", + ), + platform_instance="PROD", + inlets=[ + DatasetUrn(platform="hdfs", name="dataset1", env="PROD"), + ], + outlets=[ + DatasetUrn(platform="hdfs", name="dataset2", env="PROD"), + ], +) + +client.entities.upsert(datajob) + +``` + + + + +## Read DataFlow + +```python +# Inlined from /metadata-ingestion/examples/library/dataflow_read.py +from datahub.sdk import DataFlowUrn, DataHubClient + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use DataFlowUrn.from_string(...) +dataflow_urn = DataFlowUrn("airflow", "example_dataflow_id") + +dataflow_entity = client.entities.get(dataflow_urn) +print("DataFlow name:", dataflow_entity.name) +print("DataFlow platform:", dataflow_entity.platform) +print("DataFlow description:", dataflow_entity.description) + +``` + +#### Example Output + +```python +>> DataFlow name: example_dataflow +>> DataFlow platform: urn:li:dataPlatform:airflow +>> DataFlow description: airflow pipeline for production +``` + +## Read DataJob + +```python +# Inlined from /metadata-ingestion/examples/library/datajob_read.py +from datahub.sdk import DataHubClient, DataJobUrn + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use DataJobUrn.from_string(...) +datajob_urn = DataJobUrn("airflow", "example_dag", "example_datajob_id") + +datajob_entity = client.entities.get(datajob_urn) +print("DataJob name:", datajob_entity.name) +print("DataJob Flow URN:", datajob_entity.flow_urn) +print("DataJob description:", datajob_entity.description) + +``` + +#### Example Output + +```python +>> DataJob name: example_datajob +>> DataJob Flow URN: urn:li:dataFlow:(airflow,PROD.example_dag,PROD) +>> DataJob description: example datajob +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/datasets.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/datasets.md new file mode 100644 index 00000000..31fa8cf3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/datasets.md @@ -0,0 +1,261 @@ +--- +title: Dataset +slug: /api/tutorials/datasets +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/datasets.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Dataset + +## Why Would You Use Datasets? + +The dataset entity is one the most important entities in the metadata model. They represent collections of data that are typically represented as Tables or Views in a database (e.g. BigQuery, Snowflake, Redshift etc.), Streams in a stream-processing environment (Kafka, Pulsar etc.), bundles of data found as Files or Folders in data lake systems (S3, ADLS, etc.). +For more information about datasets, refer to our [dataset reference](/docs/generated/metamodel/entities/dataset.md). + +### Goal Of This Guide + +This guide will show you how to + +- Create: create a dataset with three columns. +- Delete: delete a dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create Dataset + + + + +> 🚫 Creating a dataset via `graphql` is currently not supported. +> Please check out [API feature comparison table](/docs/api/datahub-apis.md#datahub-api-comparison) for more information. + + + + +```java +# Inlined from /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/DatasetAdd.java +package io.datahubproject.examples; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.CorpuserUrn; +import com.linkedin.common.urn.DataPlatformUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.schema.DateType; +import com.linkedin.schema.OtherSchema; +import com.linkedin.schema.SchemaField; +import com.linkedin.schema.SchemaFieldArray; +import com.linkedin.schema.SchemaFieldDataType; +import com.linkedin.schema.SchemaMetadata; +import com.linkedin.schema.StringType; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import datahub.event.MetadataChangeProposalWrapper; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class DatasetAdd { + + private DatasetAdd() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + DatasetUrn datasetUrn = UrnUtils.toDatasetUrn("hive", "fct_users_deleted", "PROD"); + CorpuserUrn userUrn = new CorpuserUrn("ingestion"); + AuditStamp lastModified = new AuditStamp().setTime(1640692800000L).setActor(userUrn); + + SchemaMetadata schemaMetadata = + new SchemaMetadata() + .setSchemaName("customer") + .setPlatform(new DataPlatformUrn("hive")) + .setVersion(0L) + .setHash("") + .setPlatformSchema( + SchemaMetadata.PlatformSchema.create( + new OtherSchema().setRawSchema("__insert raw schema here__"))) + .setLastModified(lastModified); + + SchemaFieldArray fields = new SchemaFieldArray(); + + SchemaField field1 = + new SchemaField() + .setFieldPath("address.zipcode") + .setType( + new SchemaFieldDataType() + .setType(SchemaFieldDataType.Type.create(new StringType()))) + .setNativeDataType("VARCHAR(50)") + .setDescription( + "This is the zipcode of the address. Specified using extended form and limited to addresses in the United States") + .setLastModified(lastModified); + fields.add(field1); + + SchemaField field2 = + new SchemaField() + .setFieldPath("address.street") + .setType( + new SchemaFieldDataType() + .setType(SchemaFieldDataType.Type.create(new StringType()))) + .setNativeDataType("VARCHAR(100)") + .setDescription("Street corresponding to the address") + .setLastModified(lastModified); + fields.add(field2); + + SchemaField field3 = + new SchemaField() + .setFieldPath("last_sold_date") + .setType( + new SchemaFieldDataType().setType(SchemaFieldDataType.Type.create(new DateType()))) + .setNativeDataType("Date") + .setDescription("Date of the last sale date for this property") + .setLastModified(lastModified); + fields.add(field3); + + schemaMetadata.setFields(fields); + + MetadataChangeProposalWrapper mcpw = + MetadataChangeProposalWrapper.builder() + .entityType("dataset") + .entityUrn(datasetUrn) + .upsert() + .aspect(schemaMetadata) + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcpw, null); + System.out.println(response.get().getResponseContent()); + } +} + +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_schema.py +from datahub.sdk import DataHubClient, Dataset + +client = DataHubClient.from_env() + +dataset = Dataset( + platform="hive", + name="realestate_db.sales", + schema=[ + # tuples of (field name / field path, data type, description) + ( + "address.zipcode", + "varchar(50)", + "This is the zipcode of the address. Specified using extended form and limited to addresses in the United States", + ), + ("address.street", "varchar(100)", "Street corresponding to the address"), + ("last_sold_date", "date", "Date of the last sale date for this property"), + ], +) + +client.entities.upsert(dataset) + +``` + + + + +### Expected Outcomes of Creating Dataset + +You can now see `realestate_db.sales` dataset has been created. + +

+ +

+ +## Delete Dataset + +You may want to delete a dataset if it is no longer needed, contains incorrect or sensitive information, or if it was created for testing purposes and is no longer necessary in production. +It is possible to [delete entities via CLI](/docs/how/delete-metadata.md), but a programmatic approach is necessary for scalability. + +There are two methods of deletion: soft delete and hard delete. +**Soft delete** sets the Status aspect of the entity to Removed, which hides the entity and all its aspects from being returned by the UI. +**Hard delete** physically deletes all rows for all aspects of the entity. + +For more information about soft delete and hard delete, please refer to [Removing Metadata from DataHub](/docs/how/delete-metadata.md#delete-by-urn). + + + + +> 🚫 Hard delete with `graphql` is currently not supported. +> Please check out [API feature comparison table](/docs/api/datahub-apis.md#datahub-api-comparison) for more information. + +```json +mutation batchUpdateSoftDeleted { + batchUpdateSoftDeleted(input: + { urns: ["urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)"], + deleted: true }) +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "batchUpdateSoftDeleted": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation batchUpdateSoftDeleted { batchUpdateSoftDeleted(input: { deleted: true, urns: [\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)\"] }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "batchUpdateSoftDeleted": true }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_delete.py +from datahub.emitter.mce_builder import make_dataset_urn +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +graph = DataHubGraph( + config=DatahubClientConfig( + server="http://localhost:8080", + ) +) + +dataset_urn = make_dataset_urn(name="fct_users_created", platform="hive") + +# Soft-delete the dataset. +graph.delete_entity(urn=dataset_urn, hard=False) + +print(f"Deleted dataset {dataset_urn}") + +``` + + + + +### Expected Outcomes of Deleting Dataset + +The dataset `fct_users_deleted` has now been deleted, so if you search for a hive dataset named `fct_users_delete`, you will no longer be able to see it. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/deprecation.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/deprecation.md new file mode 100644 index 00000000..1ed9abca --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/deprecation.md @@ -0,0 +1,219 @@ +--- +title: Deprecation +slug: /api/tutorials/deprecation +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/deprecation.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Deprecation + +## Why Would You Deprecate Datasets? + +The Deprecation feature on DataHub indicates the status of an entity. For datasets, keeping the deprecation status up-to-date is important to inform users and downstream systems of changes to the dataset's availability or reliability. By updating the status, you can communicate changes proactively, prevent issues and ensure users are always using highly trusted data assets. + +### Goal Of This Guide + +This guide will show you how to read or update deprecation status of a dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +Before updating deprecation, you need to ensure the targeted dataset is already present in your datahub. +If you attempt to manipulate entities that do not exist, your operation will fail. +In this guide, we will be using data from a sample ingestion. +::: + +## Read Deprecation + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)") { + deprecation { + deprecated + decommissionTime + } + } +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "dataset": { + "deprecation": { + "deprecated": false, + "decommissionTime": null + } + } + }, + "extensions": {} +} +``` + + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "{ dataset(urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\") { deprecation { deprecated decommissionTime } } }", "variables":{} }' +``` + +Expected Response: + +```json +{ + "data": { + "dataset": { + "deprecation": { "deprecated": false, "decommissionTime": null } + } + }, + "extensions": {} +} +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_query_deprecation.py +from typing import Optional, Tuple + +from datahub.metadata.schema_classes import DeprecationClass +from datahub.sdk import DataHubClient, DatasetUrn + + +def query_dataset_deprecation( + client: DataHubClient, dataset_urn: DatasetUrn +) -> Tuple[bool, Optional[str], Optional[int]]: + """ + Query the deprecation status of a dataset. + + Args: + client: DataHub client to use for the query + dataset_urn: URN of the dataset to check + + Returns: + Tuple of (is_deprecated, deprecation_note, decommission_time_millis) + """ + dataset = client.entities.get(dataset_urn) + + deprecation = dataset._get_aspect(DeprecationClass) + if deprecation and deprecation.deprecated: + return (True, deprecation.note, deprecation.decommissionTime) + return (False, None, None) + + +def main(client: Optional[DataHubClient] = None) -> None: + """ + Main function to query dataset deprecation example. + + Args: + client: Optional DataHub client (for testing). If not provided, creates one from env. + """ + client = client or DataHubClient.from_env() + + dataset_urn = DatasetUrn(platform="hive", name="fct_users_created", env="PROD") + + is_deprecated, note, decommission_time = query_dataset_deprecation( + client, dataset_urn + ) + + if is_deprecated: + print(f"Dataset is deprecated: {note}") + if decommission_time: + print(f"Decommission time: {decommission_time}") + else: + print("Dataset is not deprecated") + + +if __name__ == "__main__": + main() + +``` + + + + +## Update Deprecation + + + + +```json +mutation updateDeprecation { + updateDeprecation(input: { urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", deprecated: true }) +} +``` + +Also note that you can update deprecation status of multiple entities or subresource using `batchUpdateDeprecation`. + +```json +mutation batchUpdateDeprecation { + batchUpdateDeprecation( + input: { + deprecated: true, + resources: [ + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)"} , + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)"} ,] + } + ) +} + +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "updateDeprecation": true + }, + "extensions": {} +} +``` + + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation updateDeprecation { updateDeprecation(input: { deprecated: true, urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "removeTag": true }, "extensions": {} } +``` + + + + + + + + +### Expected Outcomes of Updating Deprecation + +You can now see the dataset `fct_users_created` has been marked as `Deprecated.` + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/descriptions.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/descriptions.md new file mode 100644 index 00000000..f970cb64 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/descriptions.md @@ -0,0 +1,434 @@ +--- +title: Description +slug: /api/tutorials/descriptions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/descriptions.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Description + +## Why Would You Use Description on Dataset? + +Adding a description and related link to a dataset can provide important information about the data, such as its source, collection methods, and potential uses. This can help others understand the context of the data and how it may be relevant to their own work or research. Including a related link can also provide access to additional resources or related datasets, further enriching the information available to users. + +### Goal Of This Guide + +This guide will show you how to + +- Read dataset description: read a description of a dataset. +- Read column description: read a description of columns of a dataset`. +- Add dataset description: add a description and a link to dataset. +- Add column description: add a description to a column of a dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +Before adding a description, you need to ensure the targeted dataset is already present in your datahub. +If you attempt to manipulate entities that do not exist, your operation will fail. +In this guide, we will be using data from sample ingestion. +::: + +In this example, we will add a description to `user_name `column of a dataset `fct_users_deleted`. + +## Read Description on Dataset + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)") { + properties { + description + } + } +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "dataset": { + "properties": { + "description": "table containing all the users deleted on a single day" + } + } + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "query { dataset(urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)\") { properties { description } } }", "variables":{}}' +``` + +Expected Response: + +```json +{ + "data": { + "dataset": { + "properties": { + "description": "table containing all the users deleted on a single day" + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_query_description.py +from datahub.sdk import DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="fct_users_created", env="PROD") +) + +# Print the dataset description +print(dataset.description) + +``` + + + + +## Read Description on Columns + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)") { + schemaMetadata { + fields { + fieldPath + description + } + } + } +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "dataset": { + "schemaMetadata": { + "fields": [ + { + "fieldPath": "user_name", + "description": "Name of the user who was deleted" + }, + ... + { + "fieldPath": "deletion_reason", + "description": "Why the user chose to deactivate" + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "query { dataset(urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)\") { schemaMetadata { fields { fieldPath description } } } }", "variables":{}}' +``` + +Expected Response: + +```json +{ + "data": { + "dataset": { + "schemaMetadata": { + "fields": [ + { + "fieldPath": "user_name", + "description": "Name of the user who was deleted" + }, + { + "fieldPath": "timestamp", + "description": "Timestamp user was deleted at" + }, + { "fieldPath": "user_id", "description": "Id of the user deleted" }, + { + "fieldPath": "browser_id", + "description": "Cookie attached to identify the browser" + }, + { + "fieldPath": "session_id", + "description": "Cookie attached to identify the session" + }, + { + "fieldPath": "deletion_reason", + "description": "Why the user chose to deactivate" + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_query_description_on_columns.py +from datahub.sdk import DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="fct_users_created", env="PROD") +) + +# Print descriptions for each column +column_descriptions = {} +for field in dataset.schema: + column_descriptions[field.field_path] = field.description + +print(column_descriptions) + +``` + + + + +## Add Description on Dataset + + + + +```graphql +mutation updateDataset { + updateDataset( + urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + input: { + editableProperties: { + description: "## The Real Estate Sales Dataset\nThis is a really important Dataset that contains all the relevant information about sales that have happened organized by address.\n" + } + institutionalMemory: { + elements: { + author: "urn:li:corpuser:jdoe" + url: "https://wikipedia.com/real_estate" + description: "This is the definition of what real estate means" + } + } + } + ) { + urn + } +} +``` + +Expected Response: + +```json +{ + "data": { + "updateDataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + } + }, + "extensions": {} +} +``` + + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "query": "mutation updateDataset { updateDataset( urn:\"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\", input: { editableProperties: { description: \"## The Real Estate Sales Dataset\nThis is a really important Dataset that contains all the relevant information about sales that have happened organized by address.\n\" } institutionalMemory: { elements: { author: \"urn:li:corpuser:jdoe\", url: \"https://wikipedia.com/real_estate\", description: \"This is the definition of what real estate means\" } } } ) { urn } }", + "variables": {} +}' +``` + +Expected Response: + +```json +{ + "data": { + "updateDataset": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_documentation.py +from datahub.sdk import DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get(DatasetUrn(platform="hive", name="realestate_db.sales")) + +# Add dataset documentation +documentation = """## The Real Estate Sales Dataset +This is a really important Dataset that contains all the relevant information about sales that have happened organized by address. +""" +dataset.set_description(documentation) + +# Add link to institutional memory +dataset.add_link( + ( + "https://wikipedia.com/real_estate", + "This is the definition of what real estate means", # link description + ) +) + +client.entities.update(dataset) + +``` + + + + +### Expected Outcomes of Adding Description on Dataset + +You can now see the description is added to `fct_users_deleted`. + +

+ +

+ +## Add Description on Column + + + + +```json +mutation updateDescription { + updateDescription( + input: { + description: "Name of the user who was deleted. This description is updated via GrpahQL.", + resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + subResource: "user_name", + subResourceType:DATASET_FIELD + } + ) +} +``` + +Note that you can use general markdown in `description`. For example, you can do the following. + +```json +mutation updateDescription { + updateDescription( + input: { + description: """ + ### User Name + The `user_name` column is a primary key column that contains the name of the user who was deleted. + """, + resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)", + subResource: "user_name", + subResourceType:DATASET_FIELD + } + ) +} +``` + +`updateDescription` currently only supports Dataset Schema Fields, Containers. +For more information about the `updateDescription` mutation, please refer to [updateLineage](/docs/graphql/mutations/#updateDescription). + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "updateDescription": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation updateDescription { updateDescription ( input: { description: \"Name of the user who was deleted. This description is updated via GrpahQL.\", resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)\", subResource: \"user_name\", subResourceType:DATASET_FIELD }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "updateDescription": true }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_column_documentation.py +from datahub.sdk import DataHubClient, Dataset, DatasetUrn + +client = DataHubClient.from_env() + +dataset: Dataset = client.entities.get( + DatasetUrn.from_string( + "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + ) +) + +dataset["user_name"].set_description( + "Name of the user who was deleted. This description was updated via the Python SDK." +) + +client.entities.update(dataset) + +``` + + + + +### Expected Outcomes of Adding Description on Column + +You can now see column description is added to `user_name` column of `fct_users_deleted`. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/documents.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/documents.md new file mode 100644 index 00000000..dc70f34c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/documents.md @@ -0,0 +1,1022 @@ +--- +title: Document +slug: /api/tutorials/documents +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/documents.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Document + +## Why Would You Use Documents? + +Documents in DataHub are content-indexed resources that can store knowledge, documentation, FAQs, tutorials, and other textual content. They provide a centralized place to manage and search through organizational knowledge, making them accessible to both humans and AI systems. + +Documents support rich metadata including: + +- **Searchable content** with full-text search capabilities +- **Categorization** via types, domains, and owners +- **Visibility control** to show/hide documents in global search and navigation +- **Relationships** to data assets (datasets, dashboards, charts, etc.) +- **Hierarchical organization** through parent-child relationships + +### Types of Documents + +DataHub supports two types of documents: + +1. **Native Documents**: Created and stored directly in DataHub. Full content is indexed and searchable. Use `Document.create_document()` to create these. + +2. **External Documents**: References to documents stored in external systems (Notion, Confluence, Google Docs, etc.). These link to the original content via URL. Use `Document.create_external_document()` to create these. + +### Document Visibility + +Documents can be configured to: + +- **Show in global context** (default): Appear in global search results and the knowledge base sidebar +- **Hide from global context**: Only accessible through related assets. This is useful for: + - Documentation specific to a single dataset + - Context documents for AI agents + - Private notes attached to assets + +### Goal Of This Guide + +This guide will show you how to: + +- Create native and external documents +- Control document visibility +- Link documents to data assets +- Update document contents and metadata +- Publish and unpublish documents +- Delete documents + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create Document + +### Native Document + +Native documents are stored directly in DataHub with full content indexing. + + + + +```graphql +mutation createDocument { + createDocument( + input: { + id: "my-tutorial-doc" + contents: { + text: "# Getting Started with DataHub\n\nThis tutorial will help you get started..." + } + title: "DataHub Tutorial" + subType: "Tutorial" + state: PUBLISHED + } + ) +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "createDocument": "urn:li:document:my-tutorial-doc" + }, + "extensions": {} +} +``` + + + + +```python +from datahub.sdk import DataHubClient, Document + +client = DataHubClient.from_env() + +# Create a native document +doc = Document.create_document( + id="getting-started-tutorial", + title="Getting Started with DataHub", + text="# Getting Started with DataHub\n\nThis tutorial will help you get started...", + subtype="Tutorial", +) + +client.entities.upsert(doc) +print(f"Created document: {doc.urn}") +``` + + + + +### External Document + +External documents reference content stored in other platforms like Notion or Confluence. + + + + +```graphql +mutation createExternalDocument { + createDocument( + input: { + id: "notion-handbook" + contents: { text: "Summary for search indexing..." } + title: "Engineering Handbook" + subType: "Reference" + state: PUBLISHED + } + ) +} +``` + + + + +```python +from datahub.sdk import DataHubClient, Document + +client = DataHubClient.from_env() + +# Create an external document (from Notion) +doc = Document.create_external_document( + id="notion-engineering-handbook", + title="Engineering Handbook", + platform="urn:li:dataPlatform:notion", + external_url="https://notion.so/team/engineering-handbook", + external_id="notion-page-abc123", + text="Summary of the handbook for search...", # Optional + owners=["urn:li:corpuser:engineering-lead"], +) + +client.entities.upsert(doc) +print(f"Created external document: {doc.urn}") +``` + + + + +### Document Hidden from Global Context + +Documents can be hidden from global search and sidebar navigation. They remain accessible through related assets - useful for AI agent context or asset-specific documentation. + + + + +```graphql +mutation createHiddenDocument { + createDocument( + input: { + id: "dataset-context-doc" + contents: { text: "Context about the orders dataset for AI agents..." } + title: "Orders Dataset Context" + settings: { showInGlobalContext: false } + relatedAssets: [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,orders,PROD)" + ] + } + ) +} +``` + + + + +```python +from datahub.sdk import DataHubClient, Document + +client = DataHubClient.from_env() + +# Create a document hidden from global context +# Only accessible via the related asset - useful for AI agents +doc = Document.create_document( + id="orders-dataset-context", + title="Orders Dataset Context", + text="# Context for AI Agents\n\nThe orders dataset contains daily summaries...", + show_in_global_context=False, # Hidden from global search/sidebar + related_assets=["urn:li:dataset:(urn:li:dataPlatform:snowflake,orders,PROD)"], +) + +client.entities.upsert(doc) +print(f"Created AI-only context document: {doc.urn}") +``` + + + + +### Document with Full Metadata + + + + +```graphql +mutation createDocumentWithMetadata { + createDocument( + input: { + id: "faq-data-quality" + contents: { + text: "# Data Quality FAQ\n\n## Q: How do we measure data quality?\n\nA: We use..." + } + title: "Data Quality FAQ" + subType: "FAQ" + state: PUBLISHED + relatedAssets: [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_table,PROD)" + ] + owners: [{ owner: "urn:li:corpuser:john", type: TECHNICAL_OWNER }] + } + ) +} +``` + + + + +```python +from datahub.sdk import DataHubClient, Document + +client = DataHubClient.from_env() + +doc = Document.create_document( + id="faq-data-quality", + title="Data Quality FAQ", + text="# Data Quality FAQ\n\n## Q: How do we measure data quality?\n\nA: We use...", + subtype="FAQ", + related_assets=["urn:li:dataset:(urn:li:dataPlatform:snowflake,my_table,PROD)"], + owners=["urn:li:corpuser:john"], + domain="urn:li:domain:engineering", + tags=["urn:li:tag:important"], + custom_properties={"team": "data-platform", "version": "1.0"}, +) + +client.entities.upsert(doc) +print(f"Created document with metadata: {doc.urn}") +``` + + + + +## Update Document + +Update the contents, title, or visibility of an existing document. + + + + +```graphql +mutation updateDocumentContents { + updateDocumentContents( + input: { + urn: "urn:li:document:my-tutorial-doc" + contents: { + text: "# Updated Getting Started Guide\n\nThis is the updated content..." + } + } + ) +} +``` + +Update the title: + +```graphql +mutation updateDocumentTitle { + updateDocumentContents( + input: { + urn: "urn:li:document:my-tutorial-doc" + title: "Updated Tutorial Title" + } + ) +} +``` + +Update visibility settings: + +```graphql +mutation updateDocumentSettings { + updateDocumentSettings( + input: { + urn: "urn:li:document:my-tutorial-doc" + showInGlobalContext: false + } + ) +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/update_document.py +# Inlined from metadata-ingestion/examples/library/update_document.py +"""Example: Updating documents using the DataHub SDK. + +This example demonstrates how to retrieve, modify, and update documents. +""" + +from datahub.metadata.urns import DocumentUrn +from datahub.sdk import DataHubClient + +# Initialize the client +client = DataHubClient.from_env() + +# ============================================================================ +# Example 1: Retrieve and update a document's content +# ============================================================================ +# First, get the existing document from DataHub +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + # Update the text content + doc.set_text("# Updated Getting Started Guide\n\nThis is the updated content...") + + # Save changes + client.entities.upsert(doc) + print("Document contents updated!") + +# ============================================================================ +# Example 2: Update document title +# ============================================================================ +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + doc.set_title("Updated Tutorial Title") + client.entities.upsert(doc) + print("Document title updated!") + +# ============================================================================ +# Example 3: Update both contents and title with method chaining +# ============================================================================ +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + # Method chaining for multiple updates + doc.set_text("# Comprehensive Guide\n\nFully updated content...").set_title( + "Comprehensive DataHub Guide" + ) + client.entities.upsert(doc) + print("Document fully updated!") + +# ============================================================================ +# Example 4: Update document visibility (global context) +# ============================================================================ +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + # Hide document from global search and sidebar + # Useful for making documents only accessible via related assets (e.g., for AI agents) + doc.hide_from_global_context() + client.entities.upsert(doc) + print("Document hidden from global context!") + + # Later, show it again in global search/sidebar + doc.show_in_global_search() + client.entities.upsert(doc) + print("Document visible in global context again!") + +# ============================================================================ +# Example 5: Update related assets and documents +# ============================================================================ +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + # Add related assets - the document becomes accessible from these assets + doc.add_related_asset("urn:li:dataset:(urn:li:dataPlatform:snowflake,users,PROD)") + doc.add_related_asset("urn:li:dataset:(urn:li:dataPlatform:snowflake,orders,PROD)") + + # Add a related document + doc.add_related_document("urn:li:document:related-guide") + + client.entities.upsert(doc) + print("Related entities updated!") + +# ============================================================================ +# Example 6: Move a document to a different parent +# ============================================================================ +# Documents can be organized hierarchically. Moving a document changes its parent. +doc = client.entities.get(DocumentUrn("child-section")) + +if doc: + # Check current parent + print(f"Current parent: {doc.parent_document}") + + # Move to a new parent document + doc.set_parent_document("urn:li:document:new-parent-guide") + client.entities.upsert(doc) + print("Document moved to new parent!") + + # Remove from hierarchy (make it a top-level document) + doc.set_parent_document(None) + client.entities.upsert(doc) + print("Document is now a top-level document!") + +# ============================================================================ +# Example 7: Update document status +# ============================================================================ +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + # Publish the document + doc.publish() + client.entities.upsert(doc) + print("Document published!") + + # Later, unpublish it + doc.unpublish() + client.entities.upsert(doc) + print("Document unpublished!") + +``` + + + + +## Search Documents + +Search through documents with various filters. + + + + +```graphql +query searchDocuments { + searchDocuments( + input: { query: "data quality", types: ["FAQ"], start: 0, count: 10 } + ) { + total + documents { + urn + type + subType + info { + title + status { + state + } + contents { + text + } + } + } + } +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/search_documents.py +# Inlined from metadata-ingestion/examples/library/search_documents.py +"""Example: Searching documents using the DataHub SDK. + +This example demonstrates how to search for documents using the DataHub SDK. +""" + +from datahub.sdk import DataHubClient, FilterDsl + +# Initialize the client +client = DataHubClient.from_env() + +# ============================================================================ +# Example 1: Search for all documents +# ============================================================================ +# Use get_urns with entity type filter to find documents +document_urns = client.search.get_urns( + filter=FilterDsl.entity_type("document"), +) + +print("All documents:") +for urn in document_urns: + print(f" - {urn}") + +# ============================================================================ +# Example 2: Search with a text query +# ============================================================================ +# Search for documents matching "data quality" +document_urns = client.search.get_urns( + query="data quality", + filter=FilterDsl.entity_type("document"), +) + +print("\nDocuments matching 'data quality':") +for urn in document_urns: + print(f" - {urn}") + +# ============================================================================ +# Example 3: Search within a specific domain +# ============================================================================ +document_urns = client.search.get_urns( + filter=FilterDsl.and_( + FilterDsl.entity_type("document"), + FilterDsl.domain("urn:li:domain:engineering"), + ), +) + +print("\nDocuments in engineering domain:") +for urn in document_urns: + print(f" - {urn}") + +# ============================================================================ +# Example 4: Search with tags +# ============================================================================ +document_urns = client.search.get_urns( + filter=FilterDsl.and_( + FilterDsl.entity_type("document"), + FilterDsl.tag("urn:li:tag:important"), + ), +) + +print("\nDocuments with 'important' tag:") +for urn in document_urns: + print(f" - {urn}") + +``` + + + + +## Get Document + +Retrieve the full contents and metadata of a specific document. + + + + +```graphql +query getDocument { + document(urn: "urn:li:document:my-tutorial-doc") { + urn + type + subType + info { + title + source { + sourceType + externalUrl + externalId + } + status { + state + } + contents { + text + } + relatedAssets { + asset { + urn + } + } + relatedDocuments { + document { + urn + } + } + parentDocument { + document { + urn + } + } + } + settings { + showInGlobalContext + } + } +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/get_document.py +# Inlined from metadata-ingestion/examples/library/get_document.py +"""Example: Retrieving documents using the DataHub SDK. + +This example demonstrates how to get documents and access their properties. +""" + +from datahub.metadata.urns import DocumentUrn +from datahub.sdk import DataHubClient + +# Initialize the client +client = DataHubClient.from_env() + +# ============================================================================ +# Example 1: Get a document by URN +# ============================================================================ +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + print(f"Document: {doc.title}") + print(f"URN: {doc.urn}") + print(f"Status: {doc.status}") + print(f"Subtype: {doc.subtype}") + print(f"\nContents:\n{doc.text}") + + # Check document type (native vs external) + if doc.is_native: + print("\nThis is a native document (stored in DataHub)") + elif doc.is_external: + print("\nThis is an external document") + print(f" External URL: {doc.external_url}") + print(f" External ID: {doc.external_id}") + + # Check visibility + if doc.show_in_global_context: + print("Visible in global search and sidebar") + else: + print("Hidden from global context (accessible only via related assets)") + + # Check related entities + if doc.related_assets: + print(f"\nRelated assets: {len(doc.related_assets)}") + for asset in doc.related_assets: + print(f" - {asset}") + + if doc.related_documents: + print(f"\nRelated documents: {len(doc.related_documents)}") + for related_doc in doc.related_documents: + print(f" - {related_doc}") + + # Check parent document + if doc.parent_document: + print(f"\nParent document: {doc.parent_document}") + + # Get custom properties + if doc.custom_properties: + print("\nCustom properties:") + for key, value in doc.custom_properties.items(): + print(f" {key}: {value}") +else: + print("Document not found") + +# ============================================================================ +# Example 2: Check if a document exists +# ============================================================================ +doc = client.entities.get(DocumentUrn("might-not-exist")) + +if doc is not None: + print(f"\nDocument exists: {doc.urn}") +else: + print("\nDocument does not exist") + +``` + + + + +## Publish/Unpublish Document + +Control whether a document is visible to users. + + + + +Publish a document: + +```graphql +mutation publishDocument { + updateDocumentStatus( + input: { urn: "urn:li:document:my-tutorial-doc", state: PUBLISHED } + ) +} +``` + +Unpublish a document: + +```graphql +mutation unpublishDocument { + updateDocumentStatus( + input: { urn: "urn:li:document:my-tutorial-doc", state: UNPUBLISHED } + ) +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/publish_document.py +# Inlined from metadata-ingestion/examples/library/publish_document.py +"""Example: Publishing and unpublishing documents using the DataHub SDK. + +This example demonstrates how to control document visibility by +publishing or unpublishing documents. +""" + +from datahub.metadata.urns import DocumentUrn +from datahub.sdk import DataHubClient, Document + +# Initialize the client +client = DataHubClient.from_env() + +# ============================================================================ +# Example 1: Publish a document +# ============================================================================ +# Get the document +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + # Publish makes the document visible to users + doc.publish() + client.entities.upsert(doc) + print(f"Document published! Status: {doc.status}") + +# ============================================================================ +# Example 2: Unpublish a document +# ============================================================================ +doc = client.entities.get(DocumentUrn("my-tutorial-doc")) + +if doc: + # Unpublish hides the document from general users + doc.unpublish() + client.entities.upsert(doc) + print(f"Document unpublished! Status: {doc.status}") + +# ============================================================================ +# Example 3: Create a document with specific status +# ============================================================================ +# Create as published (default) +published_doc = Document.create_document( + id="new-published-doc", + title="New Published Document", + text="This document is published from the start.", +) +client.entities.upsert(published_doc) +print(f"Created published document: {published_doc.urn}") + +# Create as unpublished (work in progress) +unpublished_doc = Document.create_document( + id="new-unpublished-doc", + title="Work in Progress Document", + text="This document is not yet published.", + status="UNPUBLISHED", +) +client.entities.upsert(unpublished_doc) +print(f"Created unpublished document: {unpublished_doc.urn}") + +``` + + + + +## Delete Document + +Remove a document from DataHub. + + + + +```graphql +mutation deleteDocument { + deleteDocument(urn: "urn:li:document:my-tutorial-doc") +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/delete_document.py +# Inlined from metadata-ingestion/examples/library/delete_document.py +"""Example: Deleting documents using the DataHub SDK. + +This example demonstrates how to delete documents from DataHub. +""" + +from datahub.metadata.urns import DocumentUrn +from datahub.sdk import DataHubClient + +# Initialize the client +client = DataHubClient.from_env() + +# ============================================================================ +# Example 1: Delete a document by URN +# ============================================================================ +doc_urn = DocumentUrn("my-tutorial-doc") + +# First check if it exists +doc = client.entities.get(doc_urn) + +if doc: + # Delete the document + client.entities.delete(str(doc_urn)) + print(f"Document deleted: {doc_urn}") +else: + print(f"Document not found: {doc_urn}") + +# ============================================================================ +# Example 2: Delete multiple documents +# ============================================================================ +doc_ids_to_delete = [ + "doc-1", + "doc-2", + "doc-3", +] + +for doc_id in doc_ids_to_delete: + doc_urn = DocumentUrn(doc_id) + doc = client.entities.get(doc_urn) + if doc: + client.entities.delete(str(doc_urn)) + print(f"Deleted: {doc_urn}") + else: + print(f"Not found (skipping): {doc_urn}") + +print("Cleanup complete!") + +``` + + + + +## Advanced Operations + +### Link Related Assets + +Associate a document with data assets. Documents linked to assets can be accessed from those assets even when hidden from global context. + + + + +```graphql +mutation updateRelatedEntities { + updateDocumentRelatedEntities( + input: { + urn: "urn:li:document:my-doc" + relatedAssets: [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,my_table,PROD)" + "urn:li:dashboard:(looker,dashboard1)" + ] + relatedDocuments: ["urn:li:document:related-doc"] + } + ) +} +``` + + + + +```python +from datahub.sdk import DataHubClient, Document + +client = DataHubClient.from_env() + +doc = client.entities.get("urn:li:document:my-doc", Document) +if doc: + # Add related assets + doc.add_related_asset("urn:li:dataset:(urn:li:dataPlatform:snowflake,my_table,PROD)") + doc.add_related_asset("urn:li:dashboard:(looker,dashboard1)") + + # Add related documents + doc.add_related_document("urn:li:document:related-doc") + + client.entities.upsert(doc) + print("Related entities updated!") +``` + + + + +### Update Document Sub-Type + +Change the sub-type (e.g., "FAQ", "Tutorial", "Runbook") of a document: + + + + +```graphql +mutation updateDocumentSubType { + updateDocumentSubType( + input: { urn: "urn:li:document:my-doc", subType: "Reference" } + ) +} +``` + + + + +```python +from datahub.sdk import DataHubClient, Document + +client = DataHubClient.from_env() + +doc = client.entities.get("urn:li:document:my-doc", Document) +if doc: + doc.set_subtype("Reference") + client.entities.upsert(doc) + print(f"Sub-type updated: {doc.subtype}") +``` + + + + +### Move Document + +Move a document to a different parent (for hierarchical organization): + + + + +```graphql +mutation moveDocument { + moveDocument( + input: { + urn: "urn:li:document:child-doc" + newParent: "urn:li:document:new-parent" + } + ) +} +``` + + + + +```python +from datahub.sdk import DataHubClient, Document + +client = DataHubClient.from_env() + +doc = client.entities.get("urn:li:document:child-doc", Document) +if doc: + # Move to a new parent + doc.set_parent_document("urn:li:document:new-parent") + client.entities.upsert(doc) + print(f"Document moved! New parent: {doc.parent_document}") + + # Or make it a top-level document (no parent) + doc.set_parent_document(None) + client.entities.upsert(doc) + print("Document is now a top-level document!") +``` + + + + +## Python SDK Reference + +The Document SDK provides the following methods: + +### Creation Methods + +| Method | Description | +| ---------------------------------------- | ------------------------------------------ | +| `Document.create_document(...)` | Create a native document stored in DataHub | +| `Document.create_external_document(...)` | Create a reference to an external document | + +### Content & Metadata + +| Method | Description | +| -------------------------------------- | ------------------------------------------ | +| `doc.title` / `doc.set_title(...)` | Get/set the document title | +| `doc.text` / `doc.set_text(...)` | Get/set the document text content | +| `doc.subtype` / `doc.set_subtype(...)` | Get/set the sub-type (FAQ, Tutorial, etc.) | +| `doc.custom_properties` | Get the custom properties dictionary | +| `doc.set_custom_property(key, value)` | Set a single custom property | + +### Visibility & Lifecycle + +| Method | Description | +| ------------------------------------ | ----------------------------------------- | +| `doc.status` / `doc.set_status(...)` | Get/set PUBLISHED or UNPUBLISHED status | +| `doc.publish()` / `doc.unpublish()` | Publish or unpublish the document | +| `doc.show_in_global_context` | Check if visible in global search/sidebar | +| `doc.hide_from_global_context()` | Hide from global context (AI-only access) | +| `doc.show_in_global_search()` | Show in global context | + +### Relationships + +| Method | Description | +| -------------------------------------------------------------------- | --------------------------------- | +| `doc.related_assets` | Get list of related asset URNs | +| `doc.add_related_asset(...)` / `doc.remove_related_asset(...)` | Add/remove a related asset | +| `doc.related_documents` | Get list of related document URNs | +| `doc.add_related_document(...)` / `doc.remove_related_document(...)` | Add/remove a related document | +| `doc.parent_document` / `doc.set_parent_document(...)` | Get/set parent for hierarchy | + +### Source Information + +| Method | Description | +| ------------------ | ------------------------------------------ | +| `doc.is_native` | Check if this is a native DataHub document | +| `doc.is_external` | Check if this is an external reference | +| `doc.external_url` | Get the external URL (external docs only) | +| `doc.external_id` | Get the external system ID | + +### Metadata (via mixins) + +| Method | Description | +| -------------------------------------------- | ------------------ | +| `doc.add_tag(...)` / `doc.set_tags(...)` | Add tags | +| `doc.add_owner(...)` / `doc.set_owners(...)` | Add owners | +| `doc.set_domain(...)` | Set the domain | +| `doc.add_term(...)` / `doc.set_terms(...)` | Add glossary terms | + +For more examples, see: + +- [Python SDK Examples](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/domains.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/domains.md new file mode 100644 index 00000000..84a5fc17 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/domains.md @@ -0,0 +1,413 @@ +--- +title: Domains +slug: /api/tutorials/domains +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/domains.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Domains + +## Why Would You Use Domains? + +Domains are curated, top-level folders or categories where related assets can be explicitly grouped. Management of Domains can be centralized, or distributed out to Domain owners Currently, an asset can belong to only one Domain at a time. +For more information about domains, refer to [About DataHub Domains](/docs/domains.md). + +### Goal Of This Guide + +This guide will show you how to + +- Create a domain. +- Read domains attached to a dataset. +- Add a dataset to a domain +- Remove the domain from a dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create Domain + + + + +```json +mutation createDomain { + createDomain(input: { name: "Marketing", description: "Entities related to the marketing department" }) +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "createDomain": "" + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation createDomain { createDomain(input: { name: \"Marketing\", description: \"Entities related to the marketing department.\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "createDomain": "" }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/domain_create.py +import logging +import os + +from datahub.emitter.mce_builder import make_domain_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.metadata.schema_classes import DomainPropertiesClass + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +# Get DataHub connection details from environment +gms_server = os.getenv("DATAHUB_GMS_URL", "http://localhost:8080") +token = os.getenv("DATAHUB_GMS_TOKEN") + +domain_urn = make_domain_urn("marketing") +domain_properties_aspect = DomainPropertiesClass( + name="Marketing", description="Entities related to the marketing department" +) + +event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=domain_urn, + aspect=domain_properties_aspect, +) + +rest_emitter = DatahubRestEmitter(gms_server=gms_server, token=token) +rest_emitter.emit(event) +log.info(f"Created domain {domain_urn}") + +``` + + + + +### Expected Outcomes of Creating Domain + +You can now see `Marketing` domain has been created under `Govern > Domains`. + +

+ +

+ +### Creating a Nested Domain + +You can also create a nested domain, or a domain within another domain. + + + + +```json +mutation createDomain { + createDomain(input: { name: "Verticals", description: "An optional description", parentDomain: "urn:li:domain:marketing" }) +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation createDomain { createDomain(input: { name: \"Verticals\", description: \"Entities related to the verticals sub-domain.\", parentDomain: \"urn:li:domain:marketing\" }) }", "variables":{}}' +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/domain_create_nested.py +import logging +import os + +from datahub.emitter.mce_builder import make_domain_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.metadata.schema_classes import DomainPropertiesClass + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +domain_urn = make_domain_urn("marketing") +domain_properties_aspect = DomainPropertiesClass( + name="Verticals", + description="Entities related to the verticals sub-domain", + parentDomain="urn:li:domain:marketing", +) + +event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=domain_urn, + aspect=domain_properties_aspect, +) + +# Get DataHub connection details from environment +gms_server = os.getenv("DATAHUB_GMS_URL", "http://localhost:8080") +token = os.getenv("DATAHUB_GMS_TOKEN") + +rest_emitter = DatahubRestEmitter(gms_server=gms_server, token=token) +rest_emitter.emit(event) +log.info(f"Created domain {domain_urn}") + +``` + + + + +This query will create a new domain, "Verticals", under the "Marketing" domain. + +## Read Domains + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)") { + domain { + associatedUrn + domain { + urn + properties { + name + } + } + } + } +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "dataset": { + "domain": { + "associatedUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "domain": { + "urn": "urn:li:domain:71b3bf7b-2e3f-4686-bfe1-93172c8c4e10", + "properties": { + "name": "Marketing" + } + } + } + } + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "{ dataset(urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\") { domain { associatedUrn domain { urn properties { name } } } } }", "variables":{}}' +``` + +Expected Response: + +```json +{ + "data": { + "dataset": { + "domain": { + "associatedUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + "domain": { + "urn": "urn:li:domain:71b3bf7b-2e3f-4686-bfe1-93172c8c4e10", + "properties": { "name": "Marketing" } + } + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_query_domain.py +from datahub.sdk import DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="fct_users_created", env="PROD") +) + +# Print the dataset domain +print(dataset.domain) + +``` + + + + +## Add Domains + + + + +```json +mutation setDomain { + setDomain(domainUrn: "urn:li:domain:marketing", entityUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)") +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "setDomain": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation setDomain { setDomain(entityUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", domainUrn: "urn:li:domain:marketing")) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "setDomain": true }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_domain.py +from datahub.metadata.urns import DatasetUrn, DomainUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +dataset = client.entities.get(DatasetUrn(platform="snowflake", name="example_dataset")) + +# If you don't know the domain urn, you can look it up: +# domain_urn = client.resolve.domain(name="marketing") + +# NOTE: This will overwrite the existing domain +dataset.set_domain(DomainUrn(id="marketing")) + +client.entities.update(dataset) + +``` + + + + +### Expected Outcomes of Adding Domain + +You can now see `Marketing` domain has been added to the dataset. + +

+ +

+ +## Remove Domains + + + + +```json +mutation unsetDomain { + unsetDomain( + entityUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ) +} +``` + +Expected Response: + +```python +{ + "data": { + "removeDomain": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation unsetDomain { unsetDomain(entityUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\") }", "variables":{}}' +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_remove_domain_execute_graphql.py +# read-modify-write requires access to the DataHubGraph (RestEmitter is not enough) +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +gms_endpoint = "http://localhost:8080" +graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) + +# Query multiple aspects from entity +query = """ +mutation unsetDomain { + unsetDomain( + entityUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + ) +} +""" +result = graph.execute_graphql(query=query) + +print(result) + +``` + + + + +### Expected Outcomes of Removing Domain + +You can now see a domain `Marketing` has been removed from the `fct_users_created` dataset. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/forms.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/forms.md new file mode 100644 index 00000000..b3be8a6b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/forms.md @@ -0,0 +1,281 @@ +--- +title: Compliance Forms +slug: /api/tutorials/forms +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/forms.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Compliance Forms + +## Why Would You Use Compliance Forms? + +**DataHub Compliance Forms** streamline the process of documenting, annotating, and classifying your most critical Data Assets through a collaborative, crowdsourced approach. + +With Compliance Forms, you can execute large-scale compliance initiatives by assigning tasks (e.g., documentation, tagging, or classification requirements) to the appropriate stakeholders — data owners, stewards, and subject matter experts. + +Learn more about forms in the [Compliance Forms Feature Guide](../../../docs/features/feature-guides/compliance-forms/overview.md). + +### Goal Of This Guide + +This guide will show you how to + +- Create, Update, Read, and Delete a form +- Assign and Remove a form from entities + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed information, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + + + + +Install the relevant CLI version. Forms are available as of CLI version `0.13.1`. The corresponding DataHub Cloud release version is `v0.2.16.5` +Connect to your instance via [init](/docs/cli/#init): + +1. Run `datahub init` to update the instance you want to load into +2. Set the server to your sandbox instance, `https://{your-instance-address}/gms` +3. Set the token to your access token + + + + +## Create a Form + + + + +```graphql +mutation createForm { + createForm( + input: { + id: "metadataInitiative2024" + name: "Metadata Initiative 2024" + description: "How we want to ensure the most important data assets in our organization have all of the most important and expected pieces of metadata filled out" + type: VERIFICATION + prompts: [ + { + id: "123" + title: "retentionTime" + description: "Apply Retention Time structured property to form" + type: STRUCTURED_PROPERTY + structuredPropertyParams: { + urn: "urn:li:structuredProperty:retentionTime" + } + } + ] + actors: { + users: [ + "urn:li:corpuser:jane@email.com" + "urn:li:corpuser:john@email.com" + ] + groups: ["urn:li:corpGroup:team@email.com"] + } + } + ) { + urn + } +} +``` + + + + +Create a yaml file representing the forms you’d like to load. +For example, below file represents a form `123456` You can see the full example [here](https://github.com/datahub-project/datahub/blob/example-yaml-sp/metadata-ingestion/examples/forms/forms.yaml). + +```yaml +- id: 123456 + # urn: "urn:li:form:123456" # optional if id is provided + type: VERIFICATION # Supported Types: COMPLETION(DOCUMENTATION), VERIFICATION + name: "Metadata Initiative 2023" + description: "How we want to ensure the most important data assets in our organization have all of the most important and expected pieces of metadata filled out" + prompts: + - id: "123" + title: "Retention Time" + description: "Apply Retention Time structured property to form" + type: STRUCTURED_PROPERTY + structured_property_id: io.acryl.privacy.retentionTime + required: True # optional, will default to True + entities: # Either pass a list of urns or a group of filters. This example shows a list of urns + urns: + - urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD) + # optionally assign the form to a specific set of users and/or groups + # when omitted, form will be assigned to Asset owners + actors: + users: + - urn:li:corpuser:jane@email.com # note: these should be urns + - urn:li:corpuser:john@email.com + groups: + - urn:li:corpGroup:team@email.com # note: these should be urns +``` + +:::note +Note that the structured properties and related entities should be created before you create the form. +Please refer to the [Structured Properties Tutorial](/docs/api/tutorials/structured-properties.md) for more information. +::: + +You can apply forms to either a list of entity urns, or a list of filters. For a list of entity urns, use this structure: + +``` +entities: +urns: + - urn:li:dataset:... +``` + +For a list of filters, use this structure: + +``` +entities: +filters: + types: + - dataset # you can use entity type name or urn + platforms: + - snowflake # you can use platform name or urn + domains: + - urn:li:domain:finance # you must use domain urn + containers: + - urn:li:container:my_container # you must use container urn +``` + +Note that you can filter to entity types, platforms, domains, and/or containers. + +Use the CLI to create your properties: + +```commandline +datahub forms upsert -f {forms_yaml} +``` + +If successful, you should see `Created form urn:li:form:...` + + + + +## Update Form + + + + +```graphql +mutation updateForm { + updateForm( + input: { + urn: "urn:li:form:metadataInitiative2024" + name: "Metadata Initiative 2024" + description: "How we want to ensure the most important data assets in our organization have all of the most important and expected pieces of metadata filled out" + type: VERIFICATION + promptsToAdd: [ + { + id: "456" + title: "deprecationDate" + description: "Deprecation date for dataset" + type: STRUCTURED_PROPERTY + structuredPropertyParams: { + urn: "urn:li:structuredProperty:deprecationDate" + } + } + ] + promptsToRemove: ["123"] + } + ) { + urn + } +} +``` + + + + +## Read Property Definition + + + + +You can see the properties you created by running the following command: + +```commandline +datahub forms get --urn {urn} +``` + +For example, you can run `datahub forms get --urn urn:li:form:123456`. + +If successful, you should see metadata about your form returned like below. + +```json +{ + "urn": "urn:li:form:123456", + "name": "Metadata Initiative 2023", + "description": "How we want to ensure the most important data assets in our organization have all of the most important and expected pieces of metadata filled out", + "prompts": [ + { + "id": "123", + "title": "Retention Time", + "description": "Apply Retention Time structured property to form", + "type": "STRUCTURED_PROPERTY", + "structured_property_urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ], + "type": "VERIFICATION" +} +``` + + + + +## Delete Form + + + + +```graphql +mutation deleteForm { + deleteForm(input: { urn: "urn:li:form:metadataInitiative2024" }) +} +``` + + + + +## Assign Form to Entities + +For assigning a form to a given list of entities: + + + + +```graphql +mutation batchAssignForm { + batchAssignForm( + input: { + formUrn: "urn:li:form:myform" + entityUrns: ["urn:li:dataset:mydataset1", "urn:li:dataset:mydataset2"] + } + ) +} +``` + + + + +## Remove Form from Entities + +For removing a form from a given list of entities: + + + + +```graphql +mutation batchRemoveForm { + batchRemoveForm( + input: { + formUrn: "urn:li:form:myform" + entityUrns: ["urn:li:dataset:mydataset1", "urn:li:dataset:mydataset2"] + } + ) +} +``` + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/incidents.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/incidents.md new file mode 100644 index 00000000..ce496df6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/incidents.md @@ -0,0 +1,169 @@ +--- +title: Incidents +slug: /api/tutorials/incidents +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/incidents.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Incidents + +## Why Would You Use Incidents APIs? + +The Incidents APIs allow you to raise, retrieve, update and resolve data incidents via API. This is +useful for raising or resolving data incidents programmatically, for example from Airflow, Prefect, or Dagster DAGs. +Incidents are also useful for conditional Circuit Breaking in these pipelines. + +### Goal Of This Guide + +This guide will show you how to raise, retrieve, update and resolve data incidents via API. + +## Prerequisites + +The actor making API calls must have the `Edit Incidents` privileges for the Tables at hand. + +## Raise Incident + +You can raise a new Data Incident for an existing asset using the following APIs. + + + + +```graphql +mutation raiseIncident { + raiseIncident( + input: { + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,public.prod.purchases,PROD)" + type: OPERATIONAL + title: "Data is Delayed" + description: "Data is delayed on May 15, 2024 because of downtime in the Spark Cluster." + } + ) +} +``` + +Where `resourceUrn` is the unique identifier for the data asset (dataset, dashboard, chart, data job, or data flow) you want to raise the incident on. + +Where supported Incident Types include + +- `OPERATIONAL` +- `FRESHNESS` +- `VOLUME` +- `COLUMN` +- `SQL` +- `DATA_SCHEMA` +- `CUSTOM` + +If you see the following response, a unique identifier for the new incident will be returned. + +```json +{ + "data": { + "raiseIncident": "urn:li:incident:new-incident-id" + }, + "extensions": {} +} +``` + + + + + +``` +Python SDK support coming soon! +``` + + + + + +## Get Incidents For Data Asset + +You can use retrieve the incidents and their statuses for a given Data Asset using the following APIs. + + + + +```graphql +query getAssetIncidents { + dataset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,public.prod.purchases,PROD)" + ) { + incidents(state: ACTIVE, start: 0, count: 20) { + start + count + total + incidents { + urn + incidentType + title + description + status { + state + lastUpdated { + time + actor + } + } + } + } + } +} +``` + +Where you can filter for active incidents by passing the `ACTIVE` state and resolved incidents by passing the `RESOLVED` state. +This will return all relevant incidents for the dataset. + + + + + +``` +Python SDK support coming soon! +``` + + + + +## Resolve Incidents + +You can update the status of an incident using the following APIs. + + + + +```graphql +mutation updateIncidentStatus { + updateIncidentStatus( + input: { + state: RESOLVED + message: "The delayed data issue was resolved at 4:55pm on May 15." + } + ) +} +``` + +You can also reopen an incident by updating the state from `RESOLVED` to `ACTIVE`. + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "updateIncidentStatus": true + }, + "extensions": {} +} +``` + + + + + +``` +Python SDK support coming soon! +``` + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/lineage.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/lineage.md new file mode 100644 index 00000000..dc415eb5 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/lineage.md @@ -0,0 +1,619 @@ +--- +title: Lineage +slug: /api/tutorials/lineage +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/lineage.md +--- +# Lineage + +DataHub’s Python SDK allows you to programmatically define and retrieve lineage between metadata entities. With the DataHub Lineage SDK, you can: + +- Add **table-level and column-level lineage** across datasets, data jobs, dashboards, and charts +- Automatically **infer lineage from SQL queries** +- **Read lineage** (upstream or downstream) for a given entity or column +- **Filter lineage results** using structured filters + +## Getting Started + +To use DataHub SDK, you'll need to install [`acryl-datahub`](https://pypi.org/project/acryl-datahub/) and set up a connection to your DataHub instance. Follow the [installation guide](/docs/metadata-ingestion/cli-ingestion#installing-datahub-cli) to get started. + +Connect to your DataHub instance: + +```python +from datahub.sdk import DataHubClient + +client = DataHubClient(server="", token="") +``` + +- **server**: The URL of your DataHub GMS server + - local: `http://localhost:8080` + - hosted: `https:///gms` +- **token**: You'll need to [generate a Personal Access Token](/docs/authentication/personal-access-tokens) from your DataHub instance. + +## Add Lineage + +The `add_lineage()` method allows you to define lineage between two entities. + +### Add Entity Lineage + +You can create lineage between two datasets, data jobs, dashboards, or charts. The `upstream` and `downstream` parameters should be the URNs of the entities you want to link. + +#### Add Entity Lineage Between Datasets + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_dataset_add.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk.main_client import DataHubClient + +client = DataHubClient.from_env() + + +upstream_urn = DatasetUrn(platform="snowflake", name="sales_raw") +downstream_urn = DatasetUrn(platform="snowflake", name="sales_cleaned") +client.lineage.add_lineage(upstream=upstream_urn, downstream=downstream_urn) + +``` + +#### Add Entity Lineage Between Datajobs + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_datajob_to_datajob.py +from datahub.metadata.urns import DataFlowUrn, DataJobUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +dataflow_urn = DataFlowUrn( + orchestrator="airflow", flow_id="data_pipeline", cluster="PROD" +) + +client.lineage.add_lineage( + upstream=DataJobUrn(flow=dataflow_urn, job_id="data_job_1"), + downstream=DataJobUrn(flow=dataflow_urn, job_id="data_job_2"), +) + +``` + +:::note Lineage Combinations +For supported lineage combinations, see [Supported Lineage Combinations](#supported-lineage-combinations). +::: + +### Add Column Lineage + +You can add column-level lineage by using `column_lineage` parameter when linking datasets. + +#### Add Column Lineage with Fuzzy Matching + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_dataset_column.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +client.lineage.add_lineage( + upstream=DatasetUrn(platform="snowflake", name="sales_raw"), + downstream=DatasetUrn(platform="snowflake", name="sales_cleaned"), + column_lineage=True, # same as "auto_fuzzy", which maps columns based on name similarity +) + +``` + +When `column_lineage` is set to **True**, DataHub will automatically map columns based on their names, allowing for fuzzy matching. This is useful when upstream and downstream datasets have similar but not identical column names. (e.g. `customer_id` in upstream and `CustomerId` in downstream). See [Column Lineage Options](#column-lineage-options) for more details. + +#### Add Column Lineage with Strict Matching + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_dataset_column_auto_strict.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +client.lineage.add_lineage( + upstream=DatasetUrn(platform="snowflake", name="sales_raw"), + downstream=DatasetUrn(platform="snowflake", name="sales_cleaned"), + column_lineage="auto_strict", +) + +``` + +This will create column-level lineage with strict matching, meaning the column names must match exactly between upstream and downstream datasets. + +#### Add Column Lineage with Custom Mapping + +For custom mapping, you can use a dictionary where keys are downstream column names and values represent lists of upstream column names. This allows you to specify complex relationships. + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_dataset_column_custom_mapping.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +client.lineage.add_lineage( + upstream=DatasetUrn(platform="snowflake", name="sales_raw"), + downstream=DatasetUrn(platform="snowflake", name="sales_cleaned"), + # { downstream_column -> [upstream_columns] } + column_lineage={ + "id": ["id"], + "region": ["region", "region_id"], + "total_revenue": ["revenue"], + }, +) + +``` + +### Infer Lineage from SQL + +You can infer lineage directly from a SQL query using `infer_lineage_from_sql()`. This will parse the query, determine upstream and downstream datasets, and automatically add lineage (including column-level lineage when possible) and a query node showing the SQL transformation logic. + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_dataset_from_sql.py +from datahub.sdk.main_client import DataHubClient + +client = DataHubClient.from_env() + +sql_query = """ +CREATE TABLE sales_summary AS +SELECT + p.product_name, + c.customer_segment, + SUM(s.quantity) as total_quantity, + SUM(s.amount) as total_sales +FROM sales s +JOIN products p ON s.product_id = p.id +JOIN customers c ON s.customer_id = c.id +GROUP BY p.product_name, c.customer_segment +""" + +# sales_summary will be assumed to be in the default db/schema +# e.g. prod_db.public.sales_summary +client.lineage.infer_lineage_from_sql( + query_text=sql_query, + platform="snowflake", + default_db="prod_db", + default_schema="public", +) + +``` + +:::note DataHub SQL Parser + +Check out more information on how we handle SQL parsing below. + +- [The DataHub SQL Parser Documentation](../../lineage/sql_parsing.md) +- [Blog Post: Extracting Column-Level Lineage from SQL](https://medium.com/datahub-project/extracting-column-level-lineage-from-sql-779b8ce17567) + +::: + +### Add Query Node with Lineage + +If you provide a `transformation_text` to `add_lineage`, DataHub will create a query node that represents the transformation logic. This is useful for tracking how data is transformed between datasets. + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_dataset_add_with_query_node.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk.main_client import DataHubClient + +client = DataHubClient.from_env() + +upstream_urn = DatasetUrn(platform="snowflake", name="upstream_table") +downstream_urn = DatasetUrn(platform="snowflake", name="downstream_table") + +transformation_text = """ +from pyspark.sql import SparkSession + +spark = SparkSession.builder.appName("HighValueFilter").getOrCreate() +df = spark.read.table("customers") +high_value = df.filter("lifetime_value > 10000") +high_value.write.saveAsTable("high_value_customers") +""" + +client.lineage.add_lineage( + upstream=upstream_urn, + downstream=downstream_urn, + transformation_text=transformation_text, + column_lineage={"id": ["id", "customer_id"]}, +) + +# by passing the transformation_text, the query node will be created with the table level lineage. +# transformation_text can be any transformation logic e.g. a spark job, an airflow DAG, python script, etc. +# if you have a SQL query, we recommend using add_dataset_lineage_from_sql instead. +# note that transformation_text itself will not create a column level lineage. + +``` + +Transformation text can be any transformation logic, Python scripts, Airflow DAG code, or any other code that describes how the upstream dataset is transformed into the downstream dataset. + +

+ +

+ +:::note +Providing `transformation_text` will NOT create column lineage. You need to specify `column_lineage` parameter to enable column-level lineage. + +If you have a SQL query that describes the transformation, you can use [infer_lineage_from_sql](#infer-lineage-from-sql) to automatically parse the query and add column level lineage. +::: + +## Get Lineage + +The `get_lineage()` method allows you to retrieve lineage for a given entity. + +### Get Entity Lineage + +#### Get Upstream Lineage for a Dataset + +This will return the direct upstream entity that the dataset depends on. By default, it retrieves only the immediate upstream entities (1 hop). + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_get_basic.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk.main_client import DataHubClient + +client = DataHubClient.from_env() + +downstream_lineage = client.lineage.get_lineage( + source_urn=DatasetUrn(platform="snowflake", name="sales_summary"), + direction="downstream", +) + +print(downstream_lineage) + +``` + +#### Get Downstream Lineage for a Dataset Across Multiple Hops + +To get upstream/downstream entities that are more than one hop away, you can use the `max_hops` parameter. This allows you to traverse the lineage graph up to a specified number of hops. + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_get_with_hops.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk.main_client import DataHubClient + +client = DataHubClient.from_env() + +downstream_lineage = client.lineage.get_lineage( + source_urn=DatasetUrn(platform="snowflake", name="sales_summary"), + direction="downstream", + max_hops=2, +) + +print(downstream_lineage) + + +``` + +:::note USING MAX_HOPS +if you provide `max_hops` greater than 2, it will traverse the full lineage graph and limit the results by `count`. +::: + +#### Return Type + +`get_lineage()` returns a list of `LineageResult` objects. + +```python +results = [ + LineageResult( + urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,table_2,PROD)", + type="DATASET", + hops=1, + direction="downstream", + platform="snowflake", + name="table_2", # name of the entity + paths=[] # Only populated for column-level lineage + ) +] +``` + +### Get Column-Level Lineage + +#### Get Downstream Lineage for a Dataset Column + +You can retrieve column-level lineage by specifying the `source_column` parameter. This will return lineage paths that include the specified column. + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_column_get.py +from datahub.metadata.urns import DatasetUrn +from datahub.sdk.main_client import DataHubClient + +client = DataHubClient.from_env() + +# Get column lineage for the entire flow +# you can pass source_urn and source_column to get lineage for a specific column +# alternatively, you can pass schemaFieldUrn to source_urn. +# e.g. source_urn="urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,downstream_table),id)" +downstream_column_lineage = client.lineage.get_lineage( + source_urn=DatasetUrn(platform="snowflake", name="sales_summary"), + source_column="id", + direction="downstream", +) + +print(downstream_column_lineage) + +``` + +You can also pass `SchemaFieldUrn` as the `source_urn` to get column-level lineage. + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_column_get_from_schemafield.py +from datahub.sdk.main_client import DataHubClient + +client = DataHubClient.from_env() + +# Get column lineage for the entire flow +results = client.lineage.get_lineage( + source_urn="urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,sales_summary,PROD),id)", + direction="downstream", +) + +print(list(results)) + + +``` + +#### Return type + +The return type is the same as for entity lineage, but with additional `paths` field that contains column lineage paths. + +```python +results = [ + LineageResult( + urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,table_2,PROD)", + type="DATASET", + hops=1, + direction="downstream", + platform="snowflake", + name="table_2", # name of the entity + paths=[ + LineagePath( + urn="urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,table_1,PROD),col1)", + column_name="col1", # name of the column + entity_name="table_1", # name of the entity that contains the column + ), + LineagePath( + urn="urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:snowflake,table_2,PROD),col4)", + column_name="col4", # name of the column + entity_name="table_2", # name of the entity that contains the column + ) + ] # Only populated for column-level lineage + ) +] +``` + +For more details on how to interpret the results, see [Interpreting Lineage Results](#interpreting-lineage-results). + +### Filter Lineage Results + +You can filter by platform, type, domain, environment, and more. + +```python +# Inlined from /metadata-ingestion/examples/library/lineage_get_with_filter.py +from datahub.sdk.main_client import DataHubClient +from datahub.sdk.search_filters import FilterDsl as F + +client = DataHubClient.from_env() + +# get upstream snowflake production datasets. +results = client.lineage.get_lineage( + source_urn="urn:li:dataset:(platform,sales_agg,PROD)", + direction="upstream", + filter=F.and_(F.platform("snowflake"), F.entity_type("dataset"), F.env("PROD")), +) + +print(results) + +``` + +You can check more details about the available filters in the [Search SDK documentation](./sdk/search_client.md#filter-based-search). + +## Lineage SDK Reference + +For a full reference, see the [lineage SDK reference](../../../python-sdk/sdk-v2/lineage-client.mdx). + +### Supported Lineage Combinations + +The Lineage APIs support the following entity combinations: + +| Upstream Entity | Downstream Entity | +| --------------- | ----------------- | +| Dataset | Dataset | +| Dataset | DataJob | +| DataJob | DataJob | +| DataJob | Dataset | +| Dataset | Dashboard | +| Chart | Dashboard | +| Dashboard | Dashboard | +| Dataset | Chart | + +> ℹ️ Column-level lineage and creating query node with transformation text are **only supported** for `Dataset → Dataset` lineage. + +### Column Lineage Options + +For dataset-to-dataset lineage, you can specify `column_lineage` parameter in `add_lineage()` in several ways: + +| Value | Description | +| --------------- | --------------------------------------------------------------------------------- | +| `False` | Disable column-level lineage (default) | +| `True` | Enable column-level lineage with automatic mapping (same as "auto_fuzzy") | +| `"auto_fuzzy"` | Enable column-level lineage with fuzzy matching (useful for similar column names) | +| `"auto_strict"` | Enable column-level lineage with strict matching (exact column names required) | +| Column Mapping | A dictionary mapping downstream column names to lists of upstream column names | + +:::note `auto_fuzzy` vs `auto_strict` + +- **`auto_fuzzy`**: Automatically matches columns based on similar names, allowing for some flexibility in naming conventions. For example, these two columns would be considered a match: + - user_id → userId + - customer_id → CustomerId +- **`auto_strict`**: Requires exact column name matches between upstream and downstream datasets. For example, `customer_id` in the upstream dataset must match `customer_id` in the downstream dataset exactly. + +::: + +### Interpreting Column Lineage Results + +When retrieving column-level lineage, the results include `paths` that show how columns are related across datasets. Each path is a list of column URNs that represent the lineage from the source column to the target column. + +For example, let's say we have the following lineage across three tables: + +

+ +

+ +#### Example with `max_hops=1` + +```python +>>> client.lineage.get_lineage( + source_urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,table_1,PROD)", + source_column="col1", + direction="downstream", + max_hops=1 + ) +``` + +**Returns:** + +```python +[ + { + "urn": "...table_2...", + "hops": 1, + "paths": [ + ["...table_1.col1", "...table_2.col4"], + ["...table_1.col1", "...table_2.col5"] + ] + } +] +``` + +#### Example with `max_hops=2` + +```python +>>> client.lineage.get_lineage( + source_urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,table_1,PROD)", + source_column="col1", + direction="downstream", + max_hops=2 + ) +``` + +**Returns:** + +```python +[ + { + "urn": "...table_2...", + "hops": 1, + "paths": [ + ["...table_1.col1", "...table_2.col4"], + ["...table_1.col1", "...table_2.col5"] + ] + }, + { + "urn": "...table_3...", + "hops": 2, + "paths": [ + ["...table_1.col1", "...table_2.col4", "...table_3.col7"] + ] + } +] +``` + +## Alternative: Lineage GraphQL API + +While we generally recommend using the Python SDK for lineage, you can also use the GraphQL API to add and retrieve lineage. + +#### Add Lineage Between Datasets with GraphQL + +```graphql +mutation updateLineage { + updateLineage( + input: { + edgesToAdd: [ + { + downstreamUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" + upstreamUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" + } + ] + edgesToRemove: [] + } + ) +} +``` + +#### Get Downstream Lineage with GraphQL + +```graphql +query scrollAcrossLineage { + scrollAcrossLineage( + input: { + query: "*" + urn: "urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)" + count: 10 + direction: DOWNSTREAM + orFilters: [ + { + and: [ + { + condition: EQUAL + negated: false + field: "degree" + values: ["1", "2", "3+"] + } + ] + } + ] + } + ) { + searchResults { + degree + entity { + urn + type + } + } + } +} +``` + +#### Get Time-Filtered Lineage with GraphQL + +Filter lineage edges by their last update time using `lineageFlags`: + +```graphql +query searchAcrossLineage { + searchAcrossLineage( + input: { + query: "*" + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,analytics.orders,PROD)" + count: 10 + direction: UPSTREAM + orFilters: [{ and: [{ field: "degree", values: ["1"] }] }] + lineageFlags: { + startTimeMillis: 1625097600000 + endTimeMillis: 1627776000000 + } + } + ) { + searchResults { + entity { + urn + type + } + degree + } + } +} +``` + +This returns only upstream lineage edges that were last updated between July 1 and August 1, 2021. + +## FAQ + +**Can I get lineage at the column level?** +Yes — for dataset-to-dataset lineage, both `add_lineage()` and `get_lineage()` support column-level lineage. + +**Can I pass a SQL query and get lineage automatically?** +Yes — use `infer_lineage_from_sql()` to parse a query and extract table and column lineage. + +**Can I use filters when retrieving lineage?** +Yes — `get_lineage()` accepts structured filters via `FilterDsl`, just like in the Search SDK. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml.md new file mode 100644 index 00000000..7cb90881 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml.md @@ -0,0 +1,996 @@ +--- +title: AI/ML Framework Integration with DataHub +slug: /api/tutorials/ml +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/ml.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# AI/ML Framework Integration with DataHub + +## Why Integrate Your AI/ML System with DataHub? + +As a data practitioner, keeping track of your AI experiments, models, and their relationships can be challenging. +DataHub makes this easier by providing a central place to organize and track your AI assets. + +This guide will show you how to integrate your AI workflows with DataHub. +With integrations for popular ML platforms like MLflow and Amazon SageMaker, DataHub enables you to easily find and share AI models across your organization, track how models evolve over time, and understand how training data connects to each model. +Most importantly, it enables seamless collaboration on AI projects by making everything discoverable and connected. + +## Goals Of This Guide + +In this guide, you'll learn how to: + +- Create your basic AI components (models, experiments, runs) +- Connect these components to build a complete AI system +- Track relationships between models, data, and experiments + +## Core AI Concepts + +Here's what you need to know about the key components in DataHub: + +- **Experiments** are collections of training runs for the same project, like all attempts to build a churn predictor +- **Training Runs** are attempts to train a model within an experiment, capturing parameters and results +- **Model Groups** organize related models together, like all versions of your churn predictor +- **Models** are successful training runs registered for production use + +

+ +

+ +The hierarchy works like this: + +1. Every run belongs to an experiment +2. Successful runs can be registered as models +3. Models belong to a model group +4. Not every run becomes a model + +:::note Terminology Mapping +Different AI platforms (MLflow, Amazon SageMaker) have their own terminology. +To keep things consistent, we'll use DataHub's terms throughout this guide. +Here's how DataHub's terminology maps to these platforms: + +| DataHub | Description | MLflow | SageMaker | +| --------------- | ----------------------------------- | ------------- | ------------- | +| ML Model Group | Collection of related models | Model | Model Group | +| ML Model | Versioned artifact in a model group | Model Version | Model Version | +| ML Training Run | Single training attempt | Run | Training Job | +| ML Experiment | Collection of training runs | Experiment | Experiment | + +::: + +For platform-specific details, see our integration guides for [MLflow](/docs/generated/ingestion/sources/mlflow.md) and [Amazon SageMaker](/docs/generated/ingestion/sources/sagemaker.md). + +## Basic Setup + +To follow this tutorial, you'll need DataHub Quickstart deployed locally. +For detailed steps, see the [DataHub Quickstart Guide](/docs/quickstart.md). + +Next, set up the Python client for DataHub using `DatahubAIClient`defined in [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/ai/dh_ai_client.py). + +Create a token in DataHub UI and replace `` with your token: + +```python +from dh_ai_client import DatahubAIClient + +client = DatahubAIClient(token="", server_url="http://localhost:9002") +``` + +:::note Verifying via GraphQL +Throughout this guide, we'll show how to verify changes using GraphQL queries. +You can run these queries in the DataHub UI at `https://localhost:9002/api/graphiql`. +::: + +## Create AI Assets + +Let's create the basic building blocks of your ML system. These components will help you organize your AI work and make it discoverable by your team. + +### Create a Model Group + +A model group contains different versions of a similar model. For example, all versions of your "Customer Churn Predictor" would go in one group. + + + +Create a basic model group with just an identifier: + +```python +model_group = MLModelGroup( + id="airline_forecast_models_group", + platform="mlflow", +) + +client._emit_mcps(model_group.as_mcps()) +``` + + + +Add rich metadata like descriptions, creation timestamps, and team information: + +```python +model_group = MLModelGroup( + id="airline_forecast_models_group", + platform="mlflow", + name="Airline Forecast Models Group", + description="Group of models for airline passenger forecasting", + created=datetime.now(), + last_modified=datetime.now(), + owners=[CorpUserUrn("urn:li:corpuser:datahub")], + external_url="https://www.linkedin.com/in/datahub", + tags=["urn:li:tag:forecasting", "urn:li:tag:arima"], + terms=["urn:li:glossaryTerm:forecasting"], + custom_properties={"team": "forecasting"}, +) + +client._emit_mcps(model_group.as_mcps()) +``` + + + + +Let's verify that our model group was created: + + + +See your new model group in the DataHub UI: + +

+ +

+ + + +Query your model group to check its properties: + +```graphql +query { + mlModelGroup( + urn: "urn:li:mlModelGroup:(urn:li:dataPlatform:mlflow,airline_forecast_models_group,PROD)" + ) { + properties { + name + description + created { + time + } + } + } +} +``` + +The response will show your model group's details: + +```json +{ + "data": { + "mlModelGroup": { + "properties": { + "name": "Airline Forecast Models Group", + "description": "Group of models for airline passenger forecasting", + "created": { + "time": 1744356062485 + } + } + } + }, + "extensions": {} +} +``` + + + + +### Create a Model + +Next, let's create a specific model version that represents a trained model ready for deployment. + + + +Create a model with just the required version: + +```python +model = MLModel( + id="arima_model", + platform="mlflow", +) + +client._emit_mcps(model.as_mcps()) +``` + + + +Include metrics, parameters, and metadata for production use: + +```python +model = MLModel( + id="arima_model", + platform="mlflow", + name="ARIMA Model", + description="ARIMA model for airline passenger forecasting", + created=datetime.now(), + last_modified=datetime.now(), + owners=[CorpUserUrn("urn:li:corpuser:datahub")], + external_url="https://www.linkedin.com/in/datahub", + tags=["urn:li:tag:forecasting", "urn:li:tag:arima"], + terms=["urn:li:glossaryTerm:forecasting"], + custom_properties={"team": "forecasting"}, + version="1", + aliases=["champion"], + hyper_params={"learning_rate": "0.01"}, + training_metrics={"accuracy": "0.9"}, +) + +client._emit_mcps(model.as_mcps()) +``` + + + + +Let's verify our model: + + + +Check your model's details in the DataHub UI: + +

+ +

+ + + +Query your model's information: + +```graphql +query { + mlModel(urn: "urn:li:mlModel:(urn:li:dataPlatform:mlflow,arima_model,PROD)") { + properties { + name + description + } + versionProperties { + version { + versionTag + } + } + } +} +``` + +The response will show your model's details: + +```json +{ + "data": { + "mlModel": { + "properties": { + "name": "ARIMA Model", + "description": "ARIMA model for airline passenger forecasting" + }, + "versionProperties": { + "version": { + "versionTag": "1" + } + } + } + }, + "extensions": {} +} +``` + + + + +### Create an Experiment + +An experiment helps organize multiple training runs for a specific project. + + + +Create a basic experiment: + +```python +experiment = Container( + container_key=ContainerKey( + platform="mlflow", + name="airline_forecast_experiment" + ), + display_name="Airline Forecast Experiment" +) + +client._emit_mcps(experiment.as_mcps()) +``` + + + +Add context and metadata: + +```python +experiment = Container( + container_key=ContainerKey( + platform="mlflow", + name="airline_forecast_experiment" + ), + display_name="Airline Forecast Experiment", + description="Experiment to forecast airline passenger numbers", + extra_properties={"team": "forecasting"}, + created=datetime(2025, 4, 9, 22, 30), + last_modified=datetime(2025, 4, 9, 22, 30), + subtype=MLAssetSubTypes.MLFLOW_EXPERIMENT, +) + +client._emit_mcps(experiment.as_mcps()) +``` + + + + +Verify your experiment: + + + +See your experiment's details in the UI: + +

+ +

+ + + +Query your experiment's information: + +```graphql +query { + container(urn: "urn:li:container:airline_forecast_experiment") { + properties { + name + description + } + } +} +``` + +Check the response: + +```json +{ + "data": { + "container": { + "properties": { + "name": "Airline Forecast Experiment", + "description": "Experiment to forecast airline passenger numbers" + } + } + }, + "extensions": {} +} +``` + + + + +### Create a Training Run + +A training run captures all details about a specific model training attempt. + + + +Create a basic training run: + +```python +client.create_training_run( + run_id="simple_training_run", +) +``` + + + +Include metrics, parameters, and other important metadata: + +```python +client.create_training_run( + run_id="simple_training_run", + properties=DataProcessInstancePropertiesClass( + name="Simple Training Run", + created=AuditStampClass( + time=1628580000000, actor="urn:li:corpuser:datahub" + ), + customProperties={"team": "forecasting"}, + ), + training_run_properties=MLTrainingRunPropertiesClass( + id="simple_training_run", + outputUrls=["s3://my-bucket/output"], + trainingMetrics=[MLMetricClass(name="accuracy", value="0.9")], + hyperParams=[MLHyperParamClass(name="learning_rate", value="0.01")], + externalUrl="https:localhost:5000", + ), + run_result=RunResultType.FAILURE, + start_timestamp=1628580000000, + end_timestamp=1628580001000, +) +``` + + + + +Verify your training run: + + + +View the run details in the UI: + +

+ +

+ + + +Query your training run: + +```graphql +query { + dataProcessInstance(urn: "urn:li:dataProcessInstance:simple_training_run") { + name + created { + time + } + properties { + customProperties + } + } +} +``` + +Check the response: + +```json +{ + "data": { + "dataProcessInstance": { + "name": "Simple Training Run", + "created": { + "time": 1628580000000 + }, + "properties": { + "customProperties": { + "team": "forecasting" + } + } + } + } +} +``` + + + + +### Create a Dataset + +Datasets are crucial components in your ML system, serving as inputs and outputs for your training runs. Creating a dataset in DataHub allows you to track data lineage and understand how data flows through your ML pipeline. + + + +Create a basic dataset with minimal information: + +```python +input_dataset = Dataset( + platform="snowflake", + name="iris_input", +) +client._emit_mcps(input_dataset.as_mcps()) +``` + + + + +Create a dataset with more detailed information: + +```python +input_dataset = Dataset( + platform="snowflake", + name="iris_input", + description="Raw Iris dataset used for training ML models", + schema=[("id", "number"), ("name", "string"), ("species", "string")], + display_name="Iris Training Input Data", + tags=["urn:li:tag:ml_data", "urn:li:tag:iris"], + terms=["urn:li:glossaryTerm:raw_data"], + owners=[CorpUserUrn("urn:li:corpuser:datahub")], + custom_properties={ + "data_source": "UCI Repository", + "records": "150", + "features": "4", + }, +) +client._emit_mcps(input_dataset.as_mcps()) + +output_dataset = Dataset( + platform="snowflake", + name="iris_output", + description="Processed Iris dataset with model predictions", + schema=[("id", "number"), ("name", "string"), ("species", "string")], + display_name="Iris Model Output Data", + tags=["urn:li:tag:ml_data", "urn:li:tag:predictions"], + terms=["urn:li:glossaryTerm:model_output"], + owners=[CorpUserUrn("urn:li:corpuser:datahub")], + custom_properties={ + "model_version": "1.0", + "records": "150", + "accuracy": "0.95", + }, +) +client._emit_mcps(output_dataset.as_mcps()) +``` + + + + +Verify your datasets: + + + View dataset details in the DataHub UI: + +

+ +

+ + + + +Query your dataset's information: + +```graphql +query { + dataset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,iris_input,PROD)" + ) { + name + properties { + customProperties + } + } +} +``` + +Check the response: + +```graphql +{ + "data": { + "dataset": { + "name": "iris_input", + "properties": { + "customProperties": { + "data_source": "UCI Repository", + "records": "150", + "features": "4" + } + } + } + } +} +``` + + + + +Datasets in DataHub can also include schema information, data quality metrics, and lineage details, which are particularly valuable for ML workflows where understanding data characteristics is crucial for model performance. + +## Define Relationships + +Now let's connect these components to create a comprehensive ML system. These connections enable you to track model lineage, monitor model evolution, understand dependencies, and search effectively across your ML assets. + +### Add Model To Model Group + +Connect your model to its group: + +```python +model.add_group(model_group.urn) + +client._emit_mcps(model.as_mcps()) +``` + + + + +View model versions in the **Model Group** under the **Models** section: + +

+ +

+ +Find group information in the **Model** page under the **Group** tab: + +

+ +

+ + + +Query the model-group relationship: + +```graphql +query { + mlModel(urn: "urn:li:mlModel:(urn:li:dataPlatform:mlflow,arima_model,PROD)") { + name + properties { + groups { + urn + properties { + name + } + } + } + } +} +``` + +Check the response: + +```json +{ + "data": { + "mlModel": { + "name": "ARIMA Model", + "properties": { + "groups": [ + { + "urn": "urn:li:mlModelGroup:(urn:li:dataPlatform:mlflow,airline_forecast_models_group)", + "properties": { + "name": "Airline Forecast Models Group" + } + } + ] + } + } + } +} +``` + + + + +### Add Run To Experiment + +Connect a training run to its experiment: + +```python +client.add_run_to_experiment(run_urn=run_urn, experiment_urn=experiment_urn) +``` + + + + +Find your runs in the **Experiment** page under the **Entities** tab: + +

+ +

+ +See the experiment details in the **Run** page: + +

+ +

+ + + +Query the run-experiment relationship: + +```graphql +query { + dataProcessInstance(urn: "urn:li:dataProcessInstance:simple_training_run") { + name + parentContainers { + containers { + urn + properties { + name + } + } + } + } +} +``` + +View the relationship details: + +```json +{ + "data": { + "dataProcessInstance": { + "name": "Simple Training Run", + "parentContainers": { + "containers": [ + { + "urn": "urn:li:container:airline_forecast_experiment", + "properties": { + "name": "Airline Forecast Experiment" + } + } + ] + } + } + } +} +``` + + + + +### Add Run To Model + +Connect a training run to its resulting model: + +```python +model.add_training_job(DataProcessInstanceUrn(run_id)) + +client._emit_mcps(model.as_mcps()) +``` + +This relationship enables you to: + +- Track which runs produced each model +- Understand model provenance +- Debug model issues +- Monitor model evolution + + + + +Find the source run in the **Model** page under the **Summary** tab: + +

+ +

+ +See related models in the **Run** page under the **Lineage** tab: + +

+ +

+

+ +

+ + + + +Query the model's training jobs: + +```graphql +query { + mlModel(urn: "urn:li:mlModel:(urn:li:dataPlatform:mlflow,arima_model,PROD)") { + name + properties { + mlModelLineageInfo { + trainingJobs + } + } + } +} +``` + +View the relationship: + +```json +{ + "data": { + "mlModel": { + "name": "ARIMA Model", + "properties": { + "mlModelLineageInfo": { + "trainingJobs": ["urn:li:dataProcessInstance:simple_training_run"] + } + } + } + } +} +``` + + + + +### Add Run To Model Group + +Create a direct connection between a run and a model group: + +```python +model_group.add_training_job(DataProcessInstanceUrn(run_id)) + +client._emit_mcps(model_group.as_mcps()) +``` + +This connection lets you: + +- View model groups in the run's lineage +- Query training jobs at the group level +- Track training history for model families + + + + +See model groups in the **Run** page under the **Lineage** tab: + +

+ +

+

+ +

+ + + +Query the model group's training jobs: + +```graphql +query { + mlModelGroup( + urn: "urn:li:mlModelGroup:(urn:li:dataPlatform:mlflow,airline_forecast_models_group)" + ) { + name + properties { + mlModelLineageInfo { + trainingJobs + } + } + } +} +``` + +Check the relationship: + +```json +{ + "data": { + "mlModelGroup": { + "name": "Airline Forecast Models Group", + "properties": { + "mlModelLineageInfo": { + "trainingJobs": ["urn:li:dataProcessInstance:simple_training_run"] + } + } + } + } +} +``` + + + + +### Add Dataset To Run + +Track input and output datasets for your training runs: + +```python +client.add_input_datasets_to_run( + run_urn=run_urn, + dataset_urns=[str(input_dataset_urn)] +) + +client.add_output_datasets_to_run( + run_urn=run_urn, + dataset_urns=[str(output_dataset_urn)] +) +``` + +These connections help you: + +- Track data lineage +- Understand data dependencies +- Ensure reproducibility +- Monitor data quality impacts + +Find dataset relationships in the **Lineage** tab of either the **Dataset** or **Run** page: + +

+ +

+ +## Update Properties + +### Update Model Group Properties + +Model groups can be updated with additional information: + +```python +# Update description +model_group.set_description("Updated description for airline forecast models") + +# Add tags and terms +model_group.add_tag(TagUrn("production")) +model_group.add_term(GlossaryTermUrn("time-series")) + +# Update custom properties +model_group.set_custom_properties({ + "team": "forecasting", + "business_unit": "operations", + "status": "active" +}) + +# Save the changes +client._emit_mcps(model_group.as_mcps()) +``` + +These updates allow you to: + +- Improve documentation with detailed descriptions +- Apply consistent business context with tags and terms +- Track organizational ownership and status + +### Update Model Properties + +Models can be updated with additional information as they evolve: + +```python +# Update model version +model.set_version("2") + +# Add tags and terms +model.add_tag(TagUrn("marketing")) +model.add_term(GlossaryTermUrn("marketing")) + +# Add version alias +model.add_version_alias("challenger") + +# Save the changes +client._emit_mcps(model.as_mcps()) +``` + +These updates allow you to: + +- Track model iterations through versioning +- Apply business context with tags and terms +- Manage deployment aliases like "champion" and "challenger" + +### Update Experiment Properties + +Experiments can be updated with additional metadata as your project evolves: + +```python +# Create a container object for the existing experiment +existing_experiment = Container( + container_key=ContainerKey( + platform="mlflow", + name="airline_forecast_experiment" + ), +) + +# Update properties +existing_experiment.set_description("Updated experiment for forecasting passenger numbers") +existing_experiment.add_tag(TagUrn("time-series")) +existing_experiment.add_term(GlossaryTermUrn("forecasting")) +existing_experiment.set_custom_properties({ + "team": "forecasting", + "priority": "high", + "status": "active" +}) + +# Save the changes +client._emit_mcps(existing_experiment.as_mcps()) +``` + +These updates help you: + +- Document evolving experiment objectives +- Categorize experiments with consistent tags +- Track experiment status and priority + +## Full Overview + +Here's your complete ML system with all components connected: + +

+ +

+ +You now have a complete lineage view of your ML assets, from training data through runs to production models! + +You can check out the full code for this tutorial [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/ai/dh_ai_client_sample.py). + +## What's Next? + +To see these integrations in action: + +- Watch our [Townhall demo](https://youtu.be/_WUoVqkF2Zo?feature=shared&t=1932) showcasing the MLflow integration +- Explore our detailed documentation: + - [MLflow Integration Guide](/docs/generated/ingestion/sources/mlflow.md) + - [Amazon SageMaker Integration Guide](/docs/generated/ingestion/sources/sagemaker.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml_feature_store.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml_feature_store.md new file mode 100644 index 00000000..69fa1595 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/ml_feature_store.md @@ -0,0 +1,793 @@ +--- +title: Feature Store Integration With DataHub +slug: /api/tutorials/ml_feature_store +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/ml_feature_store.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Feature Store Integration With DataHub + +## Why Would You Integrate Feature Store with DataHub? + +Feature Store is a data management layer that stores, organizes, and manages features for machine learning models. It is a centralized repository for features that can be used across different AI/ML models. +By integrating Feature Store with DataHub, you can track the lineage of features used in AI/ML models, understand how features are generated, and how they are used to train models. + +For technical details on feature store entities, please refer to the following docs: + +- [MlFeature](/docs/generated/metamodel/entities/mlFeature.md) +- [MlPrimaryKey](/docs/generated/metamodel/entities/mlPrimaryKey.md) +- [MlFeatureTable](/docs/generated/metamodel/entities/mlFeatureTable.md) + +### Goal Of This Guide + +This guide will show you how to + +- Create feature store entities: MlFeature, MlFeatureTable, MlPrimaryKey +- Read feature store entities: MlFeature, MlFeatureTable, MlPrimaryKey +- Attach MlModel to MlFeature +- Attach MlFeatures to MlFeatureTable +- Attached MlFeatures to upstream Datasets that power them + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create ML Entities + +:::note +For creating MLModels and MLGroups, please refer to [AI/ML Integration Guide](/docs/api/tutorials/ml.md). +::: + +### Create MlFeature + +An ML Feature represents an instance of a feature that can be used across different machine learning models. Features are organized into Feature Tables to be consumed by machine learning models. For example, if we were modeling features for a Users Feature Table, the Features would be `age`, `sign_up_date`, `active_in_past_30_days` and so forth.Using Features in DataHub allows users to see the sources a feature was generated from and how a feature is used to train models. + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlfeature_create.py +import os + +import datahub.emitter.mce_builder as builder +import datahub.metadata.schema_classes as models +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Create an emitter to DataHub over REST +gms_server = os.getenv("DATAHUB_GMS_URL", "http://localhost:8080") +token = os.getenv("DATAHUB_GMS_TOKEN") +emitter = DatahubRestEmitter(gms_server=gms_server, token=token) + +dataset_urn = builder.make_dataset_urn( + name="fct_users_created", platform="hive", env="PROD" +) +feature_urn = builder.make_ml_feature_urn( + feature_table_name="users_feature_table", + feature_name="user_signup_date", +) + +# Create feature +metadata_change_proposal = MetadataChangeProposalWrapper( + entityUrn=feature_urn, + aspect=models.MLFeaturePropertiesClass( + description="Represents the date the user created their account", + # attaching a source to a feature creates lineage between the feature + # and the upstream dataset. This is how lineage between your data warehouse + # and machine learning ecosystem is established. + sources=[dataset_urn], + dataType="TIME", + ), +) + +# Emit metadata! +emitter.emit_mcp(metadata_change_proposal) +print(f"Created ML feature: {feature_urn}") + +``` + +Note that when creating a feature, you create upstream lineage to the data warehouse using `sources`. + + + + +### Create MlPrimaryKey + +An ML Primary Key represents a specific element of a Feature Table that indicates what group the other features belong to. For example, if a Feature Table contained features for Users, the ML Primary Key would likely be `user_id` or some similar unique identifier for a user. Using ML Primary Keys in DataHub allow users to indicate how ML Feature Tables are structured. + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlprimarykey_create.py +import os + +import datahub.emitter.mce_builder as builder +import datahub.metadata.schema_classes as models +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Create an emitter to DataHub over REST +gms_server = os.getenv("DATAHUB_GMS_URL", "http://localhost:8080") +token = os.getenv("DATAHUB_GMS_TOKEN") +emitter = DatahubRestEmitter(gms_server=gms_server, token=token) + +dataset_urn = builder.make_dataset_urn( + name="fct_users_created", platform="hive", env="PROD" +) +primary_key_urn = builder.make_ml_primary_key_urn( + feature_table_name="users_feature_table", + primary_key_name="user_id", +) + +# Create feature +metadata_change_proposal = MetadataChangeProposalWrapper( + entityUrn=primary_key_urn, + aspect=models.MLPrimaryKeyPropertiesClass( + description="Represents the id of the user the other features relate to.", + # attaching a source to a ml primary key creates lineage between the feature + # and the upstream dataset. This is how lineage between your data warehouse + # and machine learning ecosystem is established. + sources=[dataset_urn], + dataType="TEXT", + ), +) + +# Emit metadata! +emitter.emit_mcp(metadata_change_proposal) +print(f"Created ML primary key: {primary_key_urn}") + +``` + +Note that when creating a primary key, you create upstream lineage to the data warehouse using `sources`. + + + + +### Create MlFeatureTable + +A feature table represents a group of similar Features that can all be used together to train a model. For example, if there was a Users Feature Table, it would contain documentation around how to use the Users collection of Features and references to each Feature and ML Primary Key contained within it. + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlfeature_table_create.py +import os + +import datahub.emitter.mce_builder as builder +import datahub.metadata.schema_classes as models +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Create an emitter to DataHub over REST +gms_server = os.getenv("DATAHUB_GMS_URL", "http://localhost:8080") +token = os.getenv("DATAHUB_GMS_TOKEN") +emitter = DatahubRestEmitter(gms_server=gms_server, token=token) + +feature_table_urn = builder.make_ml_feature_table_urn( + feature_table_name="users_feature_table", platform="feast" +) + +feature_urns = [ + builder.make_ml_feature_urn( + feature_name="user_signup_date", feature_table_name="users_feature_table" + ), + builder.make_ml_feature_urn( + feature_name="user_last_active_date", feature_table_name="users_feature_table" + ), +] + +primary_key_urns = [ + builder.make_ml_primary_key_urn( + feature_table_name="users_feature_table", + primary_key_name="user_id", + ) +] + +feature_table_properties = models.MLFeatureTablePropertiesClass( + description="Test description", + # link your features to a feature table + mlFeatures=feature_urns, + # link your primary keys to the feature table + mlPrimaryKeys=primary_key_urns, +) + +# MCP creation +metadata_change_proposal = MetadataChangeProposalWrapper( + entityUrn=feature_table_urn, + aspect=feature_table_properties, +) + +# Emit metadata! +emitter.emit_mcp(metadata_change_proposal) +print(f"Created ML feature table: {feature_table_urn}") + +``` + +Note that when creating a feature table, you connect the table to its features and primary key using `mlFeatures` and `mlPrimaryKeys`. + + + + +### Expected Outcome of creating entities + +You can search the entities in DataHub UI. + +

+ +

+ +## Read ML Entities + +### Read MLFeature + + + + +```json +query { + mlFeature(urn: "urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)"){ + name + featureNamespace + description + properties { + description + dataType + version { + versionTag + } + } + } +} +``` + +Expected response: + +```json +{ + "data": { + "mlFeature": { + "name": "test_BOOL_LIST_feature", + "featureNamespace": "test_feature_table_all_feature_dtypes", + "description": null, + "properties": { + "description": null, + "dataType": "SEQUENCE", + "version": null + } + } + }, + "extensions": {} +} +``` + + + + +```json +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "query": "{ mlFeature(urn: \"urn:li:mlFeature:(test_feature_table_all_feature_dtypes,test_BOOL_LIST_feature)\") { name featureNamespace description properties { description dataType version { versionTag } } } }" +}' +``` + +Expected response: + +```json +{ + "data": { + "mlFeature": { + "name": "test_BOOL_LIST_feature", + "featureNamespace": "test_feature_table_all_feature_dtypes", + "description": null, + "properties": { + "description": null, + "dataType": "SEQUENCE", + "version": null + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlfeature_read.py +from datahub.sdk import DataHubClient, MLFeatureUrn + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use MLFeatureUrn.from_string(...) +mlfeature_urn = MLFeatureUrn( + "test_feature_table_all_feature_dtypes", "test_BOOL_feature" +) + +mlfeature_entity = client.entities.get(mlfeature_urn) +print("MLFeature name:", mlfeature_entity.name) +print("MLFeature table:", mlfeature_entity.feature_table_urn) +print("MLFeature description:", mlfeature_entity.description) + +``` + + + + +### Read MlPrimaryKey + + + + +```json +query { + mlPrimaryKey(urn: "urn:li:mlPrimaryKey:(user_features,user_id)"){ + name + featureNamespace + description + dataType + properties { + description + dataType + version { + versionTag + } + } + } +} +``` + +Expected response: + +```json +{ + "data": { + "mlPrimaryKey": { + "name": "user_id", + "featureNamespace": "user_features", + "description": "User's internal ID", + "dataType": "ORDINAL", + "properties": { + "description": "User's internal ID", + "dataType": "ORDINAL", + "version": null + } + } + }, + "extensions": {} +} +``` + + + + +```json +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "query": "query { mlPrimaryKey(urn: \"urn:li:mlPrimaryKey:(user_features,user_id)\"){ name featureNamespace description dataType properties { description dataType version { versionTag } } }}" +}' +``` + +Expected response: + +```json +{ + "data": { + "mlPrimaryKey": { + "name": "user_id", + "featureNamespace": "user_features", + "description": "User's internal ID", + "dataType": "ORDINAL", + "properties": { + "description": "User's internal ID", + "dataType": "ORDINAL", + "version": null + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlprimarykey_read.py +from datahub.sdk import DataHubClient, MLPrimaryKeyUrn + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use MLPrimaryKeyUrn.from_string(...) +mlprimarykey_urn = MLPrimaryKeyUrn("user_features", "user_id") + +mlprimarykey_entity = client.entities.get(mlprimarykey_urn) +print("MLPrimaryKey name:", mlprimarykey_entity.name) +print("MLPrimaryKey feature table:", mlprimarykey_entity.feature_table_urn) +print("MLPrimaryKey description:", mlprimarykey_entity.description) + +``` + + + + +### Read MLFeatureTable + + + + +```json +query { + mlFeatureTable(urn: "urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_all_feature_dtypes)"){ + name + description + platform { + name + } + properties { + description + mlFeatures { + name + } + } + } +} +``` + +Expected Response: + +```json +{ + "data": { + "mlFeatureTable": { + "name": "test_feature_table_all_feature_dtypes", + "description": null, + "platform": { + "name": "feast" + }, + "properties": { + "description": null, + "mlFeatures": [ + { + "name": "test_BOOL_LIST_feature" + }, + ...{ + "name": "test_STRING_feature" + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```json +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "query": "{ mlFeatureTable(urn: \"urn:li:mlFeatureTable:(urn:li:dataPlatform:feast,test_feature_table_all_feature_dtypes)\") { name description platform { name } properties { description mlFeatures { name } } } }" +}' +``` + +Expected Response: + +```json +{ + "data": { + "mlFeatureTable": { + "name": "test_feature_table_all_feature_dtypes", + "description": null, + "platform": { + "name": "feast" + }, + "properties": { + "description": null, + "mlFeatures": [ + { + "name": "test_BOOL_LIST_feature" + }, + ...{ + "name": "test_STRING_feature" + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlfeature_table_read.py +from datahub.sdk import DataHubClient, MLFeatureTableUrn + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use MLFeatureTableUrn.from_string(...) +mlfeature_table_urn = MLFeatureTableUrn( + "feast", "test_feature_table_all_feature_dtypes" +) + +mlfeature_table_entity = client.entities.get(mlfeature_table_urn) +print("MLFeature Table name:", mlfeature_table_entity.name) +print("MLFeature Table platform:", mlfeature_table_entity.platform) +print("MLFeature Table description:", mlfeature_table_entity.description) + +``` + + + + +### Read MLModel + + + + +```json +query { + mlModel(urn: "urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)"){ + name + description + properties { + description + version + type + mlFeatures + groups { + urn + name + } + } + } +} +``` + +Expected Response: + +```json +{ + "data": { + "mlModel": { + "name": "scienceModel", + "description": "A sample model for predicting some outcome.", + "properties": { + "description": "A sample model for predicting some outcome.", + "version": null, + "type": "Naive Bayes classifier", + "mlFeatures": null, + "groups": [] + } + } + }, + "extensions": {} +} +``` + + + + +```json +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "query": "{ mlModel(urn: \"urn:li:mlModel:(urn:li:dataPlatform:science,scienceModel,PROD)\") { name description properties { description version type mlFeatures groups { urn name } } } }" +}' +``` + +Expected Response: + +```json +{ + "data": { + "mlModel": { + "name": "scienceModel", + "description": "A sample model for predicting some outcome.", + "properties": { + "description": "A sample model for predicting some outcome.", + "version": null, + "type": "Naive Bayes classifier", + "mlFeatures": null, + "groups": [] + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlmodel_read.py +from datahub.metadata.urns import MlModelUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use MlModelUrn.from_string(...) +mlmodel_urn = MlModelUrn(platform="mlflow", name="my-recommendations-model") + +mlmodel_entity = client.entities.get(mlmodel_urn) +print("Model Name: ", mlmodel_entity.name) +print("Model Description: ", mlmodel_entity.description) +print("Model Group: ", mlmodel_entity.model_group) +print("Model Hyper Parameters: ", mlmodel_entity.hyper_params) + +``` + + + + +## Add ML Entities + +### Add MlFeature to MlFeatureTable + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlfeature_add_to_mlfeature_table.py +import datahub.emitter.mce_builder as builder +import datahub.metadata.schema_classes as models +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph +from datahub.metadata.schema_classes import MLFeatureTablePropertiesClass + +gms_endpoint = "http://localhost:8080" +# Create an emitter to DataHub over REST +emitter = DatahubRestEmitter(gms_server=gms_endpoint, extra_headers={}) + +feature_table_urn = builder.make_ml_feature_table_urn( + feature_table_name="my-feature-table", platform="feast" +) +feature_urns = [ + builder.make_ml_feature_urn( + feature_name="my-feature2", feature_table_name="my-feature-table" + ), +] + +# This code concatenates the new features with the existing features in the feature table. +# If you want to replace all existing features with only the new ones, you can comment out this line. +graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) +feature_table_properties = graph.get_aspect( + entity_urn=feature_table_urn, aspect_type=MLFeatureTablePropertiesClass +) +if feature_table_properties: + current_features = feature_table_properties.mlFeatures + print("current_features:", current_features) + if current_features: + feature_urns += current_features + +feature_table_properties = models.MLFeatureTablePropertiesClass(mlFeatures=feature_urns) +# MCP createion +metadata_change_proposal = MetadataChangeProposalWrapper( + entityUrn=feature_table_urn, + aspect=feature_table_properties, +) + +# Emit metadata! This is a blocking call +emitter.emit(metadata_change_proposal) + +``` + + + + +### Add MlFeature to MLModel + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlfeature_add_to_mlmodel.py +import datahub.emitter.mce_builder as builder +import datahub.metadata.schema_classes as models +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph +from datahub.metadata.schema_classes import MLModelPropertiesClass + +gms_endpoint = "http://localhost:8080" +# Create an emitter to DataHub over REST +emitter = DatahubRestEmitter(gms_server=gms_endpoint, extra_headers={}) + +model_urn = builder.make_ml_model_urn( + model_name="my-test-model", platform="science", env="PROD" +) +feature_urns = [ + builder.make_ml_feature_urn( + feature_name="my-feature3", feature_table_name="my-feature-table" + ), +] + +# This code concatenates the new features with the existing features in the model +# If you want to replace all existing features with only the new ones, you can comment out this line. +graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) +model_properties = graph.get_aspect( + entity_urn=model_urn, aspect_type=MLModelPropertiesClass +) +if model_properties: + current_features = model_properties.mlFeatures + print("current_features:", current_features) + if current_features: + feature_urns += current_features + +model_properties = models.MLModelPropertiesClass(mlFeatures=feature_urns) + +# MCP creation +metadata_change_proposal = MetadataChangeProposalWrapper( + entityUrn=model_urn, + aspect=model_properties, +) + +# Emit metadata! +emitter.emit(metadata_change_proposal) + +``` + + + + +### Add MLGroup To MLModel + + + + +```python +# Inlined from /metadata-ingestion/examples/library/mlgroup_add_to_mlmodel.py +from datahub.metadata.urns import MlModelGroupUrn +from datahub.sdk import DataHubClient +from datahub.sdk.mlmodel import MLModel + +client = DataHubClient.from_env() + +model = MLModel( + id="my-recommendations-model", + platform="mlflow", +) + +model.set_model_group( + MlModelGroupUrn( + platform="mlflow", + name="my-recommendations-model-group", + ) +) + +client.entities.upsert(model) + +``` + + + + +### Expected Outcome of Adding ML Entities + +You can access to `Features` or `Group` Tab of each entity to view the added entities. + +

+ +

+ +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/mlmodel-mlmodelgroup.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/mlmodel-mlmodelgroup.md new file mode 100644 index 00000000..d93597dc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/mlmodel-mlmodelgroup.md @@ -0,0 +1,200 @@ +--- +title: MLModel & MLModelGroup +slug: /api/tutorials/mlmodel-mlmodelgroup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/mlmodel-mlmodelgroup.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# MLModel & MLModelGroup + +## Why Would You Use MLModel and MLModelGroup? + +MLModel and MLModelGroup entities are used to represent machine learning models and their associated groups within a metadata ecosystem. They allow users to define, manage, and monitor machine learning models, including their versions, configurations, and performance metrics. + +### Goal Of This Guide + +This guide will show you how to + +- Create an MLModel or MLModelGroup. +- Associate an MLModel with an MLModelGroup. +- Read MLModel and MLModelGroup entities. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +## Create MLModelGroup + +You can create an MLModelGroup by providing the necessary attributes such as name, platform, and other metadata. + +```python +# Inlined from /metadata-ingestion/examples/library/mlmodel_group_create.py +from datahub.sdk import DataHubClient +from datahub.sdk.mlmodelgroup import MLModelGroup + +client = DataHubClient.from_env() + +mlmodel_group = MLModelGroup( + id="my-recommendations-model-group", + name="My Recommendations Model Group", + platform="mlflow", + description="Grouping of ml model related to home page recommendations", + custom_properties={ + "framework": "pytorch", + }, +) + +client.entities.upsert(mlmodel_group) +print(f"Created ML model group: {mlmodel_group.urn}") + +``` + +## Create MLModel + +You can create an MLModel by providing the necessary attributes such as name, platform, and other metadata. + +```python +# Inlined from /metadata-ingestion/examples/library/mlmodel_create_full.py +import datahub.metadata.schema_classes as models +from datahub.metadata.urns import MlFeatureUrn, MlModelGroupUrn +from datahub.sdk import DataHubClient +from datahub.sdk.mlmodel import MLModel + +client = DataHubClient.from_env() + +mlmodel = MLModel( + id="my-recommendations-model", + name="My Recommendations Model", + platform="mlflow", + model_group=MlModelGroupUrn( + platform="mlflow", name="my-recommendations-model-group" + ), + custom_properties={ + "framework": "pytorch", + }, + extra_aspects=[ + models.MLModelPropertiesClass( + mlFeatures=[ + str( + MlFeatureUrn( + feature_namespace="users_feature_table", name="user_signup_date" + ) + ), + str( + MlFeatureUrn( + feature_namespace="users_feature_table", + name="user_last_active_date", + ) + ), + ] + ) + ], + training_metrics={ + "accuracy": "1.0", + "precision": "0.95", + "recall": "0.90", + "f1_score": "0.92", + }, + hyper_params={ + "learning_rate": "0.01", + "num_epochs": "100", + "batch_size": "32", + }, +) + +client.entities.update(mlmodel) + +``` + +Note that you can associate an MLModel with an MLModelGroup by providing the group URN when creating the MLModel. + +You can also set MLModelGroup later by updating the MLModel entity as shown below. + +```python +# Inlined from /metadata-ingestion/examples/library/mlgroup_add_to_mlmodel.py +from datahub.metadata.urns import MlModelGroupUrn +from datahub.sdk import DataHubClient +from datahub.sdk.mlmodel import MLModel + +client = DataHubClient.from_env() + +model = MLModel( + id="my-recommendations-model", + platform="mlflow", +) + +model.set_model_group( + MlModelGroupUrn( + platform="mlflow", + name="my-recommendations-model-group", + ) +) + +client.entities.upsert(model) + +``` + +## Read MLModelGroup + +You can read an MLModelGroup by providing the group URN. + +```python +# Inlined from /metadata-ingestion/examples/library/mlmodel_group_read.py +from datahub.metadata.urns import MlModelGroupUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use MlModelGroupUrn.from_string(...) +mlmodel_group_urn = MlModelGroupUrn( + platform="mlflow", name="my-recommendations-model-group" +) + +mlmodel_group_entity = client.entities.get(mlmodel_group_urn) +print("Model Group Name: ", mlmodel_group_entity.name) +print("Model Group Description: ", mlmodel_group_entity.description) +print("Model Group Custom Properties: ", mlmodel_group_entity.custom_properties) + +``` + +#### Expected Output + +```python +>> Model Group Name: My Recommendations Model Group +>> Model Group Description: A group for recommendations models +>> Model Group Custom Properties: {'owner': 'John Doe', 'team': 'recommendations', 'domain': 'marketing'} +``` + +## Read MLModel + +You can read an MLModel by providing the model URN. + +```python +# Inlined from /metadata-ingestion/examples/library/mlmodel_read.py +from datahub.metadata.urns import MlModelUrn +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() + +# Or get this from the UI (share -> copy urn) and use MlModelUrn.from_string(...) +mlmodel_urn = MlModelUrn(platform="mlflow", name="my-recommendations-model") + +mlmodel_entity = client.entities.get(mlmodel_urn) +print("Model Name: ", mlmodel_entity.name) +print("Model Description: ", mlmodel_entity.description) +print("Model Group: ", mlmodel_entity.model_group) +print("Model Hyper Parameters: ", mlmodel_entity.hyper_params) + +``` + +#### Expected Output + +```python +>> Model Name: My Recommendations Model +>> Model Description: A model for recommending products to users +>> Model Group: urn:li:mlModelGroup:(urn:li:dataPlatform:mlflow,my-recommendations-model,PROD) +>> Model Hyper Parameters: [MLHyperParamClass({'name': 'learning_rate', 'description': None, 'value': '0.01', 'createdAt': None}), MLHyperParamClass({'name': 'num_epochs', 'description': None, 'value': '100', 'createdAt': None}), MLHyperParamClass({'name': 'batch_size', 'description': None, 'value': '32', 'createdAt': None})] +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/operations.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/operations.md new file mode 100644 index 00000000..8a5c05a2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/operations.md @@ -0,0 +1,182 @@ +--- +title: Operations +slug: /api/tutorials/operations +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/operations.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Operations + +## Why Would You Use Operations APIs? + +The Operations APIs allow you to report operational changes that were made to a given Dataset or Table using the 'Operation' concept. +These operations may be viewed on the Dataset Profile (e.g. as last modified time), accessed via the DataHub GraphQL API, or +used as inputs to DataHub Cloud [Freshness Assertions](/docs/managed-datahub/observe/freshness-assertions.md). + +### Goal Of This Guide + +This guide will show you how to report and query Operations for a Dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed steps, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +Before reporting operations for a dataset, you need to ensure the targeted dataset is already present in DataHub. +::: + +## Report Operations + +You can use report dataset operations to DataHub using the following APIs. + + + + +```graphql +mutation reportOperation { + reportOperation( + input: { + urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + operationType: INSERT + sourceType: DATA_PROCESS + } + ) +} +``` + +Where supported operation types include + +- `INSERT` +- `UPDATE` +- `DELETE` +- `CREATE` +- `ALTER` +- `DROP` +- `CUSTOM` + +If you want to report an operation that happened at a specific time, you can also optionally provide +the `timestampMillis` field. If not provided, the current server time will be used as the operation time. + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "reportOperation": true + }, + "extensions": {} +} +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_report_operation.py +from datahub.api.graphql import Operation + +DATAHUB_HOST = "https//:org.acryl.io/gms" +DATAHUB_TOKEN = " + + +## Read Operations + +You can use read dataset operations to DataHub using the following APIs. + + + + +```graphql +query dataset { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)") { + operations( + limit: 10, filter: [], startTimeMillis: , endTimeMillis: + ) { + timestampMillis + operationType + sourceType + } + } +} +``` + +Where startTimeMillis and endTimeMillis are optional. By default, operations are sorted by time descending. + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "dataset": { + "operations": [ + { + "timestampMillis": 1231232332, + "operationType": "INSERT", + "sourceType": "DATA_PROCESS" + } + ] + } + }, + "extensions": {} +} +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_read_operations.py +from datahub.api.graphql import Operation + +DATAHUB_HOST = "https//:org.acryl.io/gms" +DATAHUB_TOKEN = ", + # end_time_millis= +) + +``` + + + + +### Expected Outcomes of Reporting Operations + +Reported Operations will appear when displaying the Last Updated time for a Dataset on their DataHub Profile. +They will also be used when selecting the `DataHub Operation` source type under the **Advanced** settings of a Freshness +Assertion. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/owners.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/owners.md new file mode 100644 index 00000000..67a676cd --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/owners.md @@ -0,0 +1,470 @@ +--- +title: Ownership +slug: /api/tutorials/owners +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/owners.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Ownership + +## Why Would You Use Users and Groups? + +Users and groups are essential for managing ownership of data. +By creating or updating user accounts and assigning them to appropriate groups, administrators can ensure that the right people can access the data they need to do their jobs. +This helps to avoid confusion or conflicts over who is responsible for specific datasets and can improve the overall effectiveness. + +### Goal Of This Guide + +This guide will show you how to + +- Create: create or update users and groups. +- Read: read owners attached to a dataset. +- Add: add user group as an owner to a dataset. +- Remove: remove the owner from a dataset. + +## Pre-requisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed information, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +In this guide, ingesting sample data is optional. +::: + +## Upsert Users + + + + +Save this `user.yaml` as a local file. + +```yaml +- id: bar@acryl.io + first_name: The + last_name: Bar + email: bar@acryl.io + slack: "@the_bar_raiser" + description: "I like raising the bar higher" + title: "Analytics Engineer" + groups: + - foogroup@acryl.io +- id: datahub + slack: "@datahubproject" + phone: "1-800-GOT-META" + description: "The DataHub Project" + title: "Data Engineer" + picture_link: "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/datahub-logo-color-stable.svg" +``` + +Execute the following CLI command to ingest user data. +Since the user datahub already exists in the sample data, any updates made to the user information will overwrite the existing data. + +``` +datahub user upsert -f user.yaml +``` + +If you see the following logs, the operation was successful: + +```shell +Update succeeded for urn urn:li:corpuser:bar@acryl.io. +Update succeeded for urn urn:li:corpuser:datahub. +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/upsert_user.py +import logging + +from datahub.api.entities.corpuser.corpuser import CorpUser, CorpUserGenerationConfig +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +user_email = "bar@acryl.io" + +user: CorpUser = CorpUser( + id=user_email, + display_name="The Bar", + email=user_email, + title="Software Engineer", + first_name="The", + last_name="Bar", + full_name="The Bar", +) + +# Create graph client +datahub_graph = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) +for event in user.generate_mcp( + generation_config=CorpUserGenerationConfig(override_editable=False) +): + datahub_graph.emit(event) +log.info(f"Upserted user {user.urn}") + +``` + + + + +### Expected Outcomes of Upserting User + +You can see the user `The bar` has been created and the user `Datahub` has been updated under `Settings > Access > Users & Groups` + +

+ +

+ +## Upsert Group + + + + +Save this `group.yaml` as a local file. Note that the group includes a list of users who are owners and members. +Within these lists, you can refer to the users by their ids or their urns, and can additionally specify their metadata inline within the group description itself. See the example below to understand how this works and feel free to make modifications to this file locally to see the effects of your changes in your local DataHub instance. + +```yaml +id: foogroup@acryl.io +display_name: Foo Group +owners: + - datahub +members: + - bar@acryl.io # refer to a user either by id or by urn + - id: joe@acryl.io # inline specification of user + slack: "@joe_shmoe" + display_name: "Joe's Hub" +``` + +Execute the following CLI command to ingest this group's information. + +``` +datahub group upsert -f group.yaml +``` + +If you see the following logs, the operation was successful: + +```shell +Update succeeded for group urn:li:corpGroup:foogroup@acryl.io. +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/upsert_group.py +import logging + +from datahub.api.entities.corpgroup.corpgroup import ( + CorpGroup, + CorpGroupGenerationConfig, +) +from datahub.ingestion.graph.client import DataHubGraph, DataHubGraphConfig +from datahub.metadata.urns import CorpUserUrn + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +group_email = "foogroup@acryl.io" +group = CorpGroup( + id=group_email, + owners=[str(CorpUserUrn("datahub"))], + members=[ + str(CorpUserUrn("bar@acryl.io")), + str(CorpUserUrn("joe@acryl.io")), + ], + display_name="Foo Group", + email=group_email, + description="Software engineering team", + slack="@foogroup", +) + +# Create graph client +datahub_graph = DataHubGraph(DataHubGraphConfig(server="http://localhost:8080")) + +for event in group.generate_mcp( + generation_config=CorpGroupGenerationConfig( + override_editable=False, datahub_graph=datahub_graph + ) +): + datahub_graph.emit(event) +log.info(f"Upserted group {group.urn}") + +``` + + + + +### Expected Outcomes of Upserting Group + +You can see the group `Foo Group` has been created under `Settings > Access > Users & Groups` + +

+ +

+ +## Read Owners + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)") { + ownership { + owners { + owner { + ... on CorpUser { + urn + type + } + ... on CorpGroup { + urn + type + } + } + } + } + } +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "dataset": { + "ownership": { + "owners": [ + { + "owner": { + "urn": "urn:li:corpuser:jdoe", + "type": "CORP_USER" + } + }, + { + "owner": { + "urn": "urn:li:corpuser:datahub", + "type": "CORP_USER" + } + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "{ dataset(urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\") { ownership { owners { owner { ... on CorpUser { urn type } ... on CorpGroup { urn type } } } } } }", "variables":{}}' +``` + +Expected Response: + +```json +{ + "data": { + "dataset": { + "ownership": { + "owners": [ + { "owner": { "urn": "urn:li:corpuser:jdoe", "type": "CORP_USER" } }, + { "owner": { "urn": "urn:li:corpuser:datahub", "type": "CORP_USER" } } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_query_owners.py +from datahub.sdk import DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="realestate_db.sales", env="PROD") +) + +print(dataset.owners) + +``` + + + + +## Add Owners + + + + +```python +mutation addOwners { + addOwner( + input: { + ownerUrn: "urn:li:corpGroup:bfoo", + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + ownerEntityType: CORP_GROUP, + type: TECHNICAL_OWNER + } + ) +} +``` + +Expected Response: + +```python +{ + "data": { + "addOwner": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation addOwners { addOwner(input: { ownerUrn: \"urn:li:corpGroup:bfoo\", resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\", ownerEntityType: CORP_GROUP, type: TECHNICAL_OWNER }) }", "variables":{}}' +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_owner.py +from datahub.sdk import CorpUserUrn, DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get(DatasetUrn(platform="hive", name="realestate_db.sales")) + +# Add owner with the TECHNICAL_OWNER type +dataset.add_owner(CorpUserUrn("jdoe")) + +client.entities.update(dataset) + +``` + + + + +## Expected Outcomes of Adding Owner + +You can now see `bfoo` has been added as an owner to the `fct_users_created` dataset. + +

+ +

+ +## Remove Owners + + + + +```json +mutation removeOwners { + removeOwner( + input: { + ownerUrn: "urn:li:corpuser:jdoe", + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + } + ) +} +``` + +Note that you can also remove owners from multiple entities or subresource using `batchRemoveOwners`. + +```json +mutation batchRemoveOwners { + batchRemoveOwners( + input: { + ownerUrns: ["urn:li:corpuser:jdoe"], + resources: [ + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)"} , + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)"} ,] + } + ) +} +``` + +Expected Response: + +```python +{ + "data": { + "removeOwner": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation removeOwner { removeOwner(input: { ownerUrn: \"urn:li:corpuser:jdoe\", resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)\" }) }", "variables":{}}' +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_remove_owner_execute_graphql.py +# read-modify-write requires access to the DataHubGraph (RestEmitter is not enough) +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +gms_endpoint = "http://localhost:8080" +graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) + +# Query multiple aspects from entity +query = """ +mutation batchRemoveOwners { + batchRemoveOwners( + input: { + ownerUrns: ["urn:li:corpuser:jdoe"], + resources: [ + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)"} , + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)"} ,] + } + ) +} +""" +result = graph.execute_graphql(query=query) + +print(result) + +``` + + + + +### Expected Outcomes of Removing Owners + +You can now see `John Doe` has been removed as an owner from the `fct_users_created` dataset. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/bulk-assertions-sdk.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/bulk-assertions-sdk.md new file mode 100644 index 00000000..9e001ed3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/bulk-assertions-sdk.md @@ -0,0 +1,688 @@ +--- +title: Bulk Creating Smart Assertions with Python SDK +slug: /api/tutorials/sdk/bulk-assertions-sdk +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/sdk/bulk-assertions-sdk.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Bulk Creating Smart Assertions with Python SDK + + + +This guide specifically covers how to use the [DataHub Cloud Python SDK](https://pypi.org/project/acryl-datahub-cloud/) for **bulk creating smart assertions**, including: + +- Smart Freshness Assertions +- Smart Volume Assertions +- Smart Column Metric Assertions +- Smart SQL Assertions + +This is particularly useful for applying data quality checks across many tables and columns at scale. + +## Why Would You Use Bulk Assertion Creation? + +Bulk creating assertions with the Python SDK allows you to: + +- **Scale data quality**: Apply consistent assertions across hundreds or thousands of tables +- **Automate assertion management**: Programmatically create and update assertions based on metadata patterns +- **Implement governance policies**: Ensure all critical tables have appropriate data quality checks +- **Save time**: Avoid manually creating assertions one by one through the UI + +## Prerequisites + +You need: + +- DataHub Cloud Python SDK installed (`pip install acryl-datahub-cloud`) +- Valid DataHub Cloud credentials configured (server URL and access token with appropriate permissions) + +The actor making API calls must have the `Edit Assertions` and `Edit Monitors` privileges for the datasets at hand. + +:::note +Before creating assertions, you need to ensure the target datasets are already present in your DataHub instance. +If you attempt to create assertions for entities that do not exist, GMS will continuously report errors to the logs. +::: + +### Goal Of This Guide + +This guide will show you how to programmatically create large numbers of smart assertions using the DataHub Cloud Python SDK. + +## Overview + +The bulk assertion creation process follows these steps: + +1. **Discover tables**: Use search or direct table queries to find datasets +2. **Create table-level assertions**: Add freshness and volume assertions for each table +3. **Get column information**: Retrieve schema details for each table +4. **Create column-level assertions**: Add column metric assertions for each relevant column +5. **Store assertion URNs**: Save assertion identifiers for future updates + +## Setup + +Connect to your DataHub instance: + +```python +from datahub.sdk import DataHubClient + +client = DataHubClient(server="", token="") +``` + +- **server**: The URL of your DataHub GMS server + - local: `http://localhost:8080` + - hosted: `https:///gms` +- **token**: You'll need to [generate a Personal Access Token](../../../authentication/personal-access-tokens.md) from your DataHub instance. + +Alternatively, initialize via using the `from_env()` method after setting the `DATAHUB_GMS_URL` and `DATAHUB_GMS_TOKEN` env vars or by creating a `~/.datahubenv` file via running `datahub init`. + +```python +from datahub.sdk import DataHubClient + +client = DataHubClient.from_env() +``` + +## Important Considerations for Parallel Processing + +- Always run bulk assertion creation for a given dataset in a single thread to avoid race conditions. +- Always call subscription APIs for a given dataset in a single thread to avoid race conditions. + - If you are subscribing to assertions directly, make sure to also run the script on a single thread per dataset. + +## Step 1: Discover Tables + +### Option A: Get Specific Tables + +```python +from datahub.metadata.urns import DatasetUrn + +# Define specific tables you want to add assertions to +table_urns = [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.users,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.orders,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.products,PROD)", +] + +# Convert to DatasetUrn objects +datasets = [DatasetUrn.from_string(urn) for urn in table_urns] +``` + +### Option B: Search for Tables by Pattern + +For comprehensive search capabilities and filter options, see the [Search API documentation](../sdk/search_client.md). + +```python +from datahub.sdk.search_filters import FilterDsl +from datahub.metadata.urns import DatasetUrn + +# Search for tables matching criteria +def find_tables_by_pattern(client, platform="snowflake", name_pattern="production_*"): + """Find tables matching a specific pattern.""" + # Create filters for datasets on a specific platform with name pattern + filters = FilterDsl.and_( + FilterDsl.entity_type("dataset"), + FilterDsl.platform(platform), + FilterDsl.custom_filter("name", "EQUAL", [name_pattern]) + ) + + # Use the search client to find matching datasets + urns = list(client.search.get_urns(filter=filters)) + return [DatasetUrn.from_string(str(urn)) for urn in urns] + +# Use the search function +datasets = find_tables_by_pattern(client, platform="snowflake", name_pattern="production_*") +``` + +### Option C: Get Tables by Tag or Domain + +```python +def find_tables_by_tag(client, tag_name="critical"): + """Find tables with a specific tag.""" + # Create filters for datasets with a specific tag + filters = FilterDsl.and_( + FilterDsl.entity_type("dataset"), + FilterDsl.custom_filter("tags", "EQUAL", [f"urn:li:tag:{tag_name}"]) + ) + + # Use the search client to find matching datasets + urns = list(client.search.get_urns(filter=filters)) + return [DatasetUrn.from_string(str(urn)) for urn in urns] + +# Find all tables tagged as "critical" +critical_datasets = find_tables_by_tag(client, "critical") +``` + +## Step 2: Create Table-Level Assertions + +### Smart Freshness Assertions + +```python +# Storage for assertion URNs (for later updates) +assertion_registry = { + "freshness": {}, + "volume": {}, + "smart_sql": {}, + "column_metrics": {} +} + +def create_freshness_assertions(datasets, client, registry): + """Create smart freshness assertions for multiple datasets.""" + + for dataset_urn in datasets: + try: + # Create smart freshness assertion + freshness_assertion = client.assertions.sync_smart_freshness_assertion( + dataset_urn=dataset_urn, + display_name=f"Freshness Anomaly Monitor", + # Detection mechanism - information_schema is recommended + detection_mechanism="information_schema", + # Smart sensitivity setting + sensitivity="medium", # options: "low", "medium", "high" + # Tags for grouping (supports urns or plain tag names!) + tags=["automated", "freshness", "data_quality"], + # Enable the assertion + enabled=True + ) + + # Store the assertion URN for future reference + registry["freshness"][str(dataset_urn)] = str(freshness_assertion.urn) + + print(f"✅ Created freshness assertion for {dataset_urn.name}: {freshness_assertion.urn}") + + except Exception as e: + print(f"❌ Failed to create freshness assertion for {dataset_urn.name}: {e}") + +# Create freshness assertions for all datasets +create_freshness_assertions(datasets, client, assertion_registry) +``` + +### Smart Volume Assertions + +```python +def create_volume_assertions(datasets, client, registry): + """Create smart volume assertions for multiple datasets.""" + + for dataset_urn in datasets: + try: + # Create smart volume assertion + volume_assertion = client.assertions.sync_smart_volume_assertion( + dataset_urn=dataset_urn, + display_name=f"Smart Volume Check", + # Detection mechanism options + detection_mechanism="information_schema", + # Smart sensitivity setting + sensitivity="medium", + # Tags for grouping + tags=["automated", "volume", "data_quality"], + # Schedule (optional - defaults to hourly) + schedule="0 */6 * * *", # Every 6 hours + # Enable the assertion + enabled=True + ) + + # Store the assertion URN + registry["volume"][str(dataset_urn)] = str(volume_assertion.urn) + + print(f"✅ Created volume assertion for {dataset_urn.name}: {volume_assertion.urn}") + + except Exception as e: + print(f"❌ Failed to create volume assertion for {dataset_urn.name}: {e}") + +# Create volume assertions for all datasets +create_volume_assertions(datasets, client, assertion_registry) +``` + +### Smart SQL Assertions + +```python +def create_smart_sql_assertions(datasets, client, registry): + """Create smart SQL assertions for multiple datasets.""" + + # Define SQL queries to run on each table + sql_queries = { + "row_count": "SELECT COUNT(*) FROM {table_name}", + "null_check": "SELECT COUNT(*) FROM {table_name} WHERE id IS NULL", + "active_records": "SELECT COUNT(*) FROM {table_name} WHERE status = 'active'", + } + + for dataset_urn in datasets: + registry["smart_sql"][str(dataset_urn)] = {} + + for query_name, query_template in sql_queries.items(): + try: + # Build the query with the table name + table_name = dataset_urn.name + statement = query_template.format(table_name=table_name) + + # Create smart SQL assertion + sql_assertion = client.assertions.sync_smart_sql_assertion( + dataset_urn=dataset_urn, + display_name=f"Smart SQL - {query_name}", + statement=statement, + # AI-powered sensitivity setting + sensitivity="medium", # options: "low", "medium", "high" + # Tags for grouping + tags=["automated", "smart_sql", query_name], + # Schedule + schedule="0 */6 * * *", # Every 6 hours + # Enable the assertion + enabled=True + ) + + # Store the assertion URN + registry["smart_sql"][str(dataset_urn)][query_name] = str(sql_assertion.urn) + + print(f"✅ Created smart SQL assertion '{query_name}' for {dataset_urn.name}: {sql_assertion.urn}") + + except Exception as e: + print(f"❌ Failed to create smart SQL assertion '{query_name}' for {dataset_urn.name}: {e}") + +# Create smart SQL assertions for all datasets +create_smart_sql_assertions(datasets, client, assertion_registry) +``` + +## Step 3: Get Column Information + +```python +def get_dataset_columns(client, dataset_urn): + """Get column information for a dataset.""" + try: + # Get dataset using the entities client + dataset = client.entities.get(dataset_urn) + if dataset and hasattr(dataset, 'schema') and dataset.schema: + return [ + { + "name": field.field_path, + "type": field.native_data_type, + "nullable": field.nullable if hasattr(field, 'nullable') else True + } + for field in dataset.schema.fields + ] + return [] + except Exception as e: + print(f"❌ Failed to get columns for {dataset_urn}: {e}") + return [] + +# Get columns for each dataset +dataset_columns = {} +for dataset_urn in datasets: + columns = get_dataset_columns(client, dataset_urn) + dataset_columns[str(dataset_urn)] = columns + print(f"📊 Found {len(columns)} columns in {dataset_urn.name}") +``` + +## Step 4: Create Column-Level Assertions + +### Smart Column Metric Assertions + +```python +def create_column_assertions(datasets, columns_dict, client, registry): + """Create smart column metric assertions for multiple datasets and columns.""" + + # Define rules for which columns should get which assertions + assertion_rules = { + # Null count checks for critical columns + "null_checks": { + "column_patterns": ["id", "*_id", "user_id", "email"], + "metric_type": "null_count", + }, + # Unique count checks for ID columns + "unique_checks": { + "column_patterns": ["*_id", "email", "username"], + "metric_type": "unique_count", + }, + # Empty count checks for string columns + "empty_checks": { + "column_patterns": ["name", "description", "title"], + "metric_type": "empty_count", + }, + } + + for dataset_urn in datasets: + dataset_key = str(dataset_urn) + columns = columns_dict.get(dataset_key, []) + + if not columns: + print(f"⚠️ No columns found for {dataset_urn.name}") + continue + + registry["column_metrics"][dataset_key] = {} + + for column in columns: + column_name = column["name"] + column_type = column["type"].upper() + + # Apply assertion rules based on column name and type + for rule_name, rule_config in assertion_rules.items(): + if should_apply_rule(column_name, column_type, rule_config): + try: + assertion = client.assertions.sync_smart_column_metric_assertion( + dataset_urn=dataset_urn, + column_name=column_name, + metric_type=rule_config["metric_type"], + display_name=f"{rule_name.replace('_', ' ').title()} - {column_name}", + # Detection mechanism for column metrics + detection_mechanism="all_rows_query_datahub_dataset_profile", + # Tags (plain names automatically converted to URNs) + tags=["automated", "column_quality", rule_name], + enabled=True + ) + + # Store assertion URN + if column_name not in registry["column_metrics"][dataset_key]: + registry["column_metrics"][dataset_key][column_name] = {} + registry["column_metrics"][dataset_key][column_name][rule_name] = str(assertion.urn) + + print(f"✅ Created {rule_name} assertion for {dataset_urn.name}.{column_name}") + + except Exception as e: + print(f"❌ Failed to create {rule_name} assertion for {dataset_urn.name}.{column_name}: {e}") + +def should_apply_rule(column_name, column_type, rule_config): + """Determine if a rule should be applied to a column.""" + import fnmatch + + # Check column name patterns + for pattern in rule_config["column_patterns"]: + if fnmatch.fnmatch(column_name.lower(), pattern.lower()): + return True + + # Add type-based rules if needed + if rule_config.get("column_types"): + return any(col_type in column_type for col_type in rule_config["column_types"]) + + return False + +# Create column assertions +create_column_assertions(datasets, dataset_columns, client, assertion_registry) +``` + +## Step 5: Create Subscription + +Reference the [Subscriptions SDK](/docs/api/tutorials/subscriptions.md) for more information on how to create subscriptions on Datasets or Assertions. + +:::note +When creating subscriptions in bulk, you must perform the operation in a single thread to avoid race conditions. Additionally, we recommend creating subscriptions at the dataset level rather than for individual assertions, as this makes ongoing management much easier. +::: + +## Step 6: Store Assertion URNs + +### Save to File + +```python +import json +from datetime import datetime + +def save_assertion_registry(registry, filename=None): + """Save assertion URNs to a file for future reference.""" + if filename is None: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"assertion_registry_{timestamp}.json" + + # Add metadata + registry_with_metadata = { + "created_at": datetime.now().isoformat(), + "total_assertions": { + "freshness": len(registry["freshness"]), + "volume": len(registry["volume"]), + "column_metrics": sum( + len(cols) for cols in registry["column_metrics"].values() + ) + }, + "assertions": registry + } + + with open(filename, 'w') as f: + json.dump(registry_with_metadata, f, indent=2) + + print(f"💾 Saved assertion registry to {filename}") + return filename + +# Save the registry +registry_file = save_assertion_registry(assertion_registry) +``` + +### Load from File (for updates) + +```python +def load_assertion_registry(filename): + """Load assertion URNs from a previously saved file.""" + with open(filename, 'r') as f: + data = json.load(f) + return data["assertions"] + +# Later, load for updates +# assertion_registry = load_assertion_registry("assertion_registry_20240101_120000.json") +``` + +## Step 7: Update Existing Assertions + +```python +def update_existing_assertions(registry, client): + """Update existing assertions using stored URNs.""" + + # Update freshness assertions + for dataset_urn, assertion_urn in registry["freshness"].items(): + try: + updated_assertion = client.assertions.sync_smart_freshness_assertion( + dataset_urn=dataset_urn, + urn=assertion_urn, # Provide existing URN for updates + # Update any parameters as needed + sensitivity="high", # Change sensitivity + tags=["automated", "freshness", "data_quality", "updated"], + enabled=True + ) + print(f"🔄 Updated freshness assertion {assertion_urn}") + except Exception as e: + print(f"❌ Failed to update freshness assertion {assertion_urn}: {e}") + +# Update assertions when needed +# update_existing_assertions(assertion_registry, client) +``` + +## Advanced Patterns + +### Conditional Assertion Creation + +```python +def create_conditional_assertions(datasets, client): + """Create assertions based on dataset metadata conditions.""" + + for dataset_urn in datasets: + try: + # Get dataset metadata + dataset = client.entities.get(dataset_urn) + + # Check if dataset has specific tags + if dataset.tags and any("critical" in str(tag.tag) for tag in dataset.tags): + # Create more stringent assertions for critical datasets + client.assertions.sync_smart_freshness_assertion( + dataset_urn=dataset_urn, + sensitivity="high", + detection_mechanism="information_schema", + tags=["critical", "automated", "freshness"] + ) + + # Check dataset size and apply appropriate volume checks + if dataset.dataset_properties: + # Create different volume assertions based on table characteristics + pass + + except Exception as e: + print(f"❌ Error processing {dataset_urn}: {e}") +``` + +### Batch Processing with Error Handling + +```python +import time +from typing import List, Dict, Any + +def batch_create_assertions( + datasets: List[DatasetUrn], + client: DataHubClient, + batch_size: int = 10, + delay_seconds: float = 1.0 +) -> Dict[str, Any]: + """Create assertions in batches with error handling and rate limiting.""" + + results = { + "successful": [], + "failed": [], + "total_processed": 0 + } + + for i in range(0, len(datasets), batch_size): + batch = datasets[i:i + batch_size] + print(f"Processing batch {i//batch_size + 1}: {len(batch)} datasets") + + for dataset_urn in batch: + try: + # Create assertion + assertion = client.assertions.sync_smart_freshness_assertion( + dataset_urn=dataset_urn, + tags=["batch_created", "automated"], + enabled=True + ) + results["successful"].append({ + "dataset_urn": str(dataset_urn), + "assertion_urn": str(assertion.urn) + }) + + except Exception as e: + results["failed"].append({ + "dataset_urn": str(dataset_urn), + "error": str(e) + }) + + results["total_processed"] += 1 + + # Rate limiting between batches + if i + batch_size < len(datasets): + time.sleep(delay_seconds) + + return results + +# Use batch processing +batch_results = batch_create_assertions(datasets, client, batch_size=5) +print(f"Batch results: {batch_results['total_processed']} processed, " + f"{len(batch_results['successful'])} successful, " + f"{len(batch_results['failed'])} failed") +``` + +## Best Practices + +### 1. **Tag Strategy** + +- Use consistent tag names for grouping assertions: `["automated", "freshness", "critical"]` +- Plain tag names are automatically converted to URNs: `"my_tag"` → `"urn:li:tag:my_tag"` +- Create a tag hierarchy for different assertion types and priorities + +### 2. **Error Handling** + +- Always wrap assertion creation in try-catch blocks +- Log failures for later investigation +- Implement retry logic for transient failures + +### 3. **URN Management** + +- Store assertion URNs in a persistent location (file, database, etc.) +- Use meaningful file naming with timestamps +- Include metadata about when and why assertions were created + +### 4. **Performance Considerations** + +Our backend is designed to handle large scale operations. However, since writes are submitted asynchronously onto a Kafka queue, you may experience significant delays in the operations being applied. If you run into any issues, here are some tips that may help: + +- **Consider running off peak** to prevent causing spikes in Kafka lag for large bulk operations +- **Before you re-run sync** (i.e. to update), wait for GMS to complete processing the previous run to prevent inconsistencies and duplicating: i.e., check if last ingested item has reflected in GMS +- **Monitor processing status** through the DataHub UI or API to ensure operations complete successfully +- **Process datasets in batches** to avoid overwhelming the API +- **Add delays** between batch processing if needed + +### 5. **Testing Strategy** + +- Start with a small subset of datasets for testing +- Validate assertion creation before bulk processing +- Test update scenarios with existing assertions + +## Complete Example Script + +```python +#!/usr/bin/env python3 +""" +Complete example script for bulk creating smart assertions. +""" + +import json +import time +from datetime import datetime +from typing import List, Dict, Any + +from datahub.sdk import DataHubClient +from datahub.ingestion.graph.client import DataHubGraph +from datahub.metadata.urns import DatasetUrn + +def main(): + # Initialize the DataHub client + client = DataHubClient( + server="https://your-datahub-instance.com", + token="your-access-token", + ) + + # The client provides both search and entity access + + # Define target datasets + table_urns = [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,prod.analytics.users,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:snowflake,prod.analytics.orders,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:snowflake,prod.analytics.products,PROD)", + ] + + datasets = [DatasetUrn.from_string(urn) for urn in table_urns] + + # Registry to store assertion URNs + assertion_registry = { + "freshness": {}, + "volume": {}, + "column_metrics": {} + } + + print(f"🚀 Starting bulk assertion creation for {len(datasets)} datasets") + + # Step 1: Create table-level assertions + print("\n📋 Creating freshness assertions...") + create_freshness_assertions(datasets, client, assertion_registry) + + print("\n📊 Creating volume assertions...") + create_volume_assertions(datasets, client, assertion_registry) + + # Step 2: Get column information and create column assertions + print("\n🔍 Analyzing columns and creating column assertions...") + dataset_columns = {} + for dataset_urn in datasets: + columns = get_dataset_columns(client, dataset_urn) + dataset_columns[str(dataset_urn)] = columns + + create_column_assertions(datasets, dataset_columns, client, assertion_registry) + + # Step 3: Save results + print("\n💾 Saving assertion registry...") + registry_file = save_assertion_registry(assertion_registry) + + # Summary + total_assertions = ( + len(assertion_registry["freshness"]) + + len(assertion_registry["volume"]) + + sum(len(cols) for cols in assertion_registry["column_metrics"].values()) + ) + + print(f"\n✅ Bulk assertion creation complete!") + print(f" 📈 Total assertions created: {total_assertions}") + print(f" 🕐 Freshness assertions: {len(assertion_registry['freshness'])}") + print(f" 📊 Volume assertions: {len(assertion_registry['volume'])}") + print(f" 🎯 Column assertions: {sum(len(cols) for cols in assertion_registry['column_metrics'].values())}") + print(f" 💾 Registry saved to: {registry_file}") + +if __name__ == "__main__": + main() +``` + +This guide provides a comprehensive approach to bulk creating smart assertions using the DataHub Cloud Python SDK. The new tag name auto-conversion feature makes it easier to organize and manage your assertions with simple, readable tag names that are automatically converted to proper URN format. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/search_client.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/search_client.md new file mode 100644 index 00000000..6ccda7cf --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/sdk/search_client.md @@ -0,0 +1,315 @@ +--- +title: Search +slug: /api/tutorials/sdk/search_client +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/sdk/search_client.md +--- +# Search + +DataHub's Python SDK makes it easy to search and discover metadata across your data ecosystem. Whether you're exploring unknown datasets, filtering by environment, or building advanced search tools, this guide walks you through how to do it all programmatically. + +**With the Search SDK, you can:** + +- Search for data assets by keyword or using structured filters +- Filter by environment, platform, type, custom properties, or other metadata fields +- Use `AND` / `OR` / `NOT` logic for advanced queries + +## Getting Started + +To use DataHub SDK, you'll need to install [`acryl-datahub`](https://pypi.org/project/acryl-datahub/) and set up a connection to your DataHub instance. Follow the [installation guide](/docs/metadata-ingestion/cli-ingestion#installing-datahub-cli) to get started. + +Connect to your DataHub instance: + +```python +from datahub.sdk import DataHubClient + +client = DataHubClient(server="", token="") +``` + +- **server**: The URL of your DataHub GMS server + - local: `http://localhost:8080` + - hosted: `https:///gms` +- **token**: You'll need to [generate a Personal Access Token](../../../authentication/personal-access-tokens.md) from your DataHub instance. + +## Search Types + +DataHub offers two primary search approaches: + +- **Query-based search** : search using simple keywords across common fields like name, description, and column names. +- **Filter-based search** : search using structured filters to scope results by platform, environment, entity type, and other metadata fields. + +:::note Combining Query and Filters + +Query and filters can be used together for more precise searches. Check out [this example](#find-all-snowflake-datasets-related-to-forecast) for more details. + +::: + +### Query-Based Search + +Query-based search allows you to search using simple keywords. This matches across common fields like name, description, and column names. This is useful for exploration when you're unsure of the exact asset you're looking for. + +#### Find All Entities Related to Sales + +For example, the script below searches for any assets that have `sales` in their metadata. + +```python +# Inlined from /metadata-ingestion/examples/library/search_with_query.py +from datahub.sdk import DataHubClient + +client = DataHubClient(server="", token="") + +# Search for entities with "sales" in the metadata +results = client.search.get_urns(query="sales") + +print(list(results)) + +``` + +Example output: + +```python +[ + DatasetUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,sales_revenue_2023,PROD)"), + DatasetUrn("urn:li:dataset:(urn:li:dataPlatform:snowflake,sales_forecast,PROD)") +] +``` + +### Filter-Based Search + +Filter-based search allows you to scope results by platform, environment, entity type, and other structured fields. +This is useful when you want to narrow down results to specific asset types or metadata fields. + +#### Find All Snowflake Entities + +For example, the script below searches for entities on the Snowflake platform. + +```python +# Inlined from /metadata-ingestion/examples/library/search_with_filter.py +from datahub.sdk import DataHubClient, FilterDsl as F + +client = DataHubClient(server="", token="") + +# Search for entities that are on snowflake platform +results = client.search.get_urns(filter=F.platform("snowflake")) + +print(list(results)) + +``` + +#### Find All Snowflake Datasets Related to Forecast + +You can combine query and filters to refine search results further. +For example, search for anything containing "forecast" that is either a chart or a Snowflake dataset. + +```python +# Inlined from /metadata-ingestion/examples/library/search_with_query_and_filter.py +from datahub.sdk import DataHubClient, FilterDsl as F + +client = DataHubClient(server="", token="") + +# Search snowflake datasets that have "forecast" in the metadata +results = client.search.get_urns( + query="forecast", filter=F.and_(F.platform("snowflake"), F.entity_type("dataset")) +) +print(list(results)) + +``` + +For more details on available filters, see the [filter options](#filter-options). + +## Common Search Patterns + +Here are some common examples of advanced queries using filters and logical operations: + +#### Find All Dashboards + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_by_entity_type.py +from datahub.sdk import DataHubClient +from datahub.sdk.search_filters import FilterDsl as F + +# search for all dashboards +client = DataHubClient(server="", token="") +results = client.search.get_urns(filter=F.entity_type("dashboard")) + +``` + +#### Find All Snowflake Entities + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_by_platform.py +from datahub.sdk import DataHubClient +from datahub.sdk.search_filters import FilterDsl as F + +# search for all snowflake assets +client = DataHubClient(server="", token="") +results = client.search.get_urns(filter=F.platform("snowflake")) + +``` + +#### Find All Entities in the Production Environment + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_by_env.py +from datahub.sdk import DataHubClient +from datahub.sdk.search_filters import FilterDsl as F + +# search for all assets in the production environment +client = DataHubClient(server="", token="") +results = client.search.get_urns(filter=F.env("PROD")) + +``` + +#### Find All Entities in a Specific Domain + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_by_domain.py +from datahub.sdk import DataHubClient +from datahub.sdk.search_filters import FilterDsl as F + +# search for all assets in the marketing domain +client = DataHubClient.from_env() +results = client.search.get_urns(filter=F.domain("urn:li:domain:marketing")) + +``` + +#### Find All Entities With a Specific Subtype + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_by_entity_subtype.py +from datahub.sdk import DataHubClient +from datahub.sdk.search_filters import FilterDsl as F + +# search for all mlflow assets of subtype "ML Experiment" +client = DataHubClient(server="", token="") +results = client.search.get_urns( + filter=F.and_(F.platform("mlflow"), F.entity_subtype("ML Experiment")) +) + +``` + +#### Find All Entities With Specific Custom Properties + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_by_custom_property.py +from datahub.sdk import DataHubClient +from datahub.sdk.search_filters import FilterDsl as F + +client = DataHubClient(server="", token="") +# search for all assets with a custom property "my_custom_property" set to "my_value" +results = client.search.get_urns( + filter=F.has_custom_property("my_custom_property", "my_value") +) + +``` + +#### Find All Charts and Snowflake Datasets + +You can combine filters using logical operations like `and_`, `or_`, and `not_` to build advanced queries. Check the [Logical Operator Options](#logical-operator-options) for more details. + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_combined_operation.py +from datahub.sdk import DataHubClient, FilterDsl as F + +client = DataHubClient(server="", token="") + +# Search for charts or snowflake datasets +results = client.search.get_urns( + filter=F.or_( + F.entity_type("chart"), + F.and_(F.platform("snowflake"), F.entity_type("dataset")), + ) +) + +print(list(results)) + +``` + +#### Find All Charts That Are Not in the Production Environment + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_not.py +from datahub.sdk import DataHubClient, FilterDsl as F + +client = DataHubClient(server="", token="") + +# Search for charts that are not in the PROD environment. +results = client.search.get_urns( + filter=F.and_(F.entity_type("chart"), F.not_(F.env("PROD"))), +) + +print(list(results)) + +``` + +#### Advanced: Find entities by other searchable fields + +Use `F.custom_filter()` to target specific fields such as urn, name, or description. Check the [Supported Conditions for Custom Filter](#supported-conditions-for-custom-filter) for the full list of allowed `condition` values. + +```python +# Inlined from /metadata-ingestion/examples/library/search_filter_custom.py +from datahub.sdk import DataHubClient, FilterDsl as F + +client = DataHubClient(server="", token="") + +# Search for datasets that have "example_dataset" in the urn +results = client.search.get_urns( + filter=F.custom_filter(field="urn", condition="CONTAIN", values=["example_dataset"]) +) + +print(list(results)) + +``` + +:::note Searchable Fields +With `F.custom_filter()`, the fields annotated with `@Searchable` in the PDL file can be used for filtering. For example, you can filter datajob entities by fields like `name`, `description`, or `env` since they are annotated with `@Searchable` in the [DataJobInfo.pdl](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/datajob/DataJobInfo.pdl#L21). +::: + +## Search SDK Reference + +For a full reference, see the [search SDK reference](../../../../python-sdk/sdk-v2/search-client.mdx). + +### Filter Options + +The following filter options are available in the SDK: + +| Filter Type | Example Code | +| --------------- | ---------------------------------------------- | +| Platform | `F.platform("snowflake")` | +| Environment | `F.env("PROD")` | +| Entity Type | `F.entity_type("dataset")` | +| Domain | `F.domain("urn:li:domain:xyz")` | +| Subtype | `F.entity_subtype("ML Experiment")` | +| Deletion Status | `F.soft_deleted("NOT_SOFT_DELETED")` | +| Custom Property | `F.has_custom_property("department", "sales")` | + +### Logical Operator Options + +The following logical operators can be used to combine filters: + +| Operator | Example Code | Description | +| -------- | ------------- | -------------------------------------------------- | +| AND | `F.and_(...)` | Return entities matching all specified conditions. | +| OR | `F.or_(...)` | Return entities matching at least one condition. | +| NOT | `F.not_(...)` | Exclude entities that match a given condition. | + +### Supported Conditions for Custom Filter + +Use `F.custom_filter()` to apply conditions on specific fields such as urn, name, or description. + +| Condition | Description | +| -------------- | ----------------------------------------------------------------------------------------- | +| `EQUAL` | Exact match for string fields. | +| `CONTAIN` | Contains substring in string fields. | +| `START_WITH` | Begins with a specific substring. | +| `END_WITH` | Ends with a specific substring. | +| `GREATER_THAN` | For numeric or timestamp fields, checks if the value is greater than the specified value. | +| `LESS_THAN` | For numeric or timestamp fields, checks if the value is less than the specified value. | + +## FAQ + +**How do I handle authentication?** +Generate a Personal Access Token from your DataHub instance settings and pass it into the `DataHubClient`. Check out the [Personal Access Token Guide](../../../authentication/personal-access-tokens.md). + +**Can I combine query and filters?** +Yes. Use `query` along with `filter` for more precise searches. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/structured-properties.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/structured-properties.md new file mode 100644 index 00000000..8071bfc1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/structured-properties.md @@ -0,0 +1,2080 @@ +--- +title: Structured Properties +slug: /api/tutorials/structured-properties +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/structured-properties.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Structured Properties + +## Why Would You Use Structured Properties? + +Structured properties are a structured, named set of properties that can be attached to logical entities like Datasets, DataJobs, etc. +Structured properties have values that are typed and support constraints. + +Learn more about structured properties in the [Structured Properties Feature Guide](../../../docs/features/feature-guides/properties/overview.md). + +### Goal Of This Guide + +This guide will show you how to execute the following actions with structured properties. + +- Create structured properties +- List structured properties +- Read structured properties +- Delete structured properties +- Add structured properties to a dataset +- Patch structured properties (add / remove / update a single property) +- Update structured property with breaking schema changes +- Search & aggregations using structured properties + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed information, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +Additionally, you need to have the following tools installed according to the method you choose to interact with DataHub: + + + + +Install the relevant CLI version. +Structured Properties were introduced in version `0.13.1`, but we continuously improve and add new functionality, so you should always [upgrade](/docs/cli/#installation) to the latest cli for best results. +Connect to your instance via [init](/docs/cli/#init): + +- Run `datahub init` to update the instance you want to load into. +- Set the server to your sandbox instance, `https://{your-instance-address}/gms`. +- Set the token to your access token. + + + + +Requirements for OpenAPI are: + +- curl +- jq + + + + +## Create Structured Properties + +The following code will create a structured property `io.acryl.privacy.retentionTime`. + + + + + +Create a yaml file representing the properties you’d like to load. +For example, below file represents a property `io.acryl.privacy.retentionTime`. You can see the full example [here](https://github.com/datahub-project/datahub/blob/example-yaml-sp/metadata-ingestion/examples/structured_properties/struct_props.yaml). + +```yaml +- id: io.acryl.privacy.retentionTime + # - urn: urn:li:structuredProperty:io.acryl.privacy.retentionTime # optional if id is provided + qualified_name: io.acryl.privacy.retentionTime # required if urn is provided + type: number + cardinality: MULTIPLE + display_name: Retention Time + entity_types: + - dataset # or urn:li:entityType:datahub.dataset + - dataFlow + description: "Retention Time is used to figure out how long to retain records in a dataset" + allowed_values: + - value: 30 + description: 30 days, usually reserved for datasets that are ephemeral and contain pii + - value: 90 + description: Use this for datasets that drive monthly reporting but contain pii + - value: 365 + description: Use this for non-sensitive data that can be retained for longer +``` + +Use the CLI to create your properties: + +```shell +datahub properties upsert -f {properties_yaml} +``` + +If successful, you should see `Created structured property urn:li:structuredProperty:...` + + + + + +```graphql +mutation createStructuredProperty { + createStructuredProperty( + input: { + id: "retentionTime" + qualifiedName: "retentionTime" + displayName: "Retention Time" + description: "Retention Time is used to figure out how long to retain records in a dataset" + valueType: "urn:li:dataType:datahub.number" + allowedValues: [ + { + numberValue: 30 + description: "30 days, usually reserved for datasets that are ephemeral and contain pii" + } + { + numberValue: 90 + description: "description: Use this for datasets that drive monthly reporting but contain pii" + } + { + numberValue: 365 + description: "Use this for non-sensitive data that can be retained for longer" + } + ] + cardinality: SINGLE + entityTypes: [ + "urn:li:entityType:datahub.dataset" + "urn:li:entityType:datahub.dataFlow" + ] + } + ) { + urn + } +} +``` + + + + + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "displayName": "Retention Time", + "cardinality": "MULTIPLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "allowedValues": [ + { + "value": {"double": 30}, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": {"double": 60}, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": {"double": 365}, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] +}' | jq +``` + + + + + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "value": { + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "displayName": "Retention Time", + "cardinality": "MULTIPLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "allowedValues": [ + { + "value": { + "double": 30 + }, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": { + "double": 60 + }, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": { + "double": 365 + }, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] + } +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii", + "value": { + "double": 30 + } + }, + { + "description": "Use this for datasets that drive monthly reporting but contain pii", + "value": { + "double": 60 + } + }, + { + "description": "Use this for non-sensitive data that can be retained for longer", + "value": { + "double": 365 + } + } + ], + "displayName": "Retention Time", + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "cardinality": "MULTIPLE" + } + } +} +``` + + + + +## List Structured Properties + +You can list all structured properties in your DataHub instance using the following methods: + + + + +```shell +datahub properties list +``` + +This will show all properties with their full details. + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:clusterName", + "qualified_name": "clusterName", + "type": "urn:li:dataType:datahub.string", + "description": "Test Cluster Name Property", + "display_name": "Cluster's name", + "entity_types": [ + "urn:li:entityType:datahub.dataset" + ], + "cardinality": "SINGLE" +} +{ + "urn": "urn:li:structuredProperty:projectNames", + "qualified_name": "projectNames", + "type": "urn:li:dataType:datahub.string", + "description": "Test property for project name", + "display_name": "Project Name", + "entity_types": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "cardinality": "MULTIPLE", + "allowed_values": [ + { + "value": "Tracking", + "description": "test value 1 for project" + }, + { + "value": "DataHub", + "description": "test value 2 for project" + } + ] +} +``` + +If you only want to see the URNs, you can use: + +```shell +datahub properties list --no-details +``` + +Example Response: + +``` +[2025-01-08 22:23:00,625] INFO {datahub.cli.specific.structuredproperties_cli:134} - Listing structured property urns only, use --details for more information +urn:li:structuredProperty:clusterName +urn:li:structuredProperty:clusterType +urn:li:structuredProperty:io.acryl.dataManagement.deprecationDate +urn:li:structuredProperty:projectNames +``` + +To download all the structured property definitions into a single file that you can use with the `upsert` command as described in the [create section](#create-structured-properties), you can run the list command with the `--to-file` option. + +```shell +datahub properties list --to-file structured_properties.yaml +``` + +Example Response: + +```yaml + - urn: urn:li:structuredProperty:clusterName + qualified_name: clusterName + type: urn:li:dataType:datahub.string + description: Test Cluster Name Property + display_name: Cluster's name + entity_types: + - urn:li:entityType:datahub.dataset + cardinality: SINGLE + - urn: urn:li:structuredProperty:clusterType + qualified_name: clusterType + type: urn:li:dataType:datahub.string + description: Test Cluster Type Property + display_name: Cluster's type + entity_types: + - urn:li:entityType:datahub.dataset + cardinality: SINGLE + - urn: urn:li:structuredProperty:io.acryl.dataManagement.deprecationDate + qualified_name: io.acryl.dataManagement.deprecationDate + type: urn:li:dataType:datahub.date + display_name: Deprecation Date + entity_types: + - urn:li:entityType:datahub.dataset + - urn:li:entityType:datahub.dataFlow + - urn:li:entityType:datahub.dataJob + - urn:li:entityType:datahub.schemaField + cardinality: SINGLE + - urn: urn:li:structuredProperty:io.acryl.privacy.enumProperty5712 + qualified_name: io.acryl.privacy.enumProperty5712 + type: urn:li:dataType:datahub.string + description: The retention policy for the dataset + entity_types: + - urn:li:entityType:datahub.dataset + cardinality: MULTIPLE + allowed_values: + - value: foo + - value: bar +... etc. +``` + + + + + +Example Request: + +```bash +curl -X 'GET' \ + 'http://localhost:9002/openapi/v3/entity/structuredproperty?systemMetadata=false&includeSoftDelete=false&skipCache=false&aspects=structuredPropertySettings&aspects=propertyDefinition&aspects=institutionalMemory&aspects=structuredPropertyKey&aspects=status&count=10&sortCriteria=urn&sortOrder=ASCENDING&query=*' \ + -H 'accept: application/json' +``` + +Example Response: + +```json +{ + "scrollId": "...", + "entities": [ + { + "urn": "urn:li:structuredProperty:clusterName", + "propertyDefinition": { + "value": { + "immutable": false, + "qualifiedName": "clusterName", + "displayName": "Cluster's name", + "valueType": "urn:li:dataType:datahub.string", + "description": "Test Cluster Name Property", + "entityTypes": ["urn:li:entityType:datahub.dataset"], + "cardinality": "SINGLE" + } + }, + "structuredPropertyKey": { + "value": { + "id": "clusterName" + } + } + } + ] +} +``` + +Key Query Parameters: + +- `count`: Number of results to return per page (default: 10) +- `sortCriteria`: Field to sort by (default: urn) +- `sortOrder`: Sort order (ASCENDING or DESCENDING) +- `query`: Search query to filter properties (\* for all) + + + + +The list endpoint returns all structured properties in your DataHub instance. Each property includes: + +- URN: Unique identifier for the property +- Qualified Name: The property's qualified name +- Type: The data type of the property (string, number, date, etc.) +- Description: A description of the property's purpose +- Display Name: Human-readable name for the property +- Entity Types: The types of entities this property can be applied to +- Cardinality: Whether the property accepts single (SINGLE) or multiple (MULTIPLE) values +- Allowed Values: If specified, the list of allowed values for this property + +## Read a single Structured Property + +You can read an individual property you created by running the following command: + + + + +```commandline +datahub properties get --urn {urn} +``` + +For example, you can run `datahub properties get --urn urn:li:structuredProperty:io.acryl.privacy.retentionTime`. +If successful, you should see metadata about your properties returned. + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "qualified_name": "io.acryl.privacy.retentionTime", + "type": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "display_name": "Retention Time", + "entity_types": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "cardinality": "MULTIPLE", + "allowed_values": [ + { + "value": "30", + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": "90", + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": "365", + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] +} +``` + + + + +Example Request: + +```graphql +query { + structuredProperty(urn: "urn:li:structuredProperty:projectNames") { + urn + type + definition { + qualifiedName + displayName + description + cardinality + allowedValues { + value { + ... on StringValue { + stringValue + } + ... on NumberValue { + numberValue + } + } + description + } + entityTypes { + urn + info { + type + qualifiedName + } + } + } + } +} +``` + +Example Response: + +```json +{ + "data": { + "structuredProperty": { + "urn": "urn:li:structuredProperty:projectNames", + "type": "STRUCTURED_PROPERTY", + "definition": { + "qualifiedName": "projectNames", + "displayName": "Project Name", + "description": "Test property for project name", + "cardinality": "MULTIPLE", + "allowedValues": [ + { + "value": { + "stringValue": "Tracking" + }, + "description": "test value 1 for project" + }, + { + "value": { + "stringValue": "DataHub" + }, + "description": "test value 2 for project" + } + ], + "entityTypes": [ + { + "urn": "urn:li:entityType:datahub.dataset", + "info": { + "type": "DATASET", + "qualifiedName": "datahub.dataset" + } + }, + { + "urn": "urn:li:entityType:datahub.dataFlow", + "info": { + "type": "DATA_FLOW", + "qualifiedName": "datahub.dataFlow" + } + } + ] + } + } + }, + "extensions": {} +} +``` + + + + + +Example Request: + +``` +curl -X 'GET' -v \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ + -H 'accept: application/json' | jq +``` + +Example Response: + +```json +{ + "value": { + "allowedValues": [ + { + "value": { + "double": 30.0 + }, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": { + "double": 60.0 + }, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": { + "double": 365.0 + }, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ], + "qualifiedName": "io.acryl.privacy.retentionTime", + "displayName": "Retention Time", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "cardinality": "MULTIPLE" + } +} +``` + + + + + +Example Request: + +``` +curl -X 'GET' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ + -H 'accept: application/json' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii", + "value": { + "double": 30 + } + }, + { + "description": "Use this for datasets that drive monthly reporting but contain pii", + "value": { + "double": 60 + } + }, + { + "description": "Use this for non-sensitive data that can be retained for longer", + "value": { + "double": 365 + } + } + ], + "displayName": "Retention Time", + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "cardinality": "MULTIPLE" + } + } +} +``` + + + + +## Set Structured Property To a Dataset + +This action will set/replace all structured properties on the entity. See PATCH operations to add/remove a single property. + + + + +```graphql +mutation upsertStructuredProperties { + upsertStructuredProperties( + input: { + assetUrn: "urn:li:mydataset1" + structuredPropertyInputParams: [ + { + structuredPropertyUrn: "urn:li:structuredProperty:mystructuredproperty" + values: [{ stringValue: "123" }] + } + ] + } + ) { + properties { + structuredProperty { + urn + } + } + } +} +``` + + + + +You can set structured properties to a dataset by creating a dataset yaml file with structured properties. For example, below is a dataset yaml file with structured properties in both the field and dataset level. + +Please refer to the [full example here.](https://github.com/datahub-project/datahub/blob/example-yaml-sp/metadata-ingestion/examples/structured_properties/datasets.yaml) + +```yaml +- id: user_clicks_snowflake + platform: snowflake + schema: + fields: + - id: user_id + structured_properties: + io.acryl.dataManagement.deprecationDate: "2023-01-01" + structured_properties: + io.acryl.dataManagement.replicationSLA: 90 +``` + +Use the CLI to upsert your dataset yaml file: + +```commandline +datahub dataset upsert -f {dataset_yaml} +``` + +If successful, you should see `Update succeeded for urn:li:dataset:...` + + + + + +Following command will set structured properties `retentionTime` as `60.0` to a dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. +Please note that the structured property and the dataset must exist before executing this command. (You can create sample datasets using the `datahub docker ingest-sample-data`) + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} + ] + } + ] +}' | jq +``` + + + + + +Following command will set structured properties `retentionTime` as `60.0` to a dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`. +Please note that the structured property and the dataset must exist before executing this command. (You can create sample datasets using the `datahub docker ingest-sample-data`) + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} + ] + } + ] + } +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "double": 60 + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ] + } + } +} +``` + + + + +#### Expected Outcomes + +Once your datasets are uploaded, you can view them in the UI and view the properties associated with them under the Properties tab. + +

+ +

+ +Or you can run the following command to view the properties associated with the dataset: + +```commandline +datahub dataset get --urn {urn} +``` + +## Read Structured Properties From a Dataset + +For reading all structured properties from a dataset: + + + + +```graphql +query getDataset { + dataset( + urn: "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.ecommerce.customer,PROD)" + ) { + structuredProperties { + properties { + structuredProperty { + urn + type + definition { + displayName + description + allowedValues { + description + } + } + } + values { + ... on StringValue { + stringValue + } + ... on NumberValue { + numberValue + } + } + valueEntities { + urn + type + } + } + } + } +} +``` + + + + +## Remove Structured Properties From a Dataset + +For removing a structured property or list of structured properties from a dataset: + + + + +```graphql +mutation removeStructuredProperties { + removeStructuredProperties( + input: { + assetUrn: "urn:li:mydataset1" + structuredPropertyUrns: ["urn:li:structuredProperty:mystructuredproperty"] + } + ) { + properties { + structuredProperty { + urn + } + } + } +} +``` + + + + +## Patch Structured Property Value + +This section will show you how to patch a structured property value - either by removing, adding, or upserting a single property. + +### Add Structured Property Value + +For this example, we'll extend create a second structured property and apply both properties to the same dataset used previously. +After this your system should include both `io.acryl.privacy.retentionTime` and `io.acryl.privacy.retentionTime02`. + + + + +Let's start by creating the second structured property. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime02/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "qualifiedName": "io.acryl.privacy.retentionTime02", + "displayName": "Retention Time 02", + "valueType": "urn:li:dataType:datahub.string", + "allowedValues": [ + { + "value": {"string": "foo2"}, + "description": "test foo2 value" + }, + { + "value": {"string": "bar2"}, + "description": "test bar2 value" + } + ], + "cardinality": "SINGLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ] +}' | jq + +``` + +This command will attach one of each of the two properties to our test dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)` +Specifically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acryl.privacy.retentionTime02` as `bar2`. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + {"string": "bar2"} + ] + } + ] +}' | jq +``` + + + + + +Let's start by creating the second structured property. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime02/propertyDefinition' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "value": { + "qualifiedName": "io.acryl.privacy.retentionTime02", + "displayName": "Retention Time 02", + "valueType": "urn:li:dataType:datahub.string", + "allowedValues": [ + { + "value": {"string": "foo2"}, + "description": "test foo2 value" + }, + { + "value": {"string": "bar2"}, + "description": "test bar2 value" + } + ], + "cardinality": "SINGLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ] + } +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "value": { + "string": "foo2" + }, + "description": "test foo2 value" + }, + { + "value": { + "string": "bar2" + }, + "description": "test bar2 value" + } + ], + "entityTypes": ["urn:li:entityType:datahub.dataset"], + "qualifiedName": "io.acryl.privacy.retentionTime02", + "displayName": "Retention Time 02", + "cardinality": "SINGLE", + "valueType": "urn:li:dataType:datahub.string" + } + } +} +``` + +This command will attach one of each of the two properties to our test dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)` +Specically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acryl.privacy.retentionTime02` as `bar2`. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties?createIfNotExists=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + {"string": "bar2"} + ] + } + ] + } +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "double": 60 + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + }, + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + } + ] + } + } +} +``` + + + + +#### Expected Outcomes + +You can see that the dataset now has two structured properties attached to it. + +

+ +

+ +### Remove Structured Property Value + +The expected state of our test dataset include 2 structured properties. +We'd like to remove the first one (`io.acryl.privacy.retentionTime`) and preserve the second property. (`io.acryl.privacy.retentionTime02`). + + + + +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "remove", + "path": "/properties/urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` + +The response will show that the expected property has been removed. + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": { + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + } + ] + } + } + } +} +``` + + + + + +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "remove", + "path": "/properties/urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` + +The response will show that the expected property has been removed. + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + } + ] + } + } +} +``` + + + + + +#### Expected Outcomes + +You can see that the first property has been removed and the second property is still present. + +

+ +

+ +### Upsert Structured Property Value + +In this example, we'll add the property back with a different value, preserving the existing property. + + + + +```graphql +mutation updateStructuredProperty { + updateStructuredProperty( + input: { + urn: "urn:li:structuredProperty:retentionTime" + displayName: "Retention Time" + description: "Retention Time is used to figure out how long to retain records in a dataset" + newAllowedValues: [ + { + numberValue: 30 + description: "30 days, usually reserved for datasets that are ephemeral and contain pii" + } + { + numberValue: 90 + description: "Use this for datasets that drive monthly reporting but contain pii" + } + { + numberValue: 365 + description: "Use this for non-sensitive data that can be retained for longer" + } + ] + } + ) { + urn + } +} +``` + + + + +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v2/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "add", + "path": "/properties/urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "value": { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 365.0 + } + ] + } + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": { + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + }, + { + "values": [ + { + "double": 365.0 + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ] + } + } + } +} +``` + +The response shows that the property was re-added with the new value 365.0 instead of the previous value 60.0. + + + + + +```shell +curl -X 'PATCH' -v \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json-patch+json' \ + -d '{ + "patch": [ + { + "op": "add", + "path": "/properties/urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "value": { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 365.0 + } + ] + } + } + ], + "arrayPrimaryKeys": { + "properties": [ + "propertyUrn" + ] + } + }' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "structuredProperties": { + "value": { + "properties": [ + { + "values": [ + { + "string": "bar2" + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02" + }, + { + "values": [ + { + "double": 365 + } + ], + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime" + } + ] + } + } +} +``` + +The response shows that the property was re-added with the new value 365 instead of the previous value 60. + + + + + +#### Expected Outcomes + +You can see that the first property has been added back with a new value and the second property is still present. + +

+ +

+ +## Delete Structured Properties + +There are two types of deletion present in DataHub: hard and soft delete. + +:::note SOFT DELETE +A soft deleted Structured Property does not remove any underlying data on the Structured Property entity or the Structured Property's values written to other entities. +The soft delete is 100% reversible with zero data loss. When a Structured Property is soft deleted, a few operations are not available. + +Structured Property Soft Delete Effects: + +- Entities with a soft deleted Structured Property value will not return the soft deleted properties +- Updates to a soft deleted Structured Property's definition are denied +- Adding a soft deleted Structured Property's value to an entity is denied +- Search filters using a soft deleted Structured Property will be denied + +::: + +:::note HARD DELETE +A hard deleted Structured Property REMOVES all underlying data for the Structured Property entity and the Structured Property's values written to other entities. +The hard delete is NOT reversible. + +Structured Property Hard Delete Effects: + +- Structured Property entity is removed +- Structured Property values are removed via PATCH MCPs on their respective entities +- Rollback is not possible +- Elasticsearch index mappings will continue to contain references to the hard deleted property until reindex + +::: + +### Soft Delete + + + + +The following command will soft delete the test property. + +```commandline +datahub delete --urn {urn} +``` + + + + +The following command will soft delete the test property by writing to the status aspect. + +```shell +curl -X 'POST' \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ +"removed": true +}' | jq +``` + +If you want to **remove the soft delete**, you can do so by either hard deleting the status aspect or changing the removed boolean to `false` like below. + +```shell +curl -X 'POST' \ + 'http://localhost:8080/openapi/v2/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ +"removed": false +}' | jq +``` + + + + + +The following command will soft delete the test property by writing to the status aspect. + +```shell +curl -X 'POST' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "value": { + "removed": true + } +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "status": { + "value": { + "removed": true + } + } +} +``` + +If you want to **remove the soft delete**, you can do so by either hard deleting the status aspect or changing the removed boolean to `false` like below. + +```shell +curl -X 'POST' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false&createIfNotExists=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "value": { + "removed": true + } +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "status": { + "value": { + "removed": false + } + } +} +``` + + + + + +### Hard Delete + + + + +The following command will hard delete the test property. + +```commandline +datahub delete --urn {urn} --hard +``` + + + + + +The following command will hard delete the test property. + +```shell +curl -v -X 'DELETE' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime' +``` + +Example Response: + +```text +> DELETE /openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime HTTP/1.1 +> Host: localhost:8080 +> User-Agent: curl/8.4.0 +> Accept: */* +> +< HTTP/1.1 200 OK +< Date: Fri, 14 Jun 2024 17:30:27 GMT +< Content-Length: 0 +< Server: Jetty(11.0.19) +``` + + + + + +#### Index Mappings Cleanup + +After the asynchronous delete of all Structured Property values have been processed, triggered by the above +hard delete, it is possible to remove the remaining index mappings. Note that if even 1 Structured Property value remains +the mapping will not be removed for a given entity index. + +Run the DataHub system-update job (automatically run with every helm upgrade or install and quickstart) with +the following environment variables enabled. + +This will trigger an ES index which will take time to complete. During the process the entire index is recreated. + +```shell +ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true +ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE=true +``` + +## Update Structured Property With Breaking Schema Changes + +This section demonstrates how to make backwards incompatible schema changes to a Structured Property definition. + +Breaking schema changes require setting a `version` string in the Structured Property definition. + +The format `yyyyMMddhhmmss` (for example, `20240614080000`) is a recommended convention, but not a strict requirement. Semantic versioning or other formats also work. Note that only the most recently created version is active. + +:::caution IMPORTANT + +Making a breaking schema change will **invalidate existing values** for this Structured Property in search. + +- Existing values will still be viewable in the UI and retrievable via GraphQL and the API +- Existing values will **not** appear in search results, filters, or aggregations +- The old values will be cleaned up asynchronously (future work) + +If you need existing values to remain searchable, consider creating a new Structured Property with a different name instead of modifying the existing one. + +::: + +### When would you use this? + +The most common scenario is changing cardinality from `MULTIPLE` to `SINGLE`. If assets already have multiple values stored, those values don't fit the new schema and get invalidated in the search index. + +In the following example, we'll revisit the `retentionTime` structured property and apply a breaking change +by changing the cardinality from `MULTIPLE` to `SINGLE`. Normally this change would be rejected as a +backwards incompatible change since values that were previously written may have multiple values written +which would no longer be valid. + + + + +Edit the previously created definition yaml: Change the cardinality to `SINGLE` and add a `version`. + +```yaml +- id: io.acryl.privacy.retentionTime + # - urn: urn:li:structuredProperty:io.acryl.privacy.retentionTime # optional if id is provided + qualified_name: io.acryl.privacy.retentionTime # required if urn is provided + type: number + cardinality: SINGLE + version: "20240614080000" + display_name: Retention Time + entity_types: + - dataset # or urn:li:entityType:datahub.dataset + - dataFlow + description: "Retention Time is used to figure out how long to retain records in a dataset" + allowed_values: + - value: 30 + description: 30 days, usually reserved for datasets that are ephemeral and contain pii + - value: 90 + description: Use this for datasets that drive monthly reporting but contain pii + - value: 365 + description: Use this for non-sensitive data that can be retained for longer +``` + +Use the CLI to create your properties: + +```commandline +datahub properties upsert -f {properties_yaml} +``` + +If successful, you should see `Created structured property urn:li:structuredProperty:...` + + + + +Change the cardinality to `SINGLE` and add a `version`. + +```shell +curl -X 'POST' -v \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition?createIfNotExists=false' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "value": { + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "displayName": "Retention Time", + "cardinality": "SINGLE", + "version": "20240614080000", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "allowedValues": [ + { + "value": { + "double": 30 + }, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": { + "double": 60 + }, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": { + "double": 365 + }, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] + } +}' | jq +``` + +Example Response: + +```json +{ + "urn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "propertyDefinition": { + "value": { + "allowedValues": [ + { + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii", + "value": { + "double": 30 + } + }, + { + "description": "Use this for datasets that drive monthly reporting but contain pii", + "value": { + "double": 60 + } + }, + { + "description": "Use this for non-sensitive data that can be retained for longer", + "value": { + "double": 365 + } + } + ], + "displayName": "Retention Time", + "qualifiedName": "io.acryl.privacy.retentionTime", + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "version": "20240614080000", + "cardinality": "SINGLE" + } + } +} +``` + + + + +## Structured Properties - Search & Aggregation + +Currently Structured Properties can be used to filter search results. This currently excludes fulltext search. + +The following examples re-use the two previously defined Structured Properties. + +`io.acryl.privacy.retentionTime` - An example numeric property. + +`io.acryl.privacy.retentionTime02` - An example string property. + + + + +Range Query: + +Document should be returned based on the previously assigned value of 60. + +```graphql +query { + scrollAcrossEntities( + input: { + types: DATASET + count: 10 + query: "*" + orFilters: { + and: [ + { + field: "structuredProperties.io.acryl.privacy.retentionTime" + condition: GREATER_THAN + values: ["45.0"] + } + ] + } + } + ) { + searchResults { + entity { + urn + type + } + } + } +} +``` + +Exists Query: + +Document should be returned based on the previously assigned value. + +```graphql +query { + scrollAcrossEntities( + input: { + types: DATASET + count: 10 + query: "*" + orFilters: { + and: [ + { + field: "structuredProperties.io.acryl.privacy.retentionTime" + condition: EXISTS + } + ] + } + } + ) { + searchResults { + entity { + urn + type + } + } + } +} +``` + +Equality Query: + +Document should be returned based on the previously assigned value of 'bar2'. + +```graphql +query { + scrollAcrossEntities( + input: { + types: DATASET + count: 10 + query: "*" + orFilters: { + and: [ + { + field: "structuredProperties.io.acryl.privacy.retentionTime02" + condition: EQUAL + values: ["bar2"] + } + ] + } + } + ) { + searchResults { + entity { + urn + type + } + } + } +} +``` + + + + + +Unlike GraphQL which has a parsed input object for filtering, OpenAPI only includes a structured query which +relies on the `query_string` syntax. See the Elasticsearch [documentation](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-query-string-query.html) for detailed syntax. + +In order to use the `query_string` syntax we'll need to know a bit about the Structured Property's definition such +as whether it is versioned or un-unversioned and its type. This information will be added to the `query` url parameter. + +Un-versioned Example: + +Structured Property URN - `urn:li:structuredProperty:io.acryl.privacy.retentionTime` + +Elasticsearch Field Name - `structuredProperties.io_acryl_privacy_retentionTime` + +Versioned: + +Structured Property Version - `20240614080000` + +Structured Property Type - `string` + +Structured Property URN - `urn:li:structuredProperty:io.acryl.privacy.retentionTime02` + +Elasticsearch Field Name - `structuredProperties._versioned.io_acryl_privacy_retentionTime02.20240614080000.string` + +Range Query: + +query - `structuredProperties.io_acryl_privacy_retentionTime:>45` + +```shell +curl -X 'GET' \ + 'http://localhost:9002/openapi/v3/entity/dataset?systemMetadata=false&aspects=datasetKey&aspects=structuredProperties&count=10&sort=urn&sortOrder=ASCENDING&query=structuredProperties.io_acryl_privacy_retentionTime%3A%3E45' \ + -H 'accept: application/json' +``` + +Example Response: + +```json +{ + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "datasetKey": { + "value": { + "name": "SampleHiveDataset", + "platform": "urn:li:dataPlatform:hive", + "origin": "PROD" + } + }, + "structuredProperties": { + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 60 + } + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + { + "string": "bar2" + } + ] + } + ] + } + } + } + ] +} +``` + +Exists Query: + +query - `_exists_:structuredProperties.io_acryl_privacy_retentionTime` + +```shell +curl -X 'GET' \ + 'http://localhost:9002/openapi/v3/entity/dataset?systemMetadata=false&aspects=datasetKey&aspects=structuredProperties&count=10&sort=urn&sortOrder=ASCENDING&query=_exists_%3AstructuredProperties.io_acryl_privacy_retentionTime' \ + -H 'accept: application/json' +``` + +Example Response: + +```json +{ + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "datasetKey": { + "value": { + "name": "SampleHiveDataset", + "platform": "urn:li:dataPlatform:hive", + "origin": "PROD" + } + }, + "structuredProperties": { + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 60 + } + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + { + "string": "bar2" + } + ] + } + ] + } + } + } + ] +} +``` + +Equality Query: + +query - `structuredProperties._versioned.io_acryl_privacy_retentionTime02.20240614080000.string` + +```shell +curl -X 'GET' \ + 'http://localhost:9002/openapi/v3/entity/dataset?systemMetadata=false&aspects=datasetKey&aspects=structuredProperties&count=10&sort=urn&sortOrder=ASCENDING&query=structuredProperties._versioned.io_acryl_privacy_retentionTime02.20240614080000.string' \ + -H 'accept: application/json' +``` + +Example Response: + +```json +{ + "entities": [ + { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "datasetKey": { + "value": { + "name": "SampleHiveDataset", + "platform": "urn:li:dataPlatform:hive", + "origin": "PROD" + } + }, + "structuredProperties": { + "value": { + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + { + "double": 60 + } + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + { + "string": "bar2" + } + ] + } + ] + } + } + } + ] +} +``` + + + + +### Structured Property Aggregations + +Structured properties can also be used in GraphQL's aggregation queries using the same naming convention outlined above +for search filter field names. There are currently no aggregation endpoints for OpenAPI. + + + + +Aggregation Query: + +```graphql +query { + aggregateAcrossEntities( + input: { + types: [] + facets: [ + "structuredProperties.io.acryl.privacy.retentionTime02" + "structuredProperties.io.acryl.privacy.retentionTime" + ] + query: "*" + orFilters: [] + searchFlags: { maxAggValues: 100 } + } + ) { + facets { + field + aggregations { + value + count + } + } + } +} +``` + +Example Response: + +```json +{ + "data": { + "aggregateAcrossEntities": { + "facets": [ + { + "field": "structuredProperties.io.acryl.privacy.retentionTime02", + "aggregations": [ + { + "value": "bar2", + "count": 1 + } + ] + }, + { + "field": "structuredProperties.io.acryl.privacy.retentionTime", + "aggregations": [ + { + "value": "60.0", + "count": 1 + } + ] + } + ] + } + }, + "extensions": {} +} +``` + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/subscriptions.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/subscriptions.md new file mode 100644 index 00000000..663d0bbc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/subscriptions.md @@ -0,0 +1,195 @@ +--- +title: Subscriptions +slug: /api/tutorials/subscriptions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/subscriptions.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Subscriptions + + + +## Why Would You Use Subscriptions on Datasets? + +Subscriptions are a way to receive notifications when entity changes occur (e.g. deprecations, schema changes, ownership changes, etc.) or when assertions change state (pass, fail, or error). Subscriptions can be created at the dataset level (affecting any changes on the dataset, as well as all assertions on the dataset) or at the assertion level (affecting only specific assertions). + +### Goal Of This Guide + +This guide specifically covers how to use the [DataHub Cloud Python SDK](https://pypi.org/project/acryl-datahub-cloud/) for managing Subscriptions: + +- Create: create a subscription to a dataset or assertion. +- Remove: remove a subscription. + +# Prerequisites + +- DataHub Cloud Python SDK installed (`pip install acryl-datahub-cloud`) +- The actor making API calls must have the `Manage User Subscriptions` privilege for the datasets at hand. +- If subscribing to a group, the actor should also be a member of the group. + +:::note +Before creating subscriptions, you need to ensure the target datasets and groups are already present in your DataHub instance. +If you attempt to create subscriptions for entities that do not exist, GMS will continuously report errors to the logs. +::: + +## Create Subscription + +You can create subscriptions to receive notifications when assertions change state (pass, fail, or error) or when other entity changes occur. Subscriptions can be created at the dataset level (affecting any changes on the dataset, as well as all assertions on the dataset) or at the assertion level (affecting only specific assertions). + + + + +```python +# Inlined from /metadata-ingestion/examples/library/subscription_create.py +import logging + +from datahub.sdk import DataHubClient + +log = logging.getLogger(__name__) + +# Initialize the client +client = DataHubClient.from_env() + +# Subscribe to all assertion changes for a dataset +client.subscriptions.subscribe( + urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)", + subscriber_urn="urn:li:corpuser:john.doe", + # entity_change_types defaults to all available change types for datasets +) +log.info("Successfully subscribed to dataset notifications") + +# Subscribe to specific assertion changes +client.subscriptions.subscribe( + urn="urn:li:assertion:your-assertion-id", + subscriber_urn="urn:li:corpuser:john.doe", + entity_change_types=["ASSERTION_PASSED", "ASSERTION_FAILED"], +) +log.info("Successfully subscribed to specific assertion changes") + +# Subscribe a group to assertion changes +client.subscriptions.subscribe( + urn="urn:li:assertion:your-assertion-id", + subscriber_urn="urn:li:corpGroup:data-team", + entity_change_types=["ASSERTION_FAILED", "ASSERTION_ERROR"], +) +log.info("Successfully subscribed group to assertion failures and errors") + +``` + + + + +## Remove Subscription + +You can remove existing subscriptions to stop receiving notifications. The unsubscribe method supports selective removal of specific change types or complete removal of subscriptions. + + + + +```python +# Inlined from /metadata-ingestion/examples/library/subscription_remove.py +import logging + +from datahub.sdk import DataHubClient + +log = logging.getLogger(__name__) + +# Initialize the client +client = DataHubClient( + server="https://your-datahub-cloud-instance.com", token="your-token" +) + +# Unsubscribe from all changes for a dataset +client.subscriptions.unsubscribe( + urn="urn:li:dataset:(urn:li:dataPlatform:snowflake,purchases,PROD)", + subscriber_urn="urn:li:corpuser:john.doe", + # entity_change_types defaults to all existing change types +) +log.info("Successfully unsubscribed from all dataset notifications") + +# Unsubscribe from specific assertion change types +client.subscriptions.unsubscribe( + urn="urn:li:assertion:your-assertion-id", + subscriber_urn="urn:li:corpuser:john.doe", + entity_change_types=[ + "ASSERTION_PASSED" + ], # Keep ASSERTION_FAILED and ASSERTION_ERROR +) +log.info("Successfully unsubscribed from specific assertion change types") + +# Unsubscribe a group from assertion changes +client.subscriptions.unsubscribe( + urn="urn:li:assertion:your-assertion-id", + subscriber_urn="urn:li:corpGroup:data-team", + entity_change_types=["ASSERTION_FAILED", "ASSERTION_ERROR"], +) +log.info("Successfully unsubscribed group from assertion notifications") + +``` + + + + +# Available Change Types + +The following change types are available for subscriptions: + +#### Schema Changes + +- `OPERATION_COLUMN_ADDED` - When a new column is added to a dataset +- `OPERATION_COLUMN_REMOVED` - When a column is removed from a dataset +- `OPERATION_COLUMN_MODIFIED` - When an existing column is modified + +#### Operational Metadata Changes + +- `OPERATION_ROWS_INSERTED` - When rows are inserted into a dataset +- `OPERATION_ROWS_UPDATED` - When rows are updated in a dataset +- `OPERATION_ROWS_REMOVED` - When rows are removed from a dataset + +#### Assertion Events + +- `ASSERTION_PASSED` - When an assertion run passes +- `ASSERTION_FAILED` - When an assertion run fails +- `ASSERTION_ERROR` - When an assertion run encounters an error + +#### Incident Status Changes + +- `INCIDENT_RAISED` - When a new incident is raised +- `INCIDENT_RESOLVED` - When an incident is resolved + +#### Test Status Changes + +- `TEST_PASSED` - When a test passes +- `TEST_FAILED` - When a test fails + +#### Deprecation Status Changes + +- `DEPRECATED` - When an entity is marked as deprecated +- `UNDEPRECATED` - When an entity's deprecation status is removed + +#### Ingestion Status Changes + +- `INGESTION_SUCCEEDED` - When ingestion completes successfully +- `INGESTION_FAILED` - When ingestion fails + +#### Documentation Changes + +- `DOCUMENTATION_CHANGE` - When documentation is modified + +#### Ownership Changes + +- `OWNER_ADDED` - When an owner is added to an entity +- `OWNER_REMOVED` - When an owner is removed from an entity + +#### Glossary Term Changes + +- `GLOSSARY_TERM_ADDED` - When a glossary term is added to an entity +- `GLOSSARY_TERM_REMOVED` - When a glossary term is removed from an entity +- `GLOSSARY_TERM_PROPOSED` - When a glossary term is proposed for an entity + +#### Tag Changes + +- `TAG_ADDED` - When a tag is added to an entity +- `TAG_REMOVED` - When a tag is removed from an entity +- `TAG_PROPOSED` - When a tag is proposed for an entity diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/tags.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/tags.md new file mode 100644 index 00000000..fe96fb03 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/tags.md @@ -0,0 +1,509 @@ +--- +title: Tags +slug: /api/tutorials/tags +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/tags.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Tags + +## Why Would You Use Tags on Datasets? + +Tags are informal, loosely controlled labels that help in search & discovery. They can be added to datasets, dataset schemas, or containers, for an easy way to label or categorize entities – without having to associate them to a broader business glossary or vocabulary. +For more information about tags, refer to [About DataHub Tags](/docs/tags.md). + +### Goal Of This Guide + +This guide will show you how to + +- Create: create a tag. +- Read : read tags attached to a dataset. +- Add: add a tag to a column of a dataset or a dataset itself. +- Remove: remove a tag from a dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed information, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +Before modifying tags, you need to ensure the target dataset is already present in your DataHub instance. +If you attempt to manipulate entities that do not exist, your operation will fail. +In this guide, we will be using data from sample ingestion. +::: + +For more information on how to set up for GraphQL, please refer to [How To Set Up GraphQL](/docs/api/graphql/how-to-set-up-graphql.md). + +## Create Tags + +The following code creates a tag `Deprecated`. + + + + +```json +mutation createTag { + createTag(input: + { + name: "Deprecated", + id: "deprecated", + description: "Having this tag means this column or table is deprecated." + }) +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "createTag": "urn:li:tag:deprecated" + }, + "extensions": {} +} +``` + + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation createTag { createTag(input: { name: \"Deprecated\", id: \"deprecated\",description: \"Having this tag means this column or table is deprecated.\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "createTag": "urn:li:tag:deprecated" }, "extensions": {} } +``` + + + + + +```java +# Inlined from /metadata-integration/java/examples/src/main/java/io/datahubproject/examples/TagCreate.java +package io.datahubproject.examples; + +import com.linkedin.tag.TagProperties; +import datahub.client.MetadataWriteResponse; +import datahub.client.rest.RestEmitter; +import datahub.event.MetadataChangeProposalWrapper; +import java.io.IOException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +public class TagCreate { + + private TagCreate() {} + + public static void main(String[] args) + throws IOException, ExecutionException, InterruptedException { + TagProperties tagProperties = + new TagProperties() + .setName("Deprecated") + .setDescription("Having this tag means this column or table is deprecated."); + + MetadataChangeProposalWrapper mcpw = + MetadataChangeProposalWrapper.builder() + .entityType("tag") + .entityUrn("urn:li:tag:deprecated") + .upsert() + .aspect(tagProperties) + .build(); + + String token = ""; + RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080").token(token)); + Future response = emitter.emit(mcpw, null); + System.out.println(response.get().getResponseContent()); + } +} + +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/tag_create.py +from datahub.metadata.urns import CorpUserUrn +from datahub.sdk import DataHubClient, Tag + +client = DataHubClient.from_env() + +# Create a basic tag +basic_tag = Tag(name="deprecated") + +# Create a more detailed tag with all properties +detailed_tag = Tag( + name="data-quality", + display_name="Data Quality", + description="Tag used to mark datasets with quality issues or requirements", + color="#FF5733", + owners=[ + CorpUserUrn("data-team@company.com"), + ], +) + +# Upsert the tags +client.entities.upsert(basic_tag) +client.entities.upsert(detailed_tag) + +print(f"Created basic tag: {basic_tag.urn}") +print(f"Created detailed tag: {detailed_tag.urn}") + +``` + + + + +### Expected Outcome of Creating Tags + +You can now see the new tag `Deprecated` has been created. + +

+ +

+ +We can also verify this operation by programmatically searching `Deprecated` tag after running this code using the `datahub` cli. + +```shell +datahub get --urn "urn:li:tag:deprecated" --aspect tagProperties + +{ + "tagProperties": { + "description": "Having this tag means this column or table is deprecated.", + "name": "Deprecated" + } +} +``` + +## Read Tags + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)") { + tags { + tags { + tag { + name + urn + properties { + description + colorHex + } + } + } + } + } +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "dataset": { + "tags": { + "tags": [ + { + "tag": { + "name": "Legacy", + "urn": "urn:li:tag:Legacy", + "properties": { + "description": "Indicates the dataset is no longer supported", + "colorHex": null, + "name": "Legacy" + } + } + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "{dataset(urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)\") {tags {tags {tag {name urn properties { description colorHex } } } } } }", "variables":{}}' +``` + +Expected Response: + +```json +{ + "data": { + "dataset": { + "tags": { + "tags": [ + { + "tag": { + "name": "Legacy", + "urn": "urn:li:tag:Legacy", + "properties": { + "description": "Indicates the dataset is no longer supported", + "colorHex": null + } + } + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_query_tags.py +from datahub.sdk import DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="realestate_db.sales", env="PROD") +) + +print(dataset.tags) + +``` + + + + +## Add Tags + +### Add Tags to a dataset + +The following code shows you how can add tags to a dataset. +In the following code, we add a tag `Deprecated` to a dataset named `fct_users_created`. + + + + +```json +mutation addTags { + addTags( + input: { + tagUrns: ["urn:li:tag:deprecated"], + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + } + ) +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "addTags": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation addTags { addTags(input: { tagUrns: [\"urn:li:tag:deprecated\"], resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "addTags": true }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_tag.py +from datahub.sdk import DataHubClient, DatasetUrn, TagUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get(DatasetUrn(platform="hive", name="realestate_db.sales")) +dataset.add_tag(TagUrn("purchase")) + +client.entities.update(dataset) + +``` + + + + +### Add Tags to a Column of a dataset + +In the example below `subResource` is `fieldPath` in the schema. + + + + +```json +mutation addTags { + addTags( + input: { + tagUrns: ["urn:li:tag:deprecated"], + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + subResourceType:DATASET_FIELD, + subResource:"user_name"}) +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation addTags { addTags(input: { tagUrns: [\"urn:li:tag:deprecated\"], resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\", subResourceType: DATASET_FIELD, subResource: \"user_name\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "addTags": true }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_column_tag.py +from datahub.sdk import DataHubClient, DatasetUrn, TagUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="fct_users_created", env="PROD") +) + +dataset["user_name"].add_tag(TagUrn("deprecated")) + +client.entities.update(dataset) + +``` + + + + +### Expected Outcome of Adding Tags + +You can now see `Deprecated` tag has been added to `user_name` column. + +

+ +

+ +We can also verify this operation programmatically by checking the `globalTags` aspect using the `datahub` cli. + +```shell +datahub get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" --aspect globalTags +``` + +## Remove Tags + +The following code remove a tag from a dataset. +After running this code, `Deprecated` tag will be removed from a `user_name` column. + + + + +```json +mutation removeTag { + removeTag( + input: { + tagUrn: "urn:li:tag:deprecated", + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + subResourceType:DATASET_FIELD, + subResource:"user_name"}) +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation removeTag { removeTag(input: { tagUrn: \"urn:li:tag:deprecated\", resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\" }) }", "variables":{}}' +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_remove_tag_execute_graphql.py +# read-modify-write requires access to the DataHubGraph (RestEmitter is not enough) +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +gms_endpoint = "http://localhost:8080" +graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) + +# Query multiple aspects from entity +query = """ +mutation removeTag { + removeTag( + input: { + tagUrn: "urn:li:tag:deprecated", + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + subResourceType:DATASET_FIELD, + subResource:"user_name"}) +} +""" +result = graph.execute_graphql(query=query) + +print(result) + +``` + + + + +### Expected Outcome of Removing Tags + +You can now see `Deprecated` tag has been removed to `user_name` column. + +

+ +

+ +We can also verify this operation programmatically by checking the `gloablTags` aspect using the `datahub` cli. + +```shell +datahub get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" --aspect globalTags + +{ + "globalTags": { + "tags": [] + } +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/terms.md b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/terms.md new file mode 100644 index 00000000..e72f9194 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/api/tutorials/terms.md @@ -0,0 +1,509 @@ +--- +title: Terms +slug: /api/tutorials/terms +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/api/tutorials/terms.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Terms + +## Why Would You Use Terms on Datasets? + +The Business Glossary(Term) feature in DataHub helps you use a shared vocabulary within the orgarnization, by providing a framework for defining a standardized set of data concepts and then associating them with the physical assets that exist within your data ecosystem. + +For more information about terms, refer to [About DataHub Business Glossary](/docs/glossary/business-glossary.md). + +### Goal Of This Guide + +This guide will show you how to + +- Create: create a term. +- Read : read terms attached to a dataset. +- Add: add a term to a column of a dataset or a dataset itself. +- Remove: remove a term from a dataset. + +## Prerequisites + +For this tutorial, you need to deploy DataHub Quickstart and ingest sample data. +For detailed information, please refer to [DataHub Quickstart Guide](/docs/quickstart.md). + +:::note +Before modifying terms, you need to ensure the target dataset is already present in your DataHub instance. +If you attempt to manipulate entities that do not exist, your operation will fail. +In this guide, we will be using data from sample ingestion. +::: + +For more information on how to set up for GraphQL, please refer to [How To Set Up GraphQL](/docs/api/graphql/how-to-set-up-graphql.md). + +## Create Terms + +The following code creates a term `Rate of Return`. + + + + +```json +mutation createGlossaryTerm { + createGlossaryTerm(input: { + name: "Rate of Return", + id: "rateofreturn", + description: "A rate of return (RoR) is the net gain or loss of an investment over a specified time period." + }, + ) +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "createGlossaryTerm": "urn:li:glossaryTerm:rateofreturn" + }, + "extensions": {} +} +``` + + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation createGlossaryTerm { createGlossaryTerm(input: { name: \"Rate of Return\", id:\"rateofreturn\", description: \"A rate of return (RoR) is the net gain or loss of an investment over a specified time period.\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ + "data": { "createGlossaryTerm": "urn:li:glossaryTerm:rateofreturn" }, + "extensions": {} +} +``` + + + + + +```python +# Inlined from /metadata-ingestion/examples/library/glossary_term_create_simple.py +import logging +import os + +from datahub.emitter.mce_builder import make_term_urn +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Imports for metadata model classes +from datahub.metadata.schema_classes import GlossaryTermInfoClass + +log = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + +term_urn = make_term_urn("rateofreturn") +term_properties_aspect = GlossaryTermInfoClass( + definition="A rate of return (RoR) is the net gain or loss of an investment over a specified time period.", + name="Rate of Return", + termSource="", +) + +event: MetadataChangeProposalWrapper = MetadataChangeProposalWrapper( + entityUrn=term_urn, + aspect=term_properties_aspect, +) + +# Create rest emitter +rest_emitter = DatahubRestEmitter( + gms_server=os.getenv("DATAHUB_GMS_URL", "http://localhost:8080"), + token=os.getenv("DATAHUB_GMS_TOKEN"), +) +rest_emitter.emit(event) +log.info(f"Created term {term_urn}") + +``` + + + + +### Expected Outcome of Creating Terms + +You can now see the new term `Rate of Return` has been created. + +

+ +

+ +We can also verify this operation by programmatically searching `Rate of Return` term after running this code using the `datahub` cli. + +```shell +datahub get --urn "urn:li:glossaryTerm:rateofreturn" --aspect glossaryTermInfo + +{ + "glossaryTermInfo": { + "definition": "A rate of return (RoR) is the net gain or loss of an investment over a specified time period.", + "name": "Rate of Return", + "termSource": "INTERNAL" + } +} +``` + +## Read Terms + + + + +```json +query { + dataset(urn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)") { + glossaryTerms { + terms { + term { + urn + glossaryTermInfo { + name + description + } + } + } + } + } +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "dataset": { + "glossaryTerms": { + "terms": [ + { + "term": { + "urn": "urn:li:glossaryTerm:CustomerAccount", + "glossaryTermInfo": { + "name": "CustomerAccount", + "description": "account that represents an identified, named collection of balances and cumulative totals used to summarize customer transaction-related activity over a designated period of time" + } + } + } + ] + } + } + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "{dataset(urn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\") {glossaryTerms {terms {term {urn glossaryTermInfo { name description } } } } } }", "variables":{}}' +``` + +Expected Response: + +````json +{"data":{"dataset":{"glossaryTerms":{"terms":[{"term":{"urn":"urn:li:glossaryTerm:CustomerAccount","glossaryTermInfo":{"name":"CustomerAccount","description":"account that represents an identified, named collection of balances and cumulative totals used to summarize customer transaction-related activity over a designated period of time"}}}]}}},"extensions":{}}``` +```` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_query_terms.py +from datahub.sdk import DataHubClient, DatasetUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="realestate_db.sales", env="PROD") +) + +print(dataset.terms) + +``` + + + + +## Add Terms + +### Add Terms to a dataset + +The following code shows you how can add terms to a dataset. +In the following code, we add a term `Rate of Return` to a dataset named `fct_users_created`. + + + + +```json +mutation addTerms { + addTerms( + input: { + termUrns: ["urn:li:glossaryTerm:rateofreturn"], + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + } + ) +} +``` + +If you see the following response, the operation was successful: + +```python +{ + "data": { + "addTerms": true + }, + "extensions": {} +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation addTerm { addTerms(input: { termUrns: [\"urn:li:glossaryTerm:rateofreturn\"], resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "addTerms": true }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_term.py +from typing import List, Optional, Union + +from datahub.sdk import DataHubClient, DatasetUrn, GlossaryTermUrn + + +def add_terms_to_dataset( + client: DataHubClient, + dataset_urn: DatasetUrn, + term_urns: List[Union[GlossaryTermUrn, str]], +) -> None: + """ + Add glossary terms to a dataset. + + Args: + client: DataHub client to use + dataset_urn: URN of the dataset to update + term_urns: List of term URNs or term names to add + """ + dataset = client.entities.get(dataset_urn) + + for term in term_urns: + if isinstance(term, str): + resolved_term_urn = client.resolve.term(name=term) + dataset.add_term(resolved_term_urn) + else: + dataset.add_term(term) + + client.entities.update(dataset) + + +def main(client: Optional[DataHubClient] = None) -> None: + """ + Main function to add terms to dataset example. + + Args: + client: Optional DataHub client (for testing). If not provided, creates one from env. + """ + client = client or DataHubClient.from_env() + + dataset_urn = DatasetUrn(platform="hive", name="realestate_db.sales", env="PROD") + + # Add terms using both URN and name resolution + add_terms_to_dataset( + client=client, + dataset_urn=dataset_urn, + term_urns=[ + GlossaryTermUrn("Classification.HighlyConfidential"), + "PII", # Will be resolved by name + ], + ) + + +if __name__ == "__main__": + main() + +``` + + + + +### Add Terms to a Column of a Dataset + + + + +```json +mutation addTerms { + addTerms( + input: { + termUrns: ["urn:li:glossaryTerm:rateofreturn"], + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + subResourceType:DATASET_FIELD, + subResource:"user_name"}) +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation addTerms { addTerms(input: { termUrns: [\"urn:li:glossaryTerm:rateofreturn\"], resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)\", subResourceType: DATASET_FIELD, subResource: \"user_name\" }) }", "variables":{}}' +``` + +Expected Response: + +```json +{ "data": { "addTerms": true }, "extensions": {} } +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_add_column_term.py +from datahub.sdk import DataHubClient, DatasetUrn, GlossaryTermUrn + +client = DataHubClient.from_env() + +dataset = client.entities.get( + DatasetUrn(platform="hive", name="realestate_db.sales", env="PROD") +) + +dataset["address.zipcode"].add_term(GlossaryTermUrn("Classification.Location")) + +client.entities.update(dataset) + +``` + + + + +### Expected Outcome of Adding Terms + +You can now see `Rate of Return` term has been added to `user_name` column. + +

+ +

+ +## Remove Terms + +The following code remove a term from a dataset. +After running this code, `Rate of Return` term will be removed from a `user_name` column. + + + + +```json +mutation removeTerm { + removeTerm( + input: { + termUrn: "urn:li:glossaryTerm:rateofreturn", + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + subResourceType:DATASET_FIELD, + subResource:"user_name"}) +} +``` + +Note that you can also remove a term from a dataset if you don't specify `subResourceType` and `subResource`. + +```json +mutation removeTerm { + removeTerm( + input: { + termUrn: "urn:li:glossaryTerm:rateofreturn", + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)", + }) +} +``` + +Also note that you can remove terms from multiple entities or subresource using `batchRemoveTerms`. + +```json +mutation batchRemoveTerms { + batchRemoveTerms( + input: { + termUrns: ["urn:li:glossaryTerm:rateofreturn"], + resources: [ + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)"} , + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)"} ,] + } + ) +} +``` + + + + +```shell +curl --location --request POST 'http://localhost:8080/api/graphql' \ +--header 'Authorization: Bearer ' \ +--header 'Content-Type: application/json' \ +--data-raw '{ "query": "mutation removeTerm { removeTerm(input: { termUrn: \"urn:li:glossaryTerm:rateofreturn\", resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)\" }) }", "variables":{}}' +``` + + + + +```python +# Inlined from /metadata-ingestion/examples/library/dataset_remove_term_execute_graphql.py +# read-modify-write requires access to the DataHubGraph (RestEmitter is not enough) +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +gms_endpoint = "http://localhost:8080" +graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) + +# Query multiple aspects from entity +query = """ +mutation batchRemoveTerms { + batchRemoveTerms( + input: { + termUrns: ["urn:li:glossaryTerm:rateofreturn"], + resources: [ + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)"} , + { resourceUrn:"urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)"} ,] + } + ) +} +""" +result = graph.execute_graphql(query=query) + +print(result) + +``` + + + + +### Expected Outcome of Removing Terms + +You can now see `Rate of Return` term has been removed to `user_name` column. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/architecture/architecture.md b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/architecture.md new file mode 100644 index 00000000..aacafccc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/architecture.md @@ -0,0 +1,48 @@ +--- +title: DataHub Architecture Overview +sidebar_label: DataHub Architecture Overview +slug: /architecture/architecture +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/architecture/architecture.md +--- + +# DataHub Architecture Overview + +DataHub is a [3rd generation](https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained) data catalog that enables Data Discovery, Collaboration, Governance, and end-to-end Observability +that is built for the Modern Data Stack. DataHub employs a model-first philosophy, with a focus on unlocking interoperability between +disparate tools & systems. + +The figures below describe the high-level architecture of DataHub. + +

+ +

+ +

+ +

+ +For a more detailed look at the components that make up the Architecture, check out [Components](../components.md). + +## Architecture Highlights + +There are three main highlights of DataHub's architecture. + +### Schema-first approach to Metadata Modeling + +DataHub's metadata model is described using a [serialization agnostic language](https://linkedin.github.io/rest.li/pdl_schema). Both [REST](https://github.com/datahub-project/datahub/blob/master/metadata-service) as well as [GraphQL API-s](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/graphql) are supported. In addition, DataHub supports an [AVRO-based API](https://github.com/datahub-project/datahub/blob/master/metadata-events) over Kafka to communicate metadata changes and subscribe to them. Our [roadmap](../roadmap.md) includes a milestone to support no-code metadata model edits very soon, which will allow for even more ease of use, while retaining all the benefits of a typed API. Read about metadata modeling at [metadata modeling]. + +### Stream-based Real-time Metadata Management Platform + +DataHub's metadata infrastructure is stream-oriented, which allows for changes in metadata to be communicated and reflected within the platform within seconds. You can also subscribe to changes happening in DataHub's metadata, allowing you to build real-time metadata-driven systems. For example, you can build an access-control system that can observe a previously world-readable dataset adding a new schema field which contains PII, and locks down that dataset for access control reviews. + +### Federated Metadata Serving + +DataHub comes with a single [metadata service (gms)](https://github.com/datahub-project/datahub/blob/master/metadata-service) as part of the open source repository. However, it also supports federated metadata services which can be owned and operated by different teams –– in fact, that is how LinkedIn runs DataHub internally. The federated services communicate with the central search index and graph using Kafka, to support global search and discovery while still enabling decoupled ownership of metadata. This kind of architecture is very amenable for companies who are implementing [data mesh](https://martinfowler.com/articles/data-monolith-to-mesh.html). + +[metadata modeling]: ../modeling/metadata-model.md +[PDL]: https://linkedin.github.io/rest.li/pdl_schema +[metadata architectures blog post]: https://engineering.linkedin.com/blog/2020/datahub-popular-metadata-architectures-explained +[datahub-serving]: metadata-serving.md +[datahub-ingestion]: metadata-ingestion.md +[react-frontend]: ../../datahub-web-react/README.md diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/architecture/docker-containers.md b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/docker-containers.md new file mode 100644 index 00000000..b37b7301 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/docker-containers.md @@ -0,0 +1,26 @@ +--- +title: Docker Container Architecture +sidebar_label: Docker Container Architecture +slug: /architecture/docker-containers +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/architecture/docker-containers.md +--- + +# Docker Container Architecture + +When running DataHub via docker-compose. or helm, the following is a diagram of the containers involved +with running DataHub and their relationships with each other. The helm chart uses helm hooks to determine +the proper ordering of the components whereas docker-compose relies on a series of health checks. + +```text + datahub-frontend-react datahub-actions + \ / + | datahub-upgrade (NoCodeDataMigration, helm only) + | / + datahub-gms (healthy) + | + datahub-upgrade (SystemUpdate completed) + /--------------------/ | \------------------------------------------------\ + / | \ + mysql (healthy) elasticsearch (healthy) kafka-setup (completed) (if apply) neo4j (healthy) +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-ingestion.md b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-ingestion.md new file mode 100644 index 00000000..540bbe21 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-ingestion.md @@ -0,0 +1,42 @@ +--- +title: Ingestion Framework +sidebar_label: Ingestion Framework +slug: /architecture/metadata-ingestion +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/architecture/metadata-ingestion.md +--- + +# Metadata Ingestion Architecture + +DataHub supports an extremely flexible ingestion architecture that can support push, pull, asynchronous and synchronous models. +The figure below describes all the options possible for connecting your favorite system to DataHub. + +

+ +

+ +## Metadata Change Proposal: The Center Piece + +The center piece for ingestion are [Metadata Change Proposal]s which represent requests to make a metadata change to an organization's Metadata Graph. +Metadata Change Proposals can be sent over Kafka, for highly scalable async publishing from source systems. They can also be sent directly to the HTTP endpoint exposed by the DataHub service tier to get synchronous success / failure responses. + +## Pull-based Integration + +DataHub ships with a Python based [metadata-ingestion system](../../metadata-ingestion/README.md) that can connect to different sources to pull metadata from them. This metadata is then pushed via Kafka or HTTP to the DataHub storage tier. Metadata ingestion pipelines can be [integrated with Airflow](../../metadata-ingestion/README.md#lineage-with-airflow) to set up scheduled ingestion or capture lineage. If you don't find a source already supported, it is very easy to [write your own](../../metadata-ingestion/README.md#contributing). + +## Push-based Integration + +As long as you can emit a [Metadata Change Proposal (MCP)] event to Kafka or make a REST call over HTTP, you can integrate any system with DataHub. For convenience, DataHub also provides simple [Python emitters] for you to integrate into your systems to emit metadata changes (MCP-s) at the point of origin. + +## Internal Components + +### Applying Metadata Change Proposals to DataHub Metadata Service (mce-consumer-job) + +DataHub comes with a Spring job, [mce-consumer-job], which consumes the Metadata Change Proposals and writes them into the DataHub Metadata Service (datahub-gms) using the `/ingest` endpoint. + +[Metadata Change Proposal (MCP)]: ../what/mxe.md#metadata-change-proposal-mcp +[Metadata Change Proposal]: ../what/mxe.md#metadata-change-proposal-mcp +[Metadata Change Log (MCL)]: ../what/mxe.md#metadata-change-log-mcl +[equivalent Pegasus format]: https://linkedin.github.io/rest.li/how_data_is_represented_in_memory#the-data-template-layer +[mce-consumer-job]: https://github.com/datahub-project/datahub/blob/master/metadata-jobs/mce-consumer-job +[Python emitters]: ../../metadata-ingestion/README.md#using-as-a-library diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-serving.md b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-serving.md new file mode 100644 index 00000000..e3f0231b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/architecture/metadata-serving.md @@ -0,0 +1,69 @@ +--- +title: Serving Tier +sidebar_label: Serving Tier +slug: /architecture/metadata-serving +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/architecture/metadata-serving.md +--- + +# DataHub Serving Architecture + +The figure below shows the high-level system diagram for DataHub's Serving Tier. + +

+ +

+ +The primary component is called [the Metadata Service](https://github.com/datahub-project/datahub/blob/master/metadata-service) and exposes a REST API and a GraphQL API for performing CRUD operations on metadata. The service also exposes search and graph query API-s to support secondary-index style queries, full-text search queries as well as relationship queries like lineage. In addition, the [datahub-frontend](https://github.com/datahub-project/datahub/blob/master/datahub-frontend) service expose a GraphQL API on top of the metadata graph. + +## DataHub Serving Tier Components + +### Metadata Storage + +The DataHub Metadata Service persists metadata in a document store (an RDBMS like MySQL, Postgres, or Cassandra, etc.). + +### Metadata Change Log Stream (MCL) + +The DataHub Service Tier also emits a commit event [Metadata Change Log] when a metadata change has been successfully committed to persistent storage. This event is sent over Kafka. + +The MCL stream is a public API and can be subscribed to by external systems (for example, the Actions Framework) providing an extremely powerful way to react in real-time to changes happening in metadata. For example, you could build an access control enforcer that reacts to change in metadata (e.g. a previously world-readable dataset now has a pii field) to immediately lock down the dataset in question. +Note that not all MCP-s will result in an MCL, because the DataHub serving tier will ignore any duplicate changes to metadata. + +### Metadata Index Applier (mae-consumer-job) + +[Metadata Change Log]s are consumed by another Spring job, [mae-consumer-job], which applies the changes to the [graph] and [search index] accordingly. +The job is entity-agnostic and will execute corresponding graph & search index builders, which will be invoked by the job when a specific metadata aspect is changed. +The builder should instruct the job how to update the graph and search index based on the metadata change. + +To ensure that metadata changes are processed in the correct chronological order, MCLs are keyed by the entity [URN] — meaning all MAEs for a particular entity will be processed sequentially by a single thread. + +### Metadata Query Serving + +Primary-key based reads (e.g. getting schema metadata for a dataset based on the `dataset-urn`) on metadata are routed to the document store. Secondary index based reads on metadata are routed to the search index (or alternately can use the strongly consistent secondary index support described [here](https://github.com/datahub-project/datahub/blob/master/docs/architecture/)). Full-text and advanced search queries are routed to the search index. Complex graph queries such as lineage are routed to the graph index. + +[RecordTemplate]: https://github.com/linkedin/rest.li/blob/master/data/src/main/java/com/linkedin/data/template/RecordTemplate.java +[GenericRecord]: https://github.com/apache/avro/blob/master/lang/java/avro/src/main/java/org/apache/avro/generic/GenericRecord.java +[Pegasus]: https://linkedin.github.io/rest.li/DATA-Data-Schema-and-Templates +[relationship]: ../what/relationship.md +[entity]: ../what/entity.md +[aspect]: ../what/aspect.md +[GMS]: ../what/gms.md +[Metadata Change Log]: ../what/mxe.md#metadata-change-log-mcl +[rest.li]: https://rest.li +[Metadata Change Proposal (MCP)]: ../what/mxe.md#metadata-change-proposal-mcp +[Metadata Change Log (MCL)]: ../what/mxe.md#metadata-change-log-mcl +[MCP]: ../what/mxe.md#metadata-change-proposal-mcp +[MCL]: ../what/mxe.md#metadata-change-log-mcl +[equivalent Pegasus format]: https://linkedin.github.io/rest.li/how_data_is_represented_in_memory#the-data-template-layer +[graph]: ../what/graph.md +[search index]: ../what/search-index.md +[mce-consumer-job]: https://github.com/datahub-project/datahub/blob/master/metadata-jobs/mce-consumer-job +[mae-consumer-job]: https://github.com/datahub-project/datahub/blob/master/metadata-jobs/mae-consumer-job +[Remote DAO]: ../architecture/metadata-serving.md#remote-dao +[URN]: ../what/urn.md +[Metadata Modelling]: ../modeling/metadata-model.md +[Entity]: ../what/entity.md +[Relationship]: ../what/relationship.md +[Search Document]: ../what/search-document.md +[metadata aspect]: ../what/aspect.md +[Python emitters]: /docs/metadata-ingestion/#using-as-a-library diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/assertions/open-assertions-spec.md b/docs-archive/versioned_docs/version-1.5.0/docs/assertions/open-assertions-spec.md new file mode 100644 index 00000000..b63ee135 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/assertions/open-assertions-spec.md @@ -0,0 +1,488 @@ +--- +title: DataHub Open Data Quality Assertions Specification +sidebar_label: Open Data Quality Assertions Specification +slug: /assertions/open-assertions-spec +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/assertions/open-assertions-spec.md +--- +# DataHub Open Data Quality Assertions Specification + +DataHub is developing an open-source Data Quality Assertions Specification & Compiler that will allow you to declare data quality checks / expectations / assertions using a simple, universal +YAML-based format, and then compile this into artifacts that can be registered or directly executed by 3rd party Data Quality tools like [Snowflake DMFs](https://docs.snowflake.com/en/user-guide/data-quality-intro), +dbt tests, Great Expectations or DataHub Cloud natively. + +Ultimately, our goal is to provide an framework-agnostic, highly-portable format for defining Data Quality checks, making it seamless to swap out the underlying +assertion engine without service disruption for end consumers of the results of these data quality checks in catalogging tools like DataHub. + +## Integrations + +Currently, the DataHub Open Assertions Specification supports the following integrations: + +- [Snowflake DMF Assertions](snowflake/snowflake_dmfs.md) + +And is looking for contributions to build out support for the following integrations: + +- [Looking for Contributions] dbt tests +- [Looking for Contributions] Great Expectation checks + +Below, we'll look at how to define assertions in YAML, and then provide an usage overview for each support integration. + +## The Specification: Declaring Data Quality Assertions in YAML + +The following assertion types are currently supported by the DataHub YAML Assertion spec: + +- [Freshness](/docs/managed-datahub/observe/freshness-assertions.md) +- [Volume](/docs/managed-datahub/observe/volume-assertions.md) +- [Column](/docs/managed-datahub/observe/column-assertions.md) +- [Custom SQL](/docs/managed-datahub/observe/custom-sql-assertions.md) +- [Schema](/docs/managed-datahub/observe/schema-assertions.md) + +Each assertion type aims to validate a different aspect of structured table (e.g. on a data warehouse or data lake), from +structure to size to column integrity to custom metrics. + +In this section, we'll go over examples of defining each. + +### Freshness Assertions + +Freshness Assertions allow you to verify that your data was updated within the expected timeframe. +Below you'll find examples of defining different types of freshness assertions via YAML. + +#### Validating that Table is Updated Every 6 Hours + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: freshness + lookback_interval: "6 hours" + last_modified_field: updated_at + schedule: + type: interval + interval: "6 hours" # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema was updated within the last 6 hours +by issuing a Query to the table which validates determines whether an update was made using the `updated_at` column in the past 6 hours. +To use this check, we must specify the field that contains the last modified timestamp of a given row. + +The `lookback_interval` field is used to specify the "lookback window" for the assertion, whereas the `schedule` field is used to specify how often the assertion should be run. +This allows you to schedule the assertion to run at a different frequency than the lookback window, for example +to detect stale data as soon as it becomes "stale" by inspecting it more frequently. + +#### Supported Source Types + +Currently, the only supported `sourceType` for Freshness Assertions is `LAST_MODIFIED_FIELD`. In the future, +we may support additional source types, such as `HIGH_WATERMARK`, along with data source-specific types such as +`AUDIT_LOG` and `INFORMATION_SCHEMA`. + +### Volume Assertions + +Volume Assertions allow you to verify that the number of records in your dataset meets your expectations. +Below you'll find examples of defining different types of volume assertions via YAML. + +#### Validating that Tale Row Count is in Expected Range + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: volume + metric: "row_count" + condition: + type: between + min: 1000 + max: 10000 + # filters: "event_type = 'purchase'" Optionally add filters. + schedule: + type: on_table_change # Run when new data is added to the table. +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema has between 1000 and 10000 records. +Using the `condition` field, you can specify the type of comparison to be made, and the `min` and `max` fields to specify the range of values to compare against. +Using the `filters` field, you can optionally specify a SQL WHERE clause to filter the records being counted. +Using the `schedule` field you can specify when the assertion should be run, either on a fixed schedule or when new data is added to the table. +The only metric currently supported is `row_count`. + +#### Validating that Table Row Count is Less Than Value + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: volume + metric: "row_count" + condition: + type: less_than_or_equal_to + value: 1000 + # filters: "event_type = 'purchase'" Optionally add filters. + schedule: + type: on_table_change # Run when new data is added to the table. +``` + +#### Validating that Table Row Count is Greater Than Value + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: volume + metric: "row_count" + condition: + type: greater_than_or_equal_to + value: 1000 + # filters: "event_type = 'purchase'" Optionally add filters. + schedule: + type: on_table_change # Run when new data is added to the table. +``` + +#### Supported Conditions + +The full set of supported volume assertion conditions include: + +- `equal_to` +- `not_equal_to` +- `greater_than` +- `greater_than_or_equal_to` +- `less_than` +- `less_than_or_equal_to` +- `between` + +### Column Assertions + +Column Assertions allow you to verify that the values in a column meet your expectations. +Below you'll find examples of defining different types of column assertions via YAML. + +The specification currently supports 2 types of Column Assertions: + +- **Field Value**: Asserts that the values in a column meet a specific condition. +- **Field Metric**: Asserts that a specific metric aggregated across the values in a column meet a specific condition. + +We'll go over examples of each below. + +#### Field Values Assertion: Validating that All Column Values are In Expected Range + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: amount + condition: + type: between + min: 0 + max: 10 + exclude_nulls: True + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + # failure_threshold: + # type: count + # value: 10 + schedule: + type: on_table_change +``` + +This assertion checks that all values for the `amount` column in the `purchase_events` table in the `test_db.public` schema have values between 0 and 10. +Using the `field` field, you can specify the column to be asserted on, and using the `condition` field, you can specify the type of comparison to be made, +and the `min` and `max` fields to specify the range of values to compare against. +Using the `schedule` field you can specify when the assertion should be run, either on a fixed schedule or when new data is added to the table. +Using the `filters` field, you can optionally specify a SQL WHERE clause to filter the records being counted. +Using the `exclude_nulls` field, you can specify whether to exclude NULL values from the assertion, meaning that +NULL will simply be ignored if encountered, as opposed to failing the check. +Using the `failure_threshold`, we can set a threshold for the number of rows that can fail the assertion before the assertion is considered failed. + +#### Field Values Assertion: Validating that All Column Values are In Expected Set + +The validate a VARCHAR / STRING column that should contain one of a set of values: + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: product_id + condition: + type: in + value: + - "product_1" + - "product_2" + - "product_3" + exclude_nulls: False + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + # failure_threshold: + # type: count + # value: 10 + schedule: + type: on_table_change +``` + +#### Field Values Assertion: Validating that All Column Values are Email Addresses + +The validate a string column contains valid email addresses: + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: email_address + condition: + type: matches_regex + value: "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}" + exclude_nulls: False + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + # failure_threshold: + # type: count + # value: 10 + schedule: + type: on_table_change +``` + +#### Field Values Assertion: Supported Conditions + +The full set of supported field value conditions include: + +- `in` +- `not_in` +- `is_null` +- `is_not_null` +- `equal_to` +- `not_equal_to` +- `greater_than` # Numeric Only +- `greater_than_or_equal_to` # Numeric Only +- `less_than` # Numeric Only +- `less_than_or_equal_to` # Numeric Only +- `between` # Numeric Only +- `matches_regex` # String Only +- `not_empty` # String Only +- `length_greater_than` # String Only +- `length_less_than` # String Only +- `length_between` # String Only + +#### Field Metric Assertion: Validating No Missing Values in Column + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: col_date + metric: null_count + condition: + type: equal_to + value: 0 + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + schedule: + type: on_table_change +``` + +This assertion ensures that the `col_date` column in the `purchase_events` table in the `test_db.public` schema has no NULL values. + +#### Field Metric Assertion: Validating No Duplicates in Column + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: id + metric: unique_percentage + condition: + type: equal_to + value: 100 + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + schedule: + type: on_table_change +``` + +This assertion ensures that the `id` column in the `purchase_events` table in the `test_db.public` schema +has no duplicates, by checking that the unique percentage is 100%. + +#### Field Metric Assertion: Validating String Column is Never Empty String + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: field + field: name + metric: empty_percentage + condition: + type: equal_to + value: 0 + # filters: "event_type = 'purchase'" Optionally add filters for Column Assertion. + schedule: + type: on_table_change +``` + +This assertion ensures that the `name` column in the `purchase_events` table in the `test_db.public` schema is never empty, by checking that the empty percentage is 0%. + +#### Field Metric Assertion: Supported Metrics + +The full set of supported field metrics include: + +- `null_count` +- `null_percentage` +- `unique_count` +- `unique_percentage` +- `empty_count` +- `empty_percentage` +- `min` +- `max` +- `mean` +- `median` +- `stddev` +- `negative_count` +- `negative_percentage` +- `zero_count` +- `zero_percentage` + +### Field Metric Assertion: Supported Conditions + +The full set of supported field metric conditions include: + +- `equal_to` +- `not_equal_to` +- `greater_than` +- `greater_than_or_equal_to` +- `less_than` +- `less_than_or_equal_to` +- `between` + +### Custom SQL Assertions + +Custom SQL Assertions allow you to define custom SQL queries to verify your data meets your expectations. +The only condition is that the SQL query must return a single value, which will be compared against the expected value. +Below you'll find examples of defining different types of custom SQL assertions via YAML. + +SQL Assertions are useful for more complex data quality checks that can't be easily expressed using the other assertion types, +and can be used to assert on custom metrics, complex aggregations, cross-table integrity checks (JOINS) or any other SQL-based data quality check. + +#### Validating Foreign Key Integrity + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: sql + statement: | + SELECT COUNT(*) + FROM test_db.public.purchase_events AS pe + LEFT JOIN test_db.public.products AS p + ON pe.product_id = p.id + WHERE p.id IS NULL + condition: + type: equal_to + value: 0 + schedule: + type: interval + interval: "6 hours" # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema has no rows where the `product_id` column does not have a corresponding `id` in the `products` table. + +#### Comparing Row Counts Across Multiple Tables + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: sql + statement: | + SELECT COUNT(*) FROM test_db.public.purchase_events + - (SELECT COUNT(*) FROM test_db.public.purchase_events_raw) AS row_count_difference + condition: + type: equal_to + value: 0 + schedule: + type: interval + interval: "6 hours" # Run every 6 hours +``` + +This assertion checks that the number of rows in the `purchase_events` exactly matches the number of rows in an upstream `purchase_events_raw` table +by subtracting the row count of the raw table from the row count of the processed table. + +#### Supported Conditions + +The full set of supported custom SQL assertion conditions include: + +- `equal_to` +- `not_equal_to` +- `greater_than` +- `greater_than_or_equal_to` +- `less_than` +- `less_than_or_equal_to` +- `between` + +### Schema Assertions (Coming Soon) + +Schema Assertions allow you to define custom SQL queries to verify your data meets your expectations. +Below you'll find examples of defining different types of custom SQL assertions via YAML. + +The specification currently supports 2 types of Schema Assertions: + +- **Exact Match**: Asserts that the schema of a table - column names and their data types - exactly matches an expected schema +- **Contains Match** (Subset): Asserts that the schema of a table - column names and their data types - is a subset of an expected schema + +#### Validating Actual Schema Exactly Equals Expected Schema + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: schema + condition: + type: exact_match + columns: + - name: id + type: INTEGER + - name: product_id + type: STRING + - name: amount + type: DECIMAL + - name: updated_at + type: TIMESTAMP + schedule: + type: interval + interval: "6 hours" # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema has the exact schema as specified, with the exact column names and data types. + +#### Validating Actual Schema is Contains all of Expected Schema + +```yaml +version: 1 +assertions: + - entity: urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.public.purchase_events,PROD) + type: schema + condition: + type: contains + columns: + - name: id + type: integer + - name: product_id + type: string + - name: amount + type: number + schedule: + type: interval + interval: "6 hours" # Run every 6 hours +``` + +This assertion checks that the `purchase_events` table in the `test_db.public` schema contains all of the columns specified in the expected schema, with the exact column names and data types. +The actual schema can also contain additional columns not specified in the expected schema. + +#### Supported Data Types + +The following high-level data types are currently supported by the Schema Assertion spec: + +- string +- number +- boolean +- date +- timestamp +- struct +- array +- map +- union +- bytes +- enum diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/assertions/snowflake/snowflake_dmfs.md b/docs-archive/versioned_docs/version-1.5.0/docs/assertions/snowflake/snowflake_dmfs.md new file mode 100644 index 00000000..842622a3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/assertions/snowflake/snowflake_dmfs.md @@ -0,0 +1,347 @@ +--- +title: 'Snowflake DMF Assertions [BETA]' +slug: /assertions/snowflake/snowflake_dmfs +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/assertions/snowflake/snowflake_dmfs.md +--- +# Snowflake DMF Assertions [BETA] + +The DataHub Open Assertion Compiler allows you to define your Data Quality assertions in a simple YAML format, and then compile them to be executed by Snowflake Data Metric Functions. +Once compiled, you'll be able to register the compiled DMFs in your Snowflake environment, and extract their results them as part of your normal ingestion process for DataHub. +Results of Snowflake DMF assertions will be reported as normal Assertion Results, viewable on a historical timeline in the context +of the table with which they are associated. + +## Prerequisites + +- You must have a Snowflake Enterprise account, where the DMFs feature is enabled. +- You must have the necessary permissions to provision DMFs in your Snowflake environment (see below) +- You must have the necessary permissions to query the DMF results in your Snowflake environment (see below) +- You must have DataHub instance with Snowflake metadata ingested. If you do not have existing snowflake ingestion, refer [Snowflake Quickstart Guide](/docs/quick-ingestion-guides/snowflake/overview) to get started. +- You must have DataHub CLI installed and run [`datahub init`](/docs/cli/#init). + +### Permissions + +_Permissions required for registering DMFs_ + +According to the latest Snowflake docs, here are the permissions the service account performing the +DMF registration and ingestion must have: + +| Privilege | Object | Notes | +| ---------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------- | +| USAGE | Database, schema | Database and schema where snowflake DMFs will be created. This is configured in compile command described below. | +| CREATE FUNCTION | Schema | This privilege enables creating new DMF in schema configured in compile command. | +| EXECUTE DATA METRIC FUNCTION | Account | This privilege enables you to control which roles have access to server-agnostic compute resources to call the system DMF. | +| USAGE | Database, schema | These objects are the database and schema that contain the referenced table in the query. | +| OWNERSHIP | Table | This privilege enables you to associate a DMF with a referenced table. | +| USAGE | DMF | This privilege enables calling the DMF in schema configured in compile command. | + +and the roles that must be granted: + +| Role | Notes | +| -------------------------- | ------------------ | +| SNOWFLAKE.DATA_METRIC_USER | To use System DMFs | + +_Permissions required for running DMFs (scheduled DMFs run with table owner's role)_ + +Because scheduled DMFs run with the role of the table owner, the table owner must have the following privileges: + +| Privilege | Object | Notes | +| ---------------------------- | ---------------- | -------------------------------------------------------------------------------------------------------------------------- | +| USAGE | Database, schema | Database and schema where snowflake DMFs will be created. This is configured in compile command described below. | +| USAGE | DMF | This privilege enables calling the DMF in schema configured in compile power. | +| EXECUTE DATA METRIC FUNCTION | Account | This privilege enables you to control which roles have access to server-agnostic compute resources to call the system DMF. | + +and the roles that must be granted: + +| Role | Notes | +| -------------------------- | ------------------ | +| SNOWFLAKE.DATA_METRIC_USER | To use System DMFs | + +_Permissions required for querying DMF results_ + +In addition, the service account that will be executing DataHub Ingestion, and querying the DMF results, must have been granted the following system application roles: + +| Role | Notes | +| ------------------------------ | --------------------------- | +| DATA_QUALITY_MONITORING_VIEWER | Query the DMF results table | + +To learn more about Snowflake DMFs and the privileges required to provision and query them, see the [Snowflake documentation](https://docs.snowflake.com/en/user-guide/data-quality-intro). + +_Example: Granting Permissions_ + +```sql +-- setup permissions to to create DMFs and associate DMFs with table +grant usage on database "" to role "" +grant usage on schema "." to role "" +grant create function on schema "." to role "" +-- grant ownership + rest of permissions to +grant role "" to role "" + +-- setup permissions for to run DMFs on schedule +grant usage on database "" to role "" +grant usage on schema "." to role "" +grant usage on all functions in "." to role "" +grant usage on future functions in "." to role "" +grant database role SNOWFLAKE.DATA_METRIC_USER to role "" +grant execute data metric function on account to role "" + +-- setup permissions for to query DMF results +grant application role SNOWFLAKE.DATA_QUALITY_MONITORING_VIEWER to role "" +``` + +## Supported Assertion Types + +The following assertion types are currently supported by the DataHub Snowflake DMF Assertion Compiler: + +- [Freshness](/docs/managed-datahub/observe/freshness-assertions.md) +- [Volume](/docs/managed-datahub/observe/volume-assertions.md) +- [Column](/docs/managed-datahub/observe/column-assertions.md) +- [Custom SQL](/docs/managed-datahub/observe/custom-sql-assertions.md) + +Note that Schema Assertions are not currently supported. + +## Creating Snowflake DMF Assertions + +The process for declaring and running assertions backend by Snowflake DMFs consists of a few steps, which will be outlined +in the following sections. + +### Step 1. Define your Data Quality assertions using Assertion YAML files + +See the section **Declaring Assertions in YAML** below for examples of how to define assertions in YAML. + +### Step 2. Register your assertions with DataHub + +Use the DataHub CLI to register your assertions with DataHub, so they become visible in the DataHub UI: + +```bash +datahub assertions upsert -f examples/library/assertions_configuration.yml +``` + +### Step 3. Compile the assertions into Snowflake DMFs using the DataHub CLI + +Next, we'll use the `assertions compile` command to generate the SQL code for the Snowflake DMFs, +which can then be registered in Snowflake. + +```bash +datahub assertions compile -f examples/library/assertions_configuration.yml -p snowflake -x DMF_SCHEMA=. +``` + +Two files will be generated as output of running this command: + +- `dmf_definitions.sql`: This file contains the SQL code for the DMFs that will be registered in Snowflake. +- `dmf_associations.sql`: This file contains the SQL code for associating the DMFs with the target tables in Snowflake. + +By default in a folder called `target`. You can use config option `-o ` in `compile` command to write these compile artifacts in another folder. + +Each of these artifacts will be important for the next steps in the process. + +_dmf_definitions.sql_ + +This file stores the SQL code for the DMFs that will be registered in Snowflake, generated +from your YAML assertion definitions during the compile step. + +```sql +-- Example dmf_definitions.sql + +-- Start of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + + CREATE or REPLACE DATA METRIC FUNCTION + test_db.datahub_dmfs.datahub__5c32eef47bd763fece7d21c7cbf6c659 (ARGT TABLE(col_date DATE)) + RETURNS NUMBER + COMMENT = 'Created via DataHub for assertion urn:li:assertion:5c32eef47bd763fece7d21c7cbf6c659 of type volume' + AS + $$ + select case when metric <= 1000 then 1 else 0 end from (select count(*) as metric from TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ) + $$; + +-- End of Assertion 5c32eef47bd763fece7d21c7cbf6c659 +.... +``` + +_dmf_associations.sql_ + +This file stores the SQL code for associating with the target table, +along with scheduling the generated DMFs to run on at particular times. + +```sql +-- Example dmf_associations.sql + +-- Start of Assertion 5c32eef47bd763fece7d21c7cbf6c659 + + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES SET DATA_METRIC_SCHEDULE = 'TRIGGER_ON_CHANGES'; + ALTER TABLE TEST_DB.PUBLIC.TEST_ASSERTIONS_ALL_TIMES ADD DATA METRIC FUNCTION test_db.datahub_dmfs.datahub__5c32eef47bd763fece7d21c7cbf6c659 ON (col_date); + +-- End of Assertion 5c32eef47bd763fece7d21c7cbf6c659 +.... +``` + +### Step 4. Register the compiled DMFs in your Snowflake environment + +Next, you'll need to run the generated SQL from the files output in Step 3 in Snowflake. + +You can achieve this either by running the SQL files directly in the Snowflake UI, or by using the SnowSQL CLI tool: + +```bash +snowsql -f dmf_definitions.sql +snowsql -f dmf_associations.sql +``` + +::: NOTE +Scheduling Data Metric Function on table incurs Serverless Credit Usage in Snowflake. Refer [Billing and Pricing](https://docs.snowflake.com/en/user-guide/data-quality-intro#billing-and-pricing) for more details. +Please ensure you DROP Data Metric Function created via dmf_associations.sql if the assertion is no longer in use. +::: + +### Step 5. Run ingestion to report the results back into DataHub + +Once you've registered the DMFs, they will be automatically executed, either when the target table is updated or on a fixed +schedule. + +To report the results of the generated Data Quality assertions back into DataHub, you'll need to run the DataHub ingestion process with a special configuration +flag: `include_assertion_results: true`: + +```yaml +# Your DataHub Snowflake Recipe +source: + type: snowflake + config: + # ... + include_assertion_results: True + # ... +``` + +During ingestion we will query for the latest DMF results stored in Snowflake, convert them into DataHub Assertion Results, and report the results back into DataHub during your ingestion process +either via CLI or the UI visible as normal assertions. + +`datahub ingest -c snowflake.yml` + +## Ingesting External (User-Created) DMFs + +In addition to DataHub-created DMFs, you can also ingest results from your own custom Snowflake Data Metric Functions. "External" here means DMFs that were created directly in Snowflake without using DataHub's assertion compiler - they exist outside of DataHub's management. + +### Why Use External DMFs? + +You might want to ingest external DMFs if: + +- **Pre-existing DMFs**: You already have DMFs in Snowflake that were created before adopting DataHub, and you want to see their results in DataHub without recreating them +- **Custom logic**: You need DMF logic that isn't supported by DataHub's assertion compiler (e.g., complex multi-table checks) +- **Team workflows**: Different teams manage DMFs directly in Snowflake, but you want centralized visibility in DataHub +- **Gradual adoption**: You want to start monitoring existing data quality checks in DataHub before fully migrating to DataHub-managed assertions + +### Enabling External DMF Ingestion + +To ingest external DMFs, add the `include_externally_managed_dmfs` flag to your Snowflake recipe: + +```yaml +source: + type: snowflake + config: + # ... connection config ... + + # Enable assertion results ingestion (required) + include_assertion_results: true + + # Enable external DMF ingestion (new) + include_externally_managed_dmfs: true + + # Time window for assertion results + start_time: "-7 days" +``` + +Both flags must be enabled for external DMF ingestion to work. + +### Requirements for External DMFs + +**External DMFs must return `1` for SUCCESS and `0` for FAILURE.** + +DataHub interprets the `VALUE` column from Snowflake's `DATA_QUALITY_MONITORING_RESULTS` table as: + +- `VALUE = 1` → Assertion **PASSED** +- `VALUE = 0` → Assertion **FAILED** + +This is because DataHub cannot interpret arbitrary return values (e.g., "100 null rows" - is that good or bad?). You must build the pass/fail logic into your DMF. + +::: warning What if my DMF returns other values? +If your DMF returns values other than 0 or 1, DataHub will mark the assertion result as **ERROR**: + +- `VALUE = 1` → **PASSED** +- `VALUE = 0` → **FAILED** +- `VALUE != 0 and VALUE != 1` (e.g., 5, 100, -1) → **ERROR** + +The ERROR state indicates that the DMF is not configured correctly for DataHub ingestion. You can identify these cases by: + +1. Checking the ingestion logs for warnings like: `DMF 'my_dmf' returned invalid value 100. Expected 1 (pass) or 0 (fail). Marking as ERROR.` +2. Looking for assertions with ERROR status in the DataHub UI + ::: + +#### Example: Writing External DMFs Correctly + +**WRONG** - Returns raw count (DataHub can't interpret this): + +```sql +CREATE DATA METRIC FUNCTION my_null_check(ARGT TABLE(col VARCHAR)) +RETURNS NUMBER AS +$$ + SELECT COUNT(*) FROM ARGT WHERE col IS NULL +$$; +-- Returns: 0, 5, 100, etc. - DataHub can't determine pass/fail! +``` + +**CORRECT** - Returns 1 (pass) or 0 (fail): + +```sql +CREATE DATA METRIC FUNCTION my_null_check(ARGT TABLE(col VARCHAR)) +RETURNS NUMBER AS +$$ + SELECT CASE WHEN COUNT(*) = 0 THEN 1 ELSE 0 END + FROM ARGT WHERE col IS NULL +$$; +-- Returns: 1 if no nulls (pass), 0 if has nulls (fail) +``` + +**CORRECT** - With threshold: + +```sql +CREATE DATA METRIC FUNCTION my_null_check_threshold(ARGT TABLE(col VARCHAR)) +RETURNS NUMBER AS +$$ + SELECT CASE WHEN COUNT(*) <= 10 THEN 1 ELSE 0 END + FROM ARGT WHERE col IS NULL +$$; +-- Returns: 1 if ≤10 nulls (pass), 0 if >10 nulls (fail) +``` + +### How External DMFs Differ from DataHub-Created DMFs + +| Aspect | DataHub-Created DMFs | External DMFs | +| ------------------ | -------------------------------------------------- | ----------------------------------------- | +| **Naming** | Prefixed with `datahub__` | Any name | +| **Definition** | Created via `datahub assertions compile` | Created manually in Snowflake | +| **Assertion Type** | Based on YAML definition (Freshness, Volume, etc.) | CUSTOM | +| **Source** | NATIVE (defined in DataHub) | EXTERNAL | +| **URN Generation** | Extracted from DMF name (`datahub__`) | Generated from Snowflake's `REFERENCE_ID` | + +### How External DMFs Appear in DataHub UI + +External DMFs appear in DataHub with: + +- **Assertion Type**: CUSTOM +- **Source**: EXTERNAL +- **Platform Instance**: Snowflake platform instance (if configured) +- **Description**: "External Snowflake DMF: {dmf_name}" +- **Custom Properties**: + - `snowflake_dmf_name`: The DMF function name + - `snowflake_reference_id`: Snowflake's unique identifier for the DMF-table binding + - `snowflake_dmf_columns`: Comma-separated list of columns the DMF operates on + +You can view external DMF assertions in the **Quality** tab of the associated dataset in the DataHub UI. They will show pass/fail history alongside any DataHub-created assertions. + +## Caveats + +- Currently, Snowflake supports at most 1000 DMF-table associations at the moment so you can not define more than 1000 assertions for snowflake. +- Currently, Snowflake does not allow JOIN queries or non-deterministic functions in DMF definition so you can not use these in SQL for SQL assertion or in filters section. +- Currently, all DMFs scheduled on a table must follow same exact schedule, so you can not set assertions on same table to run on different schedules. +- Currently, DMFs are only supported for regular tables and not dynamic or external tables. + +## FAQ + +Coming soon! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/README.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/README.md new file mode 100644 index 00000000..343cd801 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/README.md @@ -0,0 +1,77 @@ +--- +title: DataHub Authentication Overview +sidebar_label: DataHub Authentication Overview +slug: /authentication +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/README.md +--- + +# Overview + +Authentication is the process of verifying the identity of a user or service. There are two +places where Authentication occurs inside DataHub: + +1. DataHub frontend service when a user attempts to log in to the DataHub application. +2. DataHub backend service when making API requests to DataHub. + +In this document, we'll tak a closer look at both. + +### Authentication in the Frontend + +Authentication of normal users of DataHub takes place in two phases. + +At login time, authentication is performed by either DataHub itself (via username / password entry) or a third-party Identity Provider. Once the identity +of the user has been established, and credentials validated, a persistent session token is generated for the user and stored +in a browser-side session cookie. + +DataHub provides 3 mechanisms for authentication at login time: + +- **Native Authentication** which uses username and password combinations natively stored and managed by DataHub, with users invited via an invite link. +- [Single Sign-On with OpenID Connect](guides/sso/configure-oidc-react.md) to delegate authentication responsibility to third party systems like Okta or Google/Azure Authentication. This is the recommended approach for production systems. +- [JaaS Authentication](guides/jaas.md) for simple deployments where authenticated users are part of some known list or invited as a [Native DataHub User](guides/add-users.md). + +In subsequent requests, the session token is used to represent the authenticated identity of the user, and is validated by DataHub's backend service (discussed below). +Eventually, the session token is expired (24 hours by default), at which point the end user is required to log in again. + +DataHub also supports Guest users to access the system without requiring an explicit login when enabled. The default configuration disables guest authentication. +When Guest access is enabled, accessing datahub with a configurable URL path logs the user in an existing user that is designated as the guest. The privileges of the guest user +are controlled by adjusting privileges of that designated guest user. + +### Authentication in the Backend (Metadata Service) + +When a user makes a request for Data within DataHub, the request is authenticated by DataHub's Backend (Metadata Service) via a JSON Web Token. This applies to both requests originating from the DataHub application, +and programmatic calls to DataHub APIs. There are two types of tokens that are important: + +1. **Session Tokens**: Generated for users of the DataHub web application. By default, having a duration of 24 hours. + These tokens are encoded and stored inside browser-side session cookies. The duration a session token is valid for is configurable via the `MAX_SESSION_TOKEN_AGE` environment variable + on the datahub-frontend deployment. Additionally, the `AUTH_SESSION_TTL_HOURS` configures the expiration time of the actor cookie on the user's browser which will also prompt a user login. The difference between these is that the actor cookie expiration only affects the browser session and can still be used programmatically, + but when the session expires it can no longer be used programmatically either as it is created as a JWT with an expiration claim. +2. **Personal Access Tokens**: These are tokens generated via the DataHub settings panel useful for interacting + with DataHub APIs. They can be used to automate processes like enriching documentation, ownership, tags, and more on DataHub. Learn + more about Personal Access Tokens [here](personal-access-tokens.md). +3. **OAuth Provider Tokens**: JWT tokens issued by external OAuth2/OIDC providers (like Okta, Auth0, Azure AD) can be used + for service-to-service authentication. This enables seamless integration with existing OAuth infrastructure and is ideal + for automated services and applications. Learn more about OAuth Provider authentication [here](external-oauth-providers.md). + +To learn more about DataHub's backend authentication, check out [Introducing Metadata Service Authentication](introducing-metadata-service-authentication.md). + +Credentials must be provided as Bearer Tokens inside of the **Authorization** header in any request made to DataHub's API layer. + +```shell +Authorization: Bearer +``` + +As with the frontend, the backend also can optionally enable Guest authentication. If Guest authentication is enabled, all API calls made to the backend +without an Authorization header are treated as guest users and the privileges associated with the designated guest user apply to those requests. + +Note that in DataHub local quickstarts, Authentication at the backend layer is disabled for convenience. This leaves the backend +vulnerable to unauthenticated requests and should not be used in production. To enable +backend (token-based) authentication, simply set the `METADATA_SERVICE_AUTH_ENABLED=true` environment variable +for the datahub-gms container or pod. + +It is also recommended to provide your own values for `DATAHUB_TOKEN_SERVICE_SIGNING_KEY` and `DATAHUB_TOKEN_SERVICE_SALT` which is used to sign and verify the access tokens generated by the application. Note that starting v1.5.0 the default values for the above have been removed from the code. + +### References + +For a quick video on the topic of users and groups within DataHub, have a look at [DataHub Basics — Users, Groups, & Authentication 101 +](https://youtu.be/8Osw6p9vDYY) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/changing-default-credentials.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/changing-default-credentials.md new file mode 100644 index 00000000..19d66d96 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/changing-default-credentials.md @@ -0,0 +1,161 @@ +--- +title: Changing the default user credentials +slug: /authentication/changing-default-credentials +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/changing-default-credentials.md +--- +# Changing the default user credentials + +## Default User Credential + +The 'datahub' root user is created for you by default. This user is controlled via a [user.props](https://github.com/datahub-project/datahub/blob/master/datahub-frontend/conf/user.props) file which [JaaS Authentication](./guides/jaas.md) is configured to use: + +By default, the credential file looks like this for each and every self-hosted DataHub deployment: + +``` +// default user.props +datahub:datahub +``` + +Obviously, this is not ideal from a security perspective. It is highly recommended that this file +is changed _prior_ to deploying DataHub to production at your organization. + +:::warning +Please note that deleting the `Data Hub` user in the UI **WILL NOT** disable the default user. +You will still be able to log in using the default 'datahub:datahub' credentials. +To safely delete the default credentials, please follow the guide provided below. + +::: + +## Changing the default user `datahub` + +The method for changing the default user depends on how DataHub is deployed. + +- [Helm chart](#helm-chart) + - [Deployment Guide](/docs/deploy/kubernetes.md) +- [Docker-compose](#docker-compose) + - [Deployment Guide](../../docker/README.md) +- [Quickstart](#quickstart) + - [Deployment Guide](/docs/quickstart.md) + +### Helm chart + +You'll need to create a Kubernetes secret, then mount the file as a volume to the datahub-frontend pod. + +#### 1. Create a new config file + +Create a new version [user.props](https://github.com/datahub-project/datahub/blob/master/datahub-frontend/conf/user.props) which defines the updated password for the datahub user. + +To remove the user 'datahub' from the new file, simply omit the username. Please note that you can also choose to leave the file empty. +For example, to change the password for the DataHub root user to 'newpassword', your file would contain the following: + +``` +// new user.props +datahub:newpassword +``` + +#### 2. Create a kubernetes secret + +Create a secret from your local `user.props` file. + +```shell +kubectl create secret generic datahub-users-secret --from-file=user.props=./ +``` + +#### 3. Mount the config file + +Configure your [values.yaml](https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/values.yaml#LL22C1-L22C1) to add the volume to the datahub-frontend container. + +```yaml +datahub-frontend: + ... + extraVolumes: + - name: datahub-users + secret: + defaultMode: 0444 + secretName: datahub-users-secret + extraVolumeMounts: + - name: datahub-users + mountPath: /datahub-frontend/conf/user.props + subPath: user.props +``` + +#### 4. Restart DataHub + +Restart the DataHub containers or pods to pick up the new configs. +For example, you could run the following command to upgrade the current helm deployment. + +```shell +helm upgrade datahub datahub/datahub --values +``` + +Note that if you update the secret you will need to restart the datahub-frontend pods so the changes are reflected. To update the secret in-place you can run something like this. + +``` +kubectl create secret generic datahub-users-secret --from-file=user.props=./ -o yaml --dry-run=client | kubectl apply -f - +``` + +### Docker-compose + +#### 1. Modify a config file + +Modify [user.props](https://github.com/datahub-project/datahub/blob/master/datahub-frontend/conf/user.props) which defines the updated password for the datahub user. + +To remove the user 'datahub' from the new file, simply omit the username. Please note that you can also choose to leave the file empty. +For example, to change the password for the DataHub root user to 'newpassword', your file would contain the following: + +``` +// new user.props +datahub:newpassword +``` + +#### 2. Mount the updated config file + +Change the [docker-compose.yaml](https://github.com/datahub-project/datahub/blob/master/docker/docker-compose.yml) to mount an updated user.props file to the following location inside the `datahub-frontend-react` container using a volume:`/datahub-frontend/conf/user.props` + +```yaml + datahub-frontend-react: + ... + volumes: + ... + - :/datahub-frontend/conf/user.props +``` + +#### 3. Restart DataHub + +Restart the DataHub containers or pods to pick up the new configs. + +### Quickstart + +#### 1. Modify a config file + +Modify [user.props](https://github.com/datahub-project/datahub/blob/master/datahub-frontend/conf/user.props) which defines the updated password for the datahub user. + +To remove the user 'datahub' from the new file, simply omit the username. Please note that you can also choose to leave the file empty. +For example, to change the password for the DataHub root user to 'newpassword', your file would contain the following: + +``` +// new user.props +datahub:newpassword +``` + +#### 2. Mount the updated config file + +In [docker-compose file used in quickstart](https://github.com/datahub-project/datahub/blob/master/docker/quickstart/docker-compose.quickstart.yml). +Modify the [datahub-frontend-react block](https://github.com/datahub-project/datahub/blob/master/docker/quickstart/docker-compose.quickstart.yml#L116) to contain the extra volume mount. + +```yaml + datahub-frontend-react: + ... + volumes: + ... + - :/datahub-frontend/conf/user.props +``` + +#### 3. Restart DataHub + +Run the following command. + +``` +datahub docker quickstart --quickstart-compose-file .yml +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/concepts.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/concepts.md new file mode 100644 index 00000000..1a84546b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/concepts.md @@ -0,0 +1,229 @@ +--- +title: Concepts & Key Components +slug: /authentication/concepts +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/concepts.md +--- +# Concepts & Key Components + +We introduced a few important concepts to the Metadata Service to make authentication work: + +1. Actor +2. Authenticator +3. AuthenticatorChain +4. Two-Tier Authentication System +5. AuthenticationContext +6. DataHub Access Token +7. DataHub Token Service + +In following sections, we'll take a closer look at each individually. + +

+ +

+ +_High level overview of Metadata Service Authentication_ + +## What is an Actor? + +An **Actor** is a concept within the new Authentication subsystem to represent a unique identity / principal that is initiating actions (e.g. read & write requests) +on the platform. + +An actor can be characterized by 2 attributes: + +1. **Type**: The "type" of the actor making a request. The purpose is to for example distinguish between a "user" & "service" actor. Currently, the "user" actor type is the only one + formally supported. +2. **Id**: A unique identifier for the actor within DataHub. This is commonly known as a "principal" in other systems. In the case of users, this + represents a unique "username". This username is in turn used when converting from the "Actor" concept into a Metadata Entity Urn (e.g. CorpUserUrn). + +For example, the root "datahub" super user would have the following attributes: + +``` +{ + "type": "USER", + "id": "datahub" +} +``` + +Which is mapped to the CorpUser urn: + +``` +urn:li:corpuser:datahub +``` + +for Metadata retrieval. + +## What is an Authenticator? + +An **Authenticator** is a pluggable component inside the Metadata Service that is responsible for authenticating an inbound request provided context about the request (currently, the request headers). +Authentication boils down to successfully resolving an **Actor** to associate with the inbound request. + +There can be many types of Authenticator. For example, there can be Authenticators that + +- Verify the authenticity of access tokens (ie. issued by either DataHub itself or a 3rd-party IdP) +- Authenticate username / password credentials against a remote database (ie. LDAP) + +and more! A key goal of the abstraction is _extensibility_: a custom Authenticator can be developed to authenticate requests +based on an organization's unique needs. + +DataHub ships with 3 Authenticators by default: + +- **DataHubSystemAuthenticator**: Verifies that inbound requests have originated from inside DataHub itself using a shared system identifier + and secret. This authenticator is always present. + +- **DataHubTokenAuthenticator**: Verifies that inbound requests contain a DataHub-issued Access Token (discussed further in the "DataHub Access Token" section below) in their + 'Authorization' header. This authenticator is required if Metadata Service Authentication is enabled. + +- **DataHubGuestAuthenticator**: Verifies if guest authentication is enabled with a guest user configured and allows unauthenticated users to perform operations as the designated + guest user. By default, this Authenticator is disabled. If this is required, it needs to be explicitly enabled and requires a restart of the datahub GMS service. +- + +## What is an AuthenticatorChain? + +An **AuthenticatorChain** is a series of **Authenticators** that are configured to run one-after-another. This allows +for configuring multiple ways to authenticate a given request, for example via LDAP OR via local key file. + +Only if each Authenticator within the chain fails to authenticate a request will it be rejected. + +The Authenticator Chain can be configured in the `application.yaml` file under `authentication.authenticators`: + +``` +authentication: + .... + authenticators: + # Configure the Authenticators in the chain + - type: com.datahub.authentication.Authenticator1 + ... + - type: com.datahub.authentication.Authenticator2 + .... +``` + +## What is the Two-Tier Authentication System? + +DataHub uses a **two-tier authentication system** that decouples authentication extraction from enforcement: + +### Tier 1: Authentication Extraction + +The **AuthenticationExtractionFilter** is the foundation [servlet filter](http://tutorials.jenkov.com/java-servlets/servlet-filters.html) that runs for **every request** to the Metadata Service. Its single responsibility: + +- **Extract Authentication Information**: Constructs and invokes an **AuthenticatorChain** to process credentials +- **Set Universal Context**: Always establishes an **AuthenticationContext** (see below) for every request +- **Never Enforce**: Never blocks requests - if authentication fails, it sets an anonymous context and continues + +### Tier 2: Authentication Enforcement + +The second tier consists of **enforcement mechanisms** that can be implemented in multiple ways: + +#### AuthenticationEnforcementFilter + +The default enforcement filter that: + +- **Selective Processing**: Only processes endpoints requiring authentication (excludes paths like `/health`, `/config`) +- **Context-Based Decisions**: Reads the **AuthenticationContext** set by the extraction tier +- **Request Blocking**: Returns 401 unauthorized when authentication is required but not present + +#### Additional Enforcement Options + +The decoupled design enables flexible enforcement strategies: + +- **Multiple Enforcement Filters**: Different areas can have specialized filters (e.g., admin area filter with additional privilege checks) +- **Controller-Level Enforcement**: Individual controllers can examine the **AuthenticationContext** and enforce their own rules +- **Custom Authorization Logic**: Business logic can make authentication decisions based on the established context + +### Benefits of Two-Tier Architecture + +This separation of concerns provides several advantages: + +1. **Decoupled Responsibilities**: Authentication extraction is separate from enforcement decisions +2. **Performance**: Authentication processing happens once per request, regardless of enforcement complexity +3. **Flexibility**: Multiple enforcement strategies can coexist (filters, controllers, custom logic) +4. **Extensibility**: New enforcement mechanisms can be added without changing authentication extraction +5. **Consistency**: All parts of the system have access to the same authentication context +6. **Progressive Disclosure**: As a side benefit, endpoints can provide different responses based on user authentication status + +## What is AuthenticationContext? + +The **AuthenticationContext** is a thread-local storage mechanism that bridges the extraction and enforcement tiers. It serves as the **universal authentication state** for the entire request lifecycle: + +- **Authentication Object**: Contains the result of the authentication extraction process (Actor + credentials) +- **Anonymous Support**: Set to anonymous actor when authentication extraction yields no valid credentials +- **Request Lifecycle**: Automatically established by the extraction tier and cleaned up after each request +- **Universal Access**: Available to all enforcement mechanisms - filters, controllers, or custom business logic + +This context enables **consistent authentication decisions** across all parts of the system. Whether enforcement happens in a servlet filter, a controller method, or custom business logic, they all work with the same authentication information established during the extraction phase. + +## What is a DataHub Token Service? What are Access Tokens? + +Along with Metadata Service Authentication comes an important new component called the **DataHub Token Service**. The purpose of this +component is twofold: + +1. Generate Access Tokens that grant access to the Metadata Service +2. Verify the validity of Access Tokens presented to the Metadata Service + +**Access Tokens** granted by the Token Service take the form of [Json Web Tokens](https://jwt.io/introduction), a type of stateless token which +has a finite lifespan & is verified using a unique signature. JWTs can also contain a set of claims embedded within them. Tokens issued by the Token +Service contain the following claims: + +- exp: the expiration time of the token +- version: version of the DataHub Access Token for purposes of evolvability (currently 1) +- type: The type of token, currently SESSION (used for UI-based sessions) or PERSONAL (used for personal access tokens) +- actorType: The type of the **Actor** associated with the token. Currently, USER is the only type supported. +- actorId: The id of the **Actor** associated with the token. + +Today, Access Tokens are granted by the Token Service under two scenarios: + +1. **UI Login**: When a user logs into the DataHub UI, for example via [JaaS](guides/jaas.md) or + [OIDC](guides/sso/configure-oidc-react.md), the `datahub-frontend` service issues an + request to the Metadata Service to generate a SESSION token _on behalf of_ of the user logging in. (\*Only the frontend service is authorized to perform this action). +2. **Generating Personal Access Tokens**: When a user requests to generate a Personal Access Token (described below) from the UI. + +> At present, the Token Service supports the symmetric signing method `HS256` to generate and verify tokens. + +Now that we're familiar with the concepts, we will talk concretely about what new capabilities have been built on top +of Metadata Service Authentication. + +## How do I enable Guest Authentication + +The Guest Authentication configuration is present in two configuration files - the `application.conf` for DataHub frontend, and +`application.yaml` for GMS. To enable Guest Authentication, set the environment variable `GUEST_AUTHENTICATION_ENABLED` to `true` +for both the GMS and the frontend service and restart those services. +If enabled, the default user designated as guest is called `guest`. This user must be explicitly created and privileges assigned +to control the guest user privileges. + +A recommended approach to operationalize guest access is, first, create a designated guest user account with login credentials, +but keep guest access disabled. This allows you to configure and test the exact permissions this user should have. Once you've +confirmed the privileges are set correctly, you can then enable guest access, which removes the need for login/credentials +while maintaining the verified permission settings. + +The name of the designated guest user can be changed by defining the env var `GUEST_AUTHENTICATION_USER`. +The entry URL to authenticate as the guest user is `/public` and can be changed via the env var `GUEST_AUTHENTICATION_PATH` + +Here are the relevant portions of the two configs + +For the Frontend + +```yaml +#application.conf +... +auth.guest.enabled = ${?GUEST_AUTHENTICATION_ENABLED} +# The name of the guest user id +auth.guest.user = ${?GUEST_AUTHENTICATION_USER} +# The path to bypass login page and get logged in as guest +auth.guest.path = ${?GUEST_AUTHENTICATION_PATH} +... +``` + +and for GMS + +```yaml +#application.yaml +# Required if enabled is true! A configurable chain of Authenticators +... +authenticators: + ... + - type: com.datahub.authentication.authenticator.DataHubGuestAuthenticator + configs: + guestUser: ${GUEST_AUTHENTICATION_USER:guest} + enabled: ${GUEST_AUTHENTICATION_ENABLED:false} +... +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/external-oauth-providers.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/external-oauth-providers.md new file mode 100644 index 00000000..bbc13c22 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/external-oauth-providers.md @@ -0,0 +1,267 @@ +--- +title: External OAuth Authentication +slug: /authentication/external-oauth-providers +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/external-oauth-providers.md +--- +# External OAuth Authentication + +:::note DataHub Cloud Customers +Self-service configuration of external OAuth providers is not yet available on DataHub Cloud. + +If you'd like to configure external OAuth for DataHub Cloud, please reach out to your customer support representative with the configuration values outlined below! +::: + +DataHub supports authenticating API requests using JWT tokens from external identity providers like Okta, Azure AD, Google Identity, and others. This is perfect for service-to-service authentication where your applications need to call DataHub APIs. + +## Overview + +When you configure OAuth authentication, DataHub will: + +1. Accept JWT tokens from your trusted identity provider +2. Validate the token signature and claims +3. Automatically create service accounts for authenticated users +4. Grant API access based on DataHub's permission system + +## Configuration + +Configure OAuth authentication by setting these environment variables in your DataHub deployment: + +Set these environment variables for the `datahub-gms` service: + +```bash +# Enable OAuth authentication +EXTERNAL_OAUTH_ENABLED=true + +# Required: Trusted JWT issuers (comma-separated) +EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://auth.example.com,https://okta.company.com + +# Required: Allowed JWT audiences (comma-separated) +EXTERNAL_OAUTH_ALLOWED_AUDIENCES=datahub-api,my-service-id + +# Required: JWKS endpoint for signature verification +EXTERNAL_OAUTH_JWKS_URI=https://auth.example.com/.well-known/jwks.json + +# Optional: JWT claim containing user ID (default: "sub") +EXTERNAL_OAUTH_USER_ID_CLAIM=sub + +# Optional: Signing algorithm (default: "RS256") +EXTERNAL_OAUTH_ALGORITHM=RS256 +``` + +### Docker Compose Example + +```yaml +version: "3.8" +services: + datahub-gms: + image: acryldata/datahub-gms:latest + environment: + # External OAuth Configuration + - EXTERNAL_OAUTH_ENABLED=true + - EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://my-okta-domain.okta.com/oauth2/default + - EXTERNAL_OAUTH_ALLOWED_AUDIENCES=0oa1234567890abcdef + - EXTERNAL_OAUTH_JWKS_URI=https://my-okta-domain.okta.com/oauth2/default/v1/keys + - EXTERNAL_OAUTH_USER_ID_CLAIM=sub + - EXTERNAL_OAUTH_ALGORITHM=RS256 + + # Standard DataHub settings + - DATAHUB_GMS_HOST=0.0.0.0 + - DATAHUB_GMS_PORT=8080 + # ... other configurations +``` + +### Kubernetes Example + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datahub-gms +spec: + template: + spec: + containers: + - name: datahub-gms + image: acryldata/datahub-gms:latest + env: + - name: EXTERNAL_OAUTH_ENABLED + value: "true" + - name: EXTERNAL_OAUTH_TRUSTED_ISSUERS + value: "https://login.microsoftonline.com/tenant-id/v2.0" + - name: EXTERNAL_OAUTH_ALLOWED_AUDIENCES + value: "api://datahub-prod" + - name: EXTERNAL_OAUTH_JWKS_URI + value: "https://login.microsoftonline.com/tenant-id/discovery/v2.0/keys" + # ... other environment variables +``` + +### Multiple Providers + +To support multiple OAuth providers, use comma-separated values: + +```bash +# Multiple issuers and audiences +EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://okta.company.com,https://auth0.company.com +EXTERNAL_OAUTH_ALLOWED_AUDIENCES=datahub-prod,datahub-staging,service-account-id + +# Single JWKS URI (if providers share keys) or discovery URI +EXTERNAL_OAUTH_JWKS_URI=https://okta.company.com/.well-known/jwks.json + +# Or use discovery URI to auto-derive JWKS +EXTERNAL_OAUTH_DISCOVERY_URI=https://okta.company.com/.well-known/openid-configuration +``` + +### Discovery URI vs JWKS URI + +You can specify either: + +- **JWKS URI**: Direct endpoint to signing keys (recommended for production) +- **Discovery URI**: OIDC discovery document URL (DataHub will auto-derive JWKS URI) + +```bash +# Option 1: Direct JWKS URI (faster, more reliable) +EXTERNAL_OAUTH_JWKS_URI=https://auth.example.com/.well-known/jwks.json + +# Option 2: Discovery URI (convenient, auto-derives JWKS) +EXTERNAL_OAUTH_DISCOVERY_URI=https://auth.example.com/.well-known/openid-configuration +``` + +## Provider Examples + +### Okta + +```bash +EXTERNAL_OAUTH_ENABLED=true +EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://your-domain.okta.com/oauth2/default +EXTERNAL_OAUTH_ALLOWED_AUDIENCES=0oa1234567890abcdef +EXTERNAL_OAUTH_JWKS_URI=https://your-domain.okta.com/oauth2/default/v1/keys +``` + +### Auth0 + +```bash +EXTERNAL_OAUTH_ENABLED=true +EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://your-domain.auth0.com/ +EXTERNAL_OAUTH_ALLOWED_AUDIENCES=https://your-api-identifier/ +EXTERNAL_OAUTH_JWKS_URI=https://your-domain.auth0.com/.well-known/jwks.json +``` + +### Azure AD / Microsoft Entra + +```bash +EXTERNAL_OAUTH_ENABLED=true +EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://login.microsoftonline.com/your-tenant-id/v2.0 +EXTERNAL_OAUTH_ALLOWED_AUDIENCES=api://your-app-id +EXTERNAL_OAUTH_JWKS_URI=https://login.microsoftonline.com/your-tenant-id/discovery/v2.0/keys +``` + +### Google Cloud Identity + +```bash +EXTERNAL_OAUTH_ENABLED=true +EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://accounts.google.com +EXTERNAL_OAUTH_ALLOWED_AUDIENCES=your-client-id.apps.googleusercontent.com +EXTERNAL_OAUTH_JWKS_URI=https://www.googleapis.com/oauth2/v3/certs +``` + +### Keycloak + +```bash +EXTERNAL_OAUTH_ENABLED=true +EXTERNAL_OAUTH_TRUSTED_ISSUERS=https://keycloak.company.com/realms/datahub +EXTERNAL_OAUTH_ALLOWED_AUDIENCES=datahub-client +EXTERNAL_OAUTH_JWKS_URI=https://keycloak.company.com/realms/datahub/protocol/openid-connect/certs +``` + +## Using OAuth Tokens + +Once configured, include your JWT token in the Authorization header when making API requests: + +```bash +curl -H "Authorization: Bearer YOUR_JWT_TOKEN" \ + -H "Content-Type: application/json" \ + https://your-datahub.com/api/graphql \ + -d '{"query": "{ me { corpUser { urn username }}}"}' +``` + +For Python applications: + +```python +import requests + +headers = { + 'Authorization': f'Bearer {your_jwt_token}', + 'Content-Type': 'application/json' +} + +response = requests.post( + 'https://your-datahub.com/api/graphql', + headers=headers, + json={'query': '{ me { corpUser { urn username }}}'} +) +``` + +## Best Practices + +- Use HTTPS for all JWKS URIs and discovery endpoints +- Use specific audience values (not wildcards) for better security +- Use short-lived tokens (< 1 hour recommended) +- Separate environments with different audiences (prod/staging/dev) +- Enable debug logging during setup: `DATAHUB_GMS_LOG_LEVEL=DEBUG` + +## Troubleshooting + +### Common Issues + +**"OAuth authenticator is not configured"** + +- Make sure `EXTERNAL_OAUTH_ENABLED=true` is set +- Verify all required environment variables are configured + +**"No configured OAuth provider matches token issuer"** + +- Check that your JWT issuer exactly matches `EXTERNAL_OAUTH_TRUSTED_ISSUERS` + +**"Invalid or missing audience claim"** + +- Verify your JWT audience is listed in `EXTERNAL_OAUTH_ALLOWED_AUDIENCES` + +**"Failed to load signing keys"** + +- Test your JWKS URI directly: `curl https://your-provider/.well-known/jwks.json` +- Check network connectivity from DataHub to your OAuth provider + +### Debugging + +Enable debug logging to see detailed OAuth messages: + +```bash +# Set environment variable +DATAHUB_GMS_LOG_LEVEL=DEBUG + +# Check logs +docker logs datahub-gms | grep -i oauth +``` + +### Testing Your Setup + +Decode your JWT token to verify the claims: + +```bash +# Replace with your actual token +echo "YOUR_JWT_TOKEN" | cut -d. -f2 | base64 -d | jq +``` + +Make sure the `iss` (issuer) and `aud` (audience) claims match your configuration. + +## Advanced Options + +You can customize which JWT claim contains the user ID: + +```bash +# Use email claim instead of default 'sub' +EXTERNAL_OAUTH_USER_ID_CLAIM=email +``` + +OAuth users are automatically created as service accounts with usernames like `__oauth_{issuer_domain}_{subject}`. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/add-users.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/add-users.md new file mode 100644 index 00000000..54b6147e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/add-users.md @@ -0,0 +1,238 @@ +--- +title: Onboarding Users to DataHub +slug: /authentication/guides/add-users +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/guides/add-users.md +--- +# Onboarding Users to DataHub + +New user accounts can be provisioned on DataHub in 3 ways: + +1. Shared Invite Links +2. Single Sign-On using [OpenID Connect](https://www.google.com/search?q=openid+connect&oq=openid+connect&aqs=chrome.0.0i131i433i512j0i512l4j69i60l2j69i61.1468j0j7&sourceid=chrome&ie=UTF-8) +3. Static Credential Configuration File (Self-Hosted Only) + +The first option is the easiest to get started with. The second is recommended for deploying DataHub in production. The third should +be reserved for special circumstances where access must be closely monitored and controlled, and is only relevant for Self-Hosted instances. + +# Shared Invite Links + +### Generating an Invite Link + +If you have the `Manage User Credentials` [Platform Privilege](../../authorization/access-policies-guide.md), you can invite new users to DataHub by sharing an invite link. + +To do so, navigate to the **Users & Groups** section inside of Settings page. Here you can generate a shareable invite link by clicking the `Invite Users` button. If you +do not have the correct privileges to invite users, this button will be disabled. + +

+ +

+ +To invite new users, simply share the link with others inside your organization. + +

+ +

+ +When a new user visits the link, they will be directed to a sign up screen where they can create their DataHub account. + +### Resetting User Passwords + +To reset a user's password, navigate to the Users & Groups tab, find the user who needs their password reset, +and click **Reset user password** inside the menu dropdown on the right hand side. Note that a user must have the +`Manage User Credentials` [Platform Privilege](../../authorization/access-policies-guide.md) in order to reset passwords. + +

+ +

+ +To reset the password, simply share the password reset link with the user who needs to change their password. Password reset links expire after 24 hours. + +

+ +

+ +# Configuring Single Sign-On with OpenID Connect + +Setting up Single Sign-On via OpenID Connect enables your organization's users to login to DataHub via a central Identity Provider such as + +- Azure AD +- Okta +- Keycloak +- Ping! +- Google Identity + +and many more. + +This option is strongly recommended for production deployments of DataHub. + +### DataHub Cloud + +Single Sign-On can be configured and enabled by navigating to **Settings** > **SSO** > **OIDC**. Note +that a user must have the **Manage Platform Settings** [Platform Privilege](../../authorization/access-policies-guide.md) +in order to configure SSO settings. + +To complete the integration, you'll need the following: + +1. **Client ID** - A unique identifier for your application with the identity provider +2. **Client Secret** - A shared secret to use for exchange between you and your identity provider +3. **Discovery URL** - A URL where the OpenID settings for your identity provider can be discovered. + +These values can be obtained from your Identity Provider by following Step 1 on the [OpenID Connect Authentication](sso/configure-oidc-react.md)) Guide. + +### Self-Hosted DataHub + +For information about configuring Self-Hosted DataHub to use OpenID Connect (OIDC) to +perform authentication, check out [OIDC Authentication](sso/configure-oidc-react.md). + +> **A note about user URNs**: User URNs are unique identifiers for users on DataHub. The username received from an Identity Provider +> when a user logs into DataHub via OIDC is used to construct a unique identifier for the user on DataHub. The urn is computed as: +> `urn:li:corpuser:` +> +> By default, the email address will be the username extracted from the Identity Provider. For information about customizing +> the claim should be treated as the username in DataHub, check out the [OIDC Authentication](sso/configure-oidc-react.md) documentation. + +# Static Credential Configuration File (Self-Hosted Only) + +User credentials can be managed via a [JaaS Authentication](./jaas.md) configuration file containing +static username and password combinations. By default, the credentials for the root 'datahub' users are configured +using this mechanism. It is highly recommended that admins change or remove the default credentials for this user + +## Adding new users using a user.props file + +:::NOTE +Adding users via the `user.props` will require disabling existence checks on GMS using the `METADATA_SERVICE_AUTH_ENFORCE_EXISTENCE_ENABLED=false` environment variable or using the API to enable the user prior to login. +The directions below demonstrate using the API to enable the user. +::: + +To define a set of username / password combinations that should be allowed to log in to DataHub (in addition to the root 'datahub' user), +create a new file called `user.props` at the file path `${HOME}/.datahub/plugins/frontend/auth/user.props` within the `datahub-frontend-react` container +or pod. + +This file should contain username:password specifications, with one on each line. For example, to create 2 new users, +with usernames "janesmith" and "johndoe", we would define the following file: + +``` +// custom user.props +janesmith:janespassword +johndoe:johnspassword +``` + +In order to enable the user access with the credential defined in `user.props`, set the `status` aspect on the user with an Admin user. This can be done using an API call or via the [OpenAPI UI interface](/docs/api/openapi/openapi-usage-guide.md). + +OpenAPI example enabling login for the `janesmith` user from the example above. Make sure to update the example with your access token. + +```shell +curl -X 'POST' \ + 'http://localhost:9002/openapi/v3/entity/corpuser/urn%3Ali%3Acorpuser%3Ajanesmith/status?async=false&systemMetadata=false&createIfEntityNotExists=false&createIfNotExists=true' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer ' \ + -d '{ + "value": { + "removed": false + } +}' +``` + +Once you've saved the file, simply start the DataHub containers & navigate to `http://localhost:9002/login` +to verify that your new credentials work. + +To change or remove existing login credentials, edit and save the `user.props` file. Then restart DataHub containers. + +If you want to customize the location of the `user.props` file, or if you're deploying DataHub via Helm, proceed to Step 2. + +### (Advanced) Mount custom user.props file to container + +This step is only required when mounting custom credentials into a Kubernetes pod (e.g. Helm) **or** if you want to change +the default filesystem location from which DataHub mounts a custom `user.props` file (`${HOME}/.datahub/plugins/frontend/auth/user.props)`. + +If you are deploying with `datahub docker quickstart`, or running using Docker Compose, you can most likely skip this step. + +#### Docker Compose + +You'll need to modify the `docker-compose.yml` file to mount a container volume mapping your custom user.props to the standard location inside the container +(`/etc/datahub/plugins/frontend/auth/user.props`). + +For example, to mount a user.props file that is stored on my local filesystem at `/tmp/datahub/user.props`, we'd modify the YAML for the +`datahub-web-react` config to look like the following: + +```aidl + datahub-frontend-react: + build: + context: ../ + dockerfile: docker/datahub-frontend/Dockerfile + image: acryldata/datahub-frontend-react:${DATAHUB_VERSION:-head} + ..... + # The new stuff + volumes: + - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - /tmp/datahub:/etc/datahub/plugins/frontend/auth +``` + +Once you've made this change, restarting DataHub enable authentication for the configured users. + +#### Helm + +You'll need to create a Kubernetes secret, then mount the file as a volume to the `datahub-frontend` pod. + +First, create a secret from your local `user.props` file + +```shell +kubectl create secret generic datahub-users-secret --from-file=user.props=./ +``` + +Then, configure your `values.yaml` to add the volume to the `datahub-frontend` container. + +```YAML +datahub-frontend: + ... + extraVolumes: + - name: datahub-users + secret: + defaultMode: 0444 + secretName: datahub-users-secret + extraVolumeMounts: + - name: datahub-users + mountPath: /etc/datahub/plugins/frontend/auth/user.props + subPath: user.props +``` + +Note that if you update the secret you will need to restart the `datahub-frontend` pods so the changes are reflected. To update the secret in-place you can run something like this. + +```shell +kubectl create secret generic datahub-users-secret --from-file=user.props=./ -o yaml --dry-run=client | kubectl apply -f - +``` + +> A note on user URNs: User URNs are unique identifiers for users of DataHub. The usernames defined in the `user.props` file will be used to generate the DataHub user "urn", which uniquely identifies +> the user on DataHub. The urn is computed as `urn:li:corpuser:{username}`, where "username is defined inside your user.props file." + +## Changing the default 'datahub' user credentials (Recommended) + +Please refer to [Changing the default user credentials](../changing-default-credentials.md). + +## Caveats + +### Adding User Details + +If you add a new username / password to the `user.props` file, no other information about the user will exist +about the user in DataHub (full name, email, bio, etc). This means that you will not be able to search to find the user. + +In order for the user to become searchable, simply navigate to the new user's profile page (top-right corner) and click +**Edit Profile**. Add some details like a display name, an email, and more. Then click **Save**. Now you should be able +to find the user via search. + +> You can also use our Python Emitter SDK to produce custom information about the new user via the CorpUser metadata entity. + +For a more comprehensive overview of how users & groups are managed within DataHub, check out [this video](https://www.youtube.com/watch?v=8Osw6p9vDYY). + +## FAQ + +1. Can I enable OIDC and username / password (JaaS) authentication at the same time? + +YES! If you have not explicitly disabled JaaS via an environment variable on the datahub-frontend container (AUTH_JAAS_ENABLED), +then you can always access the standard login flow at `http://your-datahub-url.com/login`. + +## Feedback / Questions / Concerns + +We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on Slack! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/jaas.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/jaas.md new file mode 100644 index 00000000..51d49e99 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/jaas.md @@ -0,0 +1,76 @@ +--- +title: JaaS Authentication +slug: /authentication/guides/jaas +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/guides/jaas.md +--- +# JaaS Authentication + +## Overview + +The DataHub frontend server comes with support for plugging in [JaaS](https://docs.oracle.com/javase/7/docs/technotes/guides/security/jaas/JAASRefGuide.html) modules. +This allows you to use a custom authentication protocol to log your users into DataHub. + +By default, we in include sample configuration of a file-based username / password authentication module ([PropertyFileLoginModule](http://archive.eclipse.org/jetty/8.0.0.M3/apidocs/org/eclipse/jetty/plus/jaas/spi/PropertyFileLoginModule.html)) +that is configured with a single username / password combination: datahub - datahub. + +To change or extend the default behavior, you have multiple options, each dependent on which deployment environment you're operating in. + +### Modify user.props file directly (Local Testing) + +The first option for customizing file-based users is to modify the file `datahub-frontend/app/conf/user.props` directly. +Once you've added your desired users, you can simply run `./dev.sh` or `./datahub-frontend/run-local-frontend` to validate your +new users can log in. + +### Mount a custom user.props file (Docker Compose) + +By default, the `datahub-frontend` container will look for a file called `user.props` mounted at the container path +`/datahub-frontend/conf/user.props`. If you wish to launch this container with a custom set of users, you'll need to override the default +file mounting when running using `docker-compose`. + +To do so, change the `datahub-frontend-react` service in the docker-compose.yml file containing it to include the custom file: + +``` +datahub-frontend-react: + build: + context: ../ + dockerfile: docker/datahub-frontend/Dockerfile + image: acryldata/datahub-frontend-react:${DATAHUB_VERSION:-head} + env_file: datahub-frontend/env/docker.env + hostname: datahub-frontend-react + container_name: datahub-frontend-react + ports: + - "9002:9002" + depends_on: + - datahub-gms + volumes: + - ./my-custom-dir/user.props:/datahub-frontend/conf/user.props +``` + +And then run `docker-compose up` against your compose file. + +## Custom JaaS Configuration + +In order to change the default JaaS module configuration, you will have to launch the `datahub-frontend-react` container with the custom `jaas.conf` file mounted as a volume +at the location `/datahub-frontend/conf/jaas.conf`. + +To do so, change the `datahub-frontend-react` service in the docker-compose.yml file containing it to include the custom file: + +``` +datahub-frontend-react: + build: + context: ../ + dockerfile: docker/datahub-frontend/Dockerfile + image: acryldata/datahub-frontend-react:${DATAHUB_VERSION:-head} + env_file: datahub-frontend/env/docker.env + hostname: datahub-frontend-react + container_name: datahub-frontend-react + ports: + - "9002:9002" + depends_on: + - datahub-gms + volumes: + - ./my-custom-dir/jaas.conf:/datahub-frontend/conf/jaas.conf +``` + +And then run `docker-compose up` against your compose file. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-behind-proxy.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-behind-proxy.md new file mode 100644 index 00000000..470559c9 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-behind-proxy.md @@ -0,0 +1,72 @@ +--- +title: OIDC Proxy Configuration +slug: /authentication/guides/sso/configure-oidc-behind-proxy +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/guides/sso/configure-oidc-behind-proxy.md +--- +# OIDC Proxy Configuration + +_Authored on 22/08/2023_ + +The `datahub-frontend-react` server can be configured to use an http proxy when retrieving the openid-configuration. +This can be needed if your infrastructure is locked down and disallows connectivity by default, using proxies for fine-grained egress control. + +## Configure http proxy and non proxy hosts + +To do this, you will need to pass a set of environment variables to the datahub-frontend-react container (e.g. in the `docker-compose.yml` file or your kubernetes manifest). + +``` +HTTP_PROXY_HOST=host of your http proxy +HTTP_PROXY_PORT=port of your http proxy +HTTPS_PROXY_HOST=host of your http(s) proxy used for https connections (often the same as the http proxy) +HTTPS_PROXY_PORT=port of your http(s) proxy used for https connections (often the same as the http proxy) +HTTP_NON_PROXY_HOSTS=localhost|datahub-gms (or any other hosts that you would like to bypass the proxy for, delimited by pipe) +``` + +## Optional: provide custom truststore + +If your upstream proxy performs SSL termination to inspect traffic, this will result in different (self-signed) certificates for HTTPS connections. +The default truststore used in the `datahub-frontend-react` docker image will not trust these kinds of connections. +To address this, you can copy or mount your own truststore (provided by the proxy or network administrators) into the docker container. + +Depending on your setup, you have a few options to achieve this: + +### Make truststore available in the frontend + +#### Option a) Build frontend docker image with your own truststore included + +To build a custom image for your frontend, with the certificates built-in, you can use the official frontend image as a base, then copy in your required files. + +Example Dockerfile: + +```dockerfile +FROM acryldata/datahub-frontend-react: +COPY /truststore-directory /certificates +``` + +Building this Dockerfile will result in your own custom docker image on your local machine. +You will then be able to tag it, publish it to your own registry, etc. + +#### Option b) Mount truststore from your host machine using a docker volume + +Adapt your docker-compose.yml to include a new volume mount in the `datahub-frontend-react` container + +```docker + datahub-frontend-react: + # ... + volumes: + # ... + - /truststore-directory:/certificates +``` + +### Reference new truststore + +Add the following environment values to the `datahub-frontend-react` container: + +``` +SSL_TRUSTSTORE_FILE=path/to/truststore.jks (e.g. /certificates) +SSL_TRUSTSTORE_TYPE=jks +SSL_TRUSTSTORE_PASSWORD=MyTruststorePassword +``` + +Once these steps are done, your frontend container will use the new truststore when validating SSL/HTTPS connections. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-react.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-react.md new file mode 100644 index 00000000..44837151 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/configure-oidc-react.md @@ -0,0 +1,214 @@ +--- +title: Configuring OIDC Authentication +slug: /authentication/guides/sso/configure-oidc-react +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/guides/sso/configure-oidc-react.md +--- +# Configuring OIDC Authentication + +The DataHub React application supports OIDC authentication built on top of the [Pac4j Play](https://github.com/pac4j/play-pac4j) library. +This enables operators of DataHub to integrate with 3rd party identity providers like Okta, Google, Keycloak, & more to authenticate their users. + +## 1. Get your credentials + +Before you do anything, you'll want to set up DataHub with your SSO provider, and get prerequisite credentials: + +1. _Client ID_ - A unique identifier for your application with the identity provider +2. _Client Secret_ - A shared secret to use for exchange between you and your identity provider. +3. _Discovery URL_ - A URL where the OIDC API of your identity provider can be discovered. + +[👉 See the steps here](./initialize-oidc.md) + +## 2. Configure DataHub Frontend Server + +### Docker + +The next step to enabling OIDC involves configuring `datahub-frontend` to enable OIDC authentication with your Identity Provider. + +To do so, you must update the `datahub-frontend` [docker.env](https://github.com/datahub-project/datahub/blob/master/docker/datahub-frontend/env/docker.env) file with the +values received from your identity provider: + +``` +# Required Configuration Values: +AUTH_OIDC_ENABLED=true +AUTH_OIDC_CLIENT_ID=your-client-id +AUTH_OIDC_CLIENT_SECRET=your-client-secret +AUTH_OIDC_DISCOVERY_URI=your-provider-discovery-url +AUTH_OIDC_BASE_URL=your-datahub-url +``` + +| Configuration | Description | Default | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------- | +| AUTH_OIDC_ENABLED | Enable delegating authentication to OIDC identity provider | | +| AUTH_OIDC_CLIENT_ID | Unique client id received from identity provider | | +| AUTH_OIDC_CLIENT_SECRET | Unique client secret received from identity provider | | +| AUTH_OIDC_DISCOVERY_URI | Location of the identity provider OIDC discovery API. Suffixed with `.well-known/openid-configuration` | | +| AUTH_OIDC_BASE_URL | The base URL of your DataHub deployment, e.g. https://yourorgdatahub.com (prod) or http://localhost:9002 (testing) | | +| AUTH_SESSION_TTL_HOURS | The length of time in hours before a user will be prompted to login again. Controls the actor cookie expiration time in the browser. Numeric value converted to hours. | 24 | +| MAX_SESSION_TOKEN_AGE | Determines the expiration time of a session token. Session tokens are stateless so this determines at what time a session token may no longer be used and a valid session token can be used until this time has passed. Accepts a valid relative Java date style String. | 24h | + +Providing these configs will cause DataHub to delegate authentication to your identity +provider, requesting the "oidc email profile" scopes and parsing the "preferred_username" claim from +the authenticated profile as the DataHub CorpUser identity. + +:::note + +By default, the login callback endpoint exposed by DataHub will be located at `${AUTH_OIDC_BASE_URL}/callback/oidc`. This must **exactly** match the login redirect URL you've registered with your identity provider in step 1. + +::: + +### Kubernetes + +In Kubernetes, you can add the above env variables in the `values.yaml` as follows. + +```yaml +datahub-frontend: + ... + extraEnvs: + - name: AUTH_OIDC_ENABLED + value: "true" + - name: AUTH_OIDC_CLIENT_ID + value: your-client-id + - name: AUTH_OIDC_CLIENT_SECRET + value: your-client-secret + - name: AUTH_OIDC_DISCOVERY_URI + value: your-provider-discovery-url + - name: AUTH_OIDC_BASE_URL + value: your-datahub-url +``` + +You can also package OIDC client secrets into a k8s secret by running + +``` +kubectl create secret generic datahub-oidc-secret --from-literal=secret=<> +``` + +Then set the secret env as follows. + +```yaml +- name: AUTH_OIDC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: datahub-oidc-secret + key: secret +``` + +### Advanced OIDC Configurations + +You can optionally customize the flow further using advanced configurations. These allow +you to specify the OIDC scopes requested, how the DataHub username is parsed from the claims returned by the identity provider, and how users and groups are extracted and provisioned from the OIDC claim set. + +``` +# Optional Configuration Values: +AUTH_OIDC_USER_NAME_CLAIM=your-custom-claim +AUTH_OIDC_USER_NAME_CLAIM_REGEX=your-custom-regex +AUTH_OIDC_SCOPE=your-custom-scope +AUTH_OIDC_CLIENT_AUTHENTICATION_METHOD=authentication-method +``` + +| Configuration | Description | Default | +| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | +| AUTH_OIDC_USER_NAME_CLAIM | The attribute that will contain the username used on the DataHub platform. By default, this is "email" providedas part of the standard `email` scope. | | +| AUTH_OIDC_USER_NAME_CLAIM_REGEX | A regex string used for extracting the username from the userNameClaim attribute. For example, if the userNameClaim field will contain an email address, and we want to omit the domain name suffix of the email, we can specify a customregex to do so. (e.g. `([^@]+)`) | | +| AUTH_OIDC_SCOPE | A string representing the scopes to be requested from the identity provider, granted by the end user. For more info, see [OpenID Connect Scopes](https://auth0.com/docs/scopes/openid-connect-scopes). | | +| AUTH_OIDC_CLIENT_AUTHENTICATION_METHOD | a string representing the token authentication method to use with the identity provider. Default value is `client_secret_basic`, which uses HTTP Basic authentication. Another option is `client_secret_post`, which includes the client_id and secret_id as form parameters in the HTTP POST request. For more info, see [OAuth 2.0 Client Authentication](https://darutk.medium.com/oauth-2-0-client-authentication-4b5f929305d4) | client_secret_basic | +| AUTH_OIDC_PREFERRED_JWS_ALGORITHM | Can be used to select a preferred signing algorithm for id tokens. Examples include: `RS256` or `HS256`. If your IdP includes `none` before `RS256`/`HS256` in the list of signing algorithms, then this value **MUST** be set. | | + +### User & Group Provisioning (JIT Provisioning) + +By default, DataHub will optimistically attempt to provision users and groups that do not already exist at the time of login. +For users, we extract information like first name, last name, display name, & email to construct a basic user profile. If a groups claim is present, +we simply extract their names. + +The default provisioning behavior can be customized using the following configs. + +``` +# User and groups provisioning +AUTH_OIDC_JIT_PROVISIONING_ENABLED=true +AUTH_OIDC_PRE_PROVISIONING_REQUIRED=false +AUTH_OIDC_EXTRACT_GROUPS_ENABLED=false +AUTH_OIDC_GROUPS_CLAIM= +``` + +| Configuration | Description | Default | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| AUTH_OIDC_JIT_PROVISIONING_ENABLED | Whether DataHub users & groups should be provisioned on login if they do not exist. | true | +| AUTH_OIDC_PRE_PROVISIONING_REQUIRED | Whether the user should already exist in DataHub when they login, failing login if they are not. This is appropriate for situations in which users and groups are batch ingested and tightly controlled inside your environment. | false | +| AUTH_OIDC_EXTRACT_GROUPS_ENABLED | Only applies if `AUTH_OIDC_JIT_PROVISIONING_ENABLED` is set to true. This determines whether we should attempt to extract a list of group names from a particular claim in the OIDC attributes. Note that if this is enabled, each login will re-sync group membership with the groups in your Identity Provider, clearing the group membership that has been assigned through the DataHub UI. Enable with care! | false | +| AUTH_OIDC_GROUPS_CLAIM | Only applies if `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` is set to true. This determines which OIDC claims will contain a list of string group names. Accepts multiple claim names with comma-separated values. I.e: `groups, teams, departments`. | groups | + +## 3. Restart datahub-frontend-react + +Once configured, restarting the `datahub-frontend-react` container will enable an indirect authentication flow in which DataHub delegates authentication to the specified identity provider. + +``` +docker-compose -p datahub -f docker-compose.yml -f docker-compose.override.yml up datahub-frontend-react +``` + +Navigate to your DataHub domain to see SSO in action. + +:::caution +By default, enabling OIDC will _not_ disable the dummy JAAS authentication path, which can be reached at the `/login` +route of the React app. To disable this authentication path, additionally specify the following config: +`AUTH_JAAS_ENABLED=false` +::: + +## Summary + +Once a user is authenticated by the identity provider, DataHub will extract a username from the provided claims +and grant DataHub access to the user by setting a pair of session cookies. + +A brief summary of the steps that occur when the user navigates to the React app are as follows: + +1. A `GET` to the `/authenticate` endpoint in `datahub-frontend` server is initiated +2. The `/authenticate` attempts to authenticate the request via session cookies +3. If auth fails, the server issues a redirect to the Identity Provider's login experience +4. The user logs in with the Identity Provider +5. The Identity Provider authenticates the user and redirects back to DataHub's registered login redirect URL, providing an authorization code which + can be used to retrieve information on behalf of the authenticated user +6. DataHub fetches the authenticated user's profile and extracts a username to identify the user on DataHub (eg. `urn:li:corpuser:username)` +7. DataHub sets session cookies for the newly authenticated user +8. DataHub redirects the user to the homepage ("/") + +## Troubleshooting + +
+No users can log in. Instead, I get redirected to the login page with an error. What do I do? + +This can occur for a variety of reasons, but most often it is due to misconfiguration of Single-Sign On, either on the DataHub +side or on the Identity Provider side. + +- Verify that all values are consistent across them (e.g. the host URL where DataHub is deployed), and that no values are misspelled (client id, client secret). +- Verify that the scopes requested are supported by your Identity Provider and that the claim (i.e. attribute) DataHub uses for uniquely identifying the user is supported by your Identity Provider (refer to Identity Provider OpenID Connect documentation). By default, this claim is `email`. +- Make sure the Discovery URI you've configured (`AUTH_OIDC_DISCOVERY_URI`) is accessible where the datahub-frontend container is running. You can do this by issuing a basic CURL to the address (**Pro-Tip**: you may also visit the address in your browser to check more specific details about your Identity Provider). +- Check the container logs for the `datahub-frontend` container. This should hopefully provide some additional context around why exactly the login handoff is not working. + +If all else fails, feel free to reach out to the DataHub Community on Slack for real-time support. + +
+ +
+ +I'm seeing an error in the `datahub-frontend` logs when a user tries to login: Caused by: java.lang.RuntimeException: Failed to resolve user name claim from profile provided by Identity Provider. Missing attribute. Attribute: 'email', Regex: '(.*)', Profile: { .... + + +This indicates that your Identity Provider does not provide the claim with name 'email', which DataHub +uses by default to uniquely identify users within your organization. + +To fix this, you may need to + +1. Change the claim that is used as the unique user identifier to something else by changing the `AUTH_OIDC_USER_NAME_CLAIM` (e.g. to "name" or "preferred*username") \_OR* +2. Change the environment variable `AUTH_OIDC_SCOPE` to include the scope required to retrieve the claim with name "email" + +For the `datahub-frontend` container / pod. + +
+ +## Reference + +Check the documentation for your Identity Provider to learn more about the scope claims supported. + +- [Registering an App in Okta](https://developer.okta.com/docs/guides/add-an-external-idp/openidconnect/main/) +- [OpenID Connect in Google Identity](https://developers.google.com/identity/protocols/oauth2/openid-connect) +- [OpenID Connect authentication with Azure Active Directory](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/auth-oidc) +- [Keycloak - Securing Applications and Services Guide](https://www.keycloak.org/docs/latest/securing_apps/) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/initialize-oidc.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/initialize-oidc.md new file mode 100644 index 00000000..af2cfdd2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/guides/sso/initialize-oidc.md @@ -0,0 +1,229 @@ +--- +title: Prerequisites for OIDC Authentication +slug: /authentication/guides/sso/initialize-oidc +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/guides/sso/initialize-oidc.md +--- +# Prerequisites for OIDC Authentication + +This guide will walk you through the following steps with your identity provider: + +1. Create and register an application with your identity provider. +2. Obtain client credentials and discovery URI to be used in DataHub. + +Choose your identity provider to get started: + +## Google Identity + +### Step 1. Create and Register your App + +#### 1. Create a project in the Google API Console + +Using an account linked to your organization, navigate to the [Google API Console](https://console.developers.google.com/) and select **New project**. + +Within this project, we will configure the OAuth2.0 screen and credentials. + +

+ +

+ +#### 2. Create OAuth2.0 consent screen + +Navigate to **OAuth consent screen**. This is where you'll configure the screen your users see when attempting to log in to DataHub. Select **Internal** (if you only want your company users to have access) and then click **Create**. + +

+ +

+ +_Note that in order to complete this step you should be logged into a Google account associated with your organization._ + +Fill out the details in the App Information & Domain sections. Make sure the 'Application Home Page' provided matches where DataHub is deployed at your organization. + +

+ +

+ +Once you've completed this, **Save & Continue**. + +#### 3. Configure the appropriate scopes + +Next, click **Add or Remove Scopes**. Select the following scopes and click **Save & Continue**. + +- `.../auth/userinfo.email` +- `.../auth/userinfo.profile` +- `openid` + +

+ +

+ +### Step 2. Create Client Credentials + +The following steps will walk you through generating a Client ID and Client Secret. + +1. Navigate to the **Credentials** tab and click **Create Credentials**. +2. Select **OAuth client ID** as the credential type. +3. On the next screen, select **Web application** as your Application Type. +4. In **Authorized JavaScript Origins**, add the domain where you are hosting DataHub, i.e. `https://your-datahub-domain.com`. +5. In **Authorized Redirect URLs**, add the domain where you are hosting DataHub with the path `/callback/oidc` appended, i.e. `https://your-datahub-domain.com/callback/oidc`. +6. Click **Create**. + +

+ +

+ +This will generate a **Client ID** and **Client Secret**: + +

+ +

+ +You will need these values in the next step, in addition to the following **Discovery URI**: + +``` +https://accounts.google.com/.well-known/openid-configuration` +``` + +## Okta Identity + +### Step 1. Create and Register your App + +#### 1. Create an application in Okta Developer Console + +Log in to your Okta admin account and navigate to the developer console. From there: + +1. Select **Applications** +2. Click **Add Application** +3. Click **Create New App** +4. Select **OpenID Connect** as the Sign On method +5. Choose **Web** as the Platform +6. Click **Create** + +#### 2. Configure application settings + +Under **General Settings**, provide a name for your application and configure the following URIs: + +- Login Redirect URI: `https://your-datahub-domain.com/callback/oidc` +- Logout Redirect URI: `https://your-datahub-domain.com/login` + +

+ +

+ +#### 3. Configure Okta Tile (Optional) + +If you plan to enable DataHub login as an Okta tile, configure the Initiate Login URI: + +- For production: `https://your-datahub-domain.com/authenticate` +- For local testing: `http://localhost:9002` + +### Step 2. Locate Client Credentials and Discovery URI + +After registering your app, navigate to the **General** tab to find the following Client Credential values: + +- **Client ID**: Public identifier for the client that is required for all OAuth flows. +- **Client Secret**: Secret used by the client to exchange an authorization code for a token. + +

+ +

+ +You will need these values in the next step, in addition to the following **Discovery URI**: + +``` +https://your-okta-domain.com/.well-known/openid-configuration +``` + +## Azure AD + +### Step 1. Create and Register your App + +#### 1. Create an application in Microsoft Azure portal + +Using an account linked to your organization, navigate to the [Microsoft Azure Portal](https://portal.azure.com). From there: + +1. Select **App Registrations**. +2. Click **New Registration** to register a new app. +3. Provide a Name for the application and choose the supported account types. +4. Under **Redirect URI**, choose **Web** and enter `https://your-datahub-domain.com/callback/oidc`. NOTE: You can add more later. + +

+ +

+ +5. Click **Register**. + +#### 2. Configure Logout URL + +Once registration is complete, you will need to configure the Logout URL, which is required for SSO to work correctly. + +1. Navigate to **Authentication** from the left-side navigation menu. +2. Set **Front-channel logout URL** to `https://your-datahub-domain.com/login`. +3. Optionally add additional Redirect URIs, such as `http://localhost:9002/callback/oidc` for local testing. +4. Click **Save**. + +

+ +

+ +### Step 2. Client Credentials and Discovery URI + +#### 1. Generate a Client Secret + +You are now ready to create and configure client credentials: + +1. Click **Certificates & secrets** from the left-side navigation menu. +2. Select **Client secrets**, then **New client secret**. +3. Provide a Name for the secret and set an expiry. +4. Click **Add**. +5. Copy the secret **`Value`** to be used as the **Client Secret** in DataHub SSO configuration; **Azure will not display this again**. + +

+ +

+ +#### 2. Configure API Permissions + +Next, you will configure the appropriate API permissions to enable SSO with DataHub. + +1. Click **API permissions** from the left-side navigation menu. +2. Click **Add a permission**. +3. Under the **Microsoft APIs** tab, select **Microsoft Graph**, then **Delegated permissions**. +4. Under the **OpenId permissions** category, select the following: + +- `User.Read` +- `profile` +- `email` +- `openid` + +5. Click **Add permissions**. + +

+ +

+ +#### 3. Locate Client Credentials and Discovery URI + +Now that you have registered your app, generated a client secret, and configured the appropriate permissions, you are now ready to enable Azure AD SSO with DataHub. + +You will need the following values in the next step: + +- **Application (client) ID**: Find this on the **Overview** tab. This will map to **Client ID** in DataHub. +- **Client Secret**: Stored as `Value` in the Client secret you created, above. This will map to **Client Secret** in DataHub. +- **Directory (tenant) ID**: Located on the **Overview** tab. This will map to **Discovery URI** in DataHub. It will be formatted as `https://login.microsoftonline.com/{tenant ID}/v2.0/.well-known/openid-configuration`. + +

+ +

+ +### Next Steps + +Once you have your **Client ID**, **Client Secret**, and **Discovery URI**, you may proceed with next steps. + +### DataHub Cloud + +If you're deployed with DataHub Cloud, you can enable OIDC SSO with a few clicks. [👉 See the guide here](../../../managed-datahub/integrations/oidc-sso-integration.md). + +### Open Source + +If you're self-deployed with DataHub Core, you'll need to configure your frontend server within your deployment environment. [👉 See the guide here](./configure-oidc-react.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/introducing-metadata-service-authentication.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/introducing-metadata-service-authentication.md new file mode 100644 index 00000000..ac88bdd4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/introducing-metadata-service-authentication.md @@ -0,0 +1,200 @@ +--- +title: Metadata Service Authentication +slug: /authentication/introducing-metadata-service-authentication +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/introducing-metadata-service-authentication.md +--- +# Metadata Service Authentication + +## Introduction + +This document provides a technical overview of the how authentication works in DataHub's backend aimed at developers evaluating or operating DataHub. +It includes a characterization of the motivations for the feature, the key components in its design, the new capabilities it provides, & configuration instructions. + +## Background + +Let's recall 2 critical components of DataHub's architecture: + +- **DataHub Frontend Proxy** (datahub-frontend) - Resource server that routes requests to downstream Metadata Service +- **DataHub Metadata Service** (datahub-gms) - Source of truth for storing and serving DataHub Metadata Graph. + +Previously, Authentication was exclusively handled by the Frontend Proxy. This service would perform the following steps +when a user navigated to `http://localhost:9002/`: + +a. Check for the presence of a special `PLAY_SESSION` cookie. + +b. If cookie was present + valid, redirect to the home page + +c. If cookie was invalid, redirect to either a) the DataHub login screen (for [JAAS authentication](guides/jaas.md) or b) a [configured OIDC Identity Provider](guides/sso/configure-oidc-react.md) to perform authentication. + +Once authentication had succeeded at the frontend proxy layer, a stateless (token-based) session cookie (PLAY_SESSION) would be set in the users browser. +All subsequent requests, including the GraphQL requests issued by the React UI, would be authenticated using this session cookie. Once a request had made it beyond +the frontend service layer, it was assumed to have been already authenticated. Hence, there was **no native authentication inside of the Metadata Service**. + +### Problems with this approach + +The major challenge with this situation is that requests to the backend Metadata Service were completely unauthenticated. There were 2 options for folks who required authentication at the Metadata Service layer: + +1. Set up a proxy in front of Metadata Service that performed authentication +2. [A more recent possibility] Route requests to Metadata Service through DataHub Frontend Proxy, including the PLAY_SESSION + Cookie with every request. + +Neither of which are ideal. Setting up a proxy to do authentication takes time & expertise. Extracting and setting a session cookie from the browser for programmatic is +clunky & unscalable. On top of that, extending the authentication system was difficult, requiring implementing a new [Play module](https://www.playframework.com/documentation/2.8.8/api/java/play/mvc/Security.Authenticator.html) within DataHub Frontend. + +## Introducing Authentication in DataHub Metadata Service + +To address these problems, we introduced configurable Authentication inside the **Metadata Service** itself, +meaning that requests are no longer considered trusted until they are authenticated by the Metadata Service. + +Why push Authentication down? In addition to the problems described above, we wanted to plan for a future +where Authentication of Kafka-based-writes could be performed in the same manner as Rest writes. + +## Configuring Metadata Service Authentication + +Metadata Service Authentication is currently **opt-in**. This means that you may continue to use DataHub without Metadata Service Authentication without interruption. +To enable Metadata Service Authentication: + +- set the `METADATA_SERVICE_AUTH_ENABLED` environment variable to "true" for the `datahub-gms` AND `datahub-frontend` containers / pods. + +OR + +- change the Metadata Service `application.yaml` configuration file to set `authentication.enabled` to "true" AND +- change the Frontend Proxy Service `application.config` configuration file to set `metadataService.auth.enabled` to "true" + +:::note +It is recommended that users provide their own values for `authentication.tokenService.signingKey` and `authentication.tokenService.salt` by updating `application.yaml` in Metadata Service or setting the corresponding environment variables `DATAHUB_TOKEN_SERVICE_SIGNING_KEY` and `DATAHUB_TOKEN_SERVICE_SALT` +::: + +After setting the configuration flag, simply restart the Metadata Service to start enforcing Authentication. + +Once enabled, all requests to the Metadata Service will need to be authenticated; if you're using the default Authenticators +that ship with DataHub, this means that all requests will need to present an Access Token in the Authorization Header as follows: + +``` +Authorization: Bearer +``` + +For users logging into the UI, this process will be handled for you. When logging in, a cookie will be set in your browser that internally +contains a valid Access Token for the Metadata Service. When browsing the UI, this token will be extracted and sent to the Metadata Service +to authenticate each request. + +For users who want to access the Metadata Service programmatically, i.e. for running ingestion, the current recommendation is to generate +a **Personal Access Token** (described above) from the root "datahub" user account, and using this token when configuring your [Ingestion Recipes](../../metadata-ingestion/README.md#recipes). +To configure the token for use in ingestion, simply populate the "token" configuration for the `datahub-rest` sink: + +``` +source: + # source configs +sink: + type: "datahub-rest" + config: + ... + token: +``` + +> Note that ingestion occurring via `datahub-kafka` sink will continue to be Unauthenticated _for now_. Soon, we will be introducing +> support for providing an access token in the event payload itself to authenticate ingestion requests over Kafka. + +### The Role of DataHub Frontend Proxy Going Forward + +With these changes, DataHub Frontend Proxy will continue to play a vital part in the complex dance of Authentication. It will serve as the place +where UI-based session authentication originates and will continue to support 3rd Party SSO configuration (OIDC) +and JAAS configuration as it does today. + +The major improvement is that the Frontend Service will validate credentials provided at UI login time +and generate a DataHub **Access Token**, embedding it into traditional session cookie (which will continue to work). + +In summary, DataHub Frontend Service will continue to play a vital role to Authentication. It's scope, however, will likely +remain limited to concerns specific to the React UI. + +## Where to go from here + +These changes represent the first milestone in Metadata Service Authentication. They will serve as a foundation upon which we can build new features, prioritized based on Community demand: + +1. **Dynamic Authenticator Plugins**: Configure + register custom Authenticator implementations, without forking DataHub. +2. **Service Accounts**: Create service accounts and generate Access tokens on their behalf. +3. **Kafka Ingestion Authentication**: Authenticate ingestion requests coming from the Kafka ingestion sink inside the Metadata Service. +4. **Access Token Management**: Ability to view, manage, and revoke access tokens that have been generated. (Currently, access tokens include no server side state, and thus cannot be revoked once granted) + +...and more! To advocate for these features or others, reach out on [Slack](https://datahubspace.slack.com/join/shared_invite/zt-nx7i0dj7-I3IJYC551vpnvvjIaNRRGw#/shared-invite/email). + +## Q&As + +### What if I don't want to use Metadata Service Authentication? + +That's perfectly fine, for now. Metadata Service Authentication is disabled by default, only enabled if you provide the +environment variable `METADATA_SERVICE_AUTH_ENABLED` to the `datahub-gms` container or change the `authentication.enabled` to "true" +inside your DataHub Metadata Service configuration (`application.yaml`). + +That being said, we will be recommending that you enable Authentication for production use cases, to prevent +arbitrary actors from ingesting metadata into DataHub. + +### If I enable Metadata Service Authentication, will ingestion stop working? + +If you enable Metadata Service Authentication, you will want to provide a value for the "token" configuration value +when using the `datahub-rest` sink in your [Ingestion Recipes](/docs/metadata-ingestion/#recipes). See +the [Rest Sink Docs](/docs/metadata-ingestion/sink_docs/datahub#config-details) for configuration details. + +We'd recommend generating a Personal Access Token (described above) from a trusted DataHub Account (e.g. root 'datahub' user) when configuring +your Ingestion sources. + +Note that you can also provide the "extraHeaders" configuration in `datahub-rest` sink to specify a custom header to +pass with each request. This can be used in conjunction to authenticate using a custom Authenticator, for example. + +### How do I generate an Access Token for a service account? + +There is no formal concept of "service account" or "bot" on DataHub (yet). For now, we recommend you configure any +programmatic clients of DataHub to use a Personal Access Token generated from a user with the correct privileges, for example +the root "datahub" user account. + +### I want to authenticate requests using a custom Authenticator? How do I do this? + +You can configure DataHub to add your custom **Authenticator** to the **Authentication Chain** by changing the `application.yaml` configuration file for the Metadata Service: + +```yml +authentication: + enabled: true # Enable Metadata Service Authentication + .... + authenticators: # Configure an Authenticator Chain + - type: # E.g. com.linkedin.datahub.authentication.CustomAuthenticator + configs: # Specific configs that should be passed into 'init' method of Authenticator + customConfig1: +``` + +Notice that you will need to have a class that implements the `Authenticator` interface with a zero-argument constructor available on the classpath +of the Metadata Service java process. + +We love contributions! Feel free to raise a PR to contribute an Authenticator back if it's generally useful. + +### Now that I can make authenticated requests to either DataHub Proxy Service and DataHub Metadata Service, which should I use? + +Previously, we were recommending that folks contact the Metadata Service directly when doing things like + +- ingesting Metadata via recipes +- issuing programmatic requests to the Rest.li APIs +- issuing programmatic requests to the GraphQL APIs + +With these changes, we will be shifting to the recommendation that folks direct all traffic, whether it's programmatic or not, +to the **DataHub Frontend Proxy**, as routing to Metadata Service endpoints is currently available at the path `/api/gms`. +This recommendation is in effort to minimize the exposed surface area of DataHub to make securing, operating, maintaining, and developing +the platform simpler. + +In practice, this will require migrating Metadata [Ingestion Recipes](../../metadata-ingestion/README.md#recipes) use the `datahub-rest` sink to pointing at a slightly different +host + path. + +Example recipe that proxies through DataHub Frontend + +```yml +source: + # source configs +sink: + type: "datahub-rest" + config: + ... + token: +``` + +## Feedback / Questions / Concerns + +We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on [Slack](https://datahubspace.slack.com/join/shared_invite/zt-nx7i0dj7-I3IJYC551vpnvvjIaNRRGw#/shared-invite/email)! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authentication/personal-access-tokens.md b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/personal-access-tokens.md new file mode 100644 index 00000000..ce5e713d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authentication/personal-access-tokens.md @@ -0,0 +1,114 @@ +--- +title: Personal Access Tokens +slug: /authentication/personal-access-tokens +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authentication/personal-access-tokens.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Personal Access Tokens + + + +Personal Access Tokens, or PATs for short, allow users to represent themselves in code and programmatically use DataHub's APIs in deployments where security is a concern. + +Used along-side with [authentication-enabled metadata service](introducing-metadata-service-authentication.md), PATs add a layer of protection to DataHub where only authorized users are able to perform actions in an automated way. + +## Personal Access Tokens Setup, Prerequisites, and Permissions + +To use PATs, two things are required: + +1. Metadata Authentication must have been enabled in GMS. See `Configuring Metadata Service Authentication` in [authentication-enabled metadata service](introducing-metadata-service-authentication.md). +2. Users must have been granted the `Generate Personal Access Tokens` or `Manage All Access Tokens` Privilege via a [DataHub Policy](../authorization/policies.md). + +Once configured, users should be able to navigate to **'Settings'** > **'Access Tokens'** > **'Generate Personal Access Token'** to generate a token: + +

+ +

+ +If you have configured permissions correctly the `Generate new token` should be clickable. + +:::note + +If you see `Token based authentication is currently disabled. Contact your DataHub administrator to enable this feature.` then you must enable authentication in the metadata service (step 1 of the prerequisites). + +::: + +## Creating Personal Access Tokens + +Once in the Manage Access Tokens Settings Tab: + +1. Click `Generate new token` where a form should appear. + +

+ +

+ +2. Fill out the information as needed and click `Create`. +

+ +

+ +3. Save the token text somewhere secure! This is what will be used later on! +

+ +

+ +## Using Personal Access Tokens + +Once a token has been generated, the user that created it will subsequently be able to make authenticated HTTP requests, assuming he/she has permissions to do so, to DataHub frontend proxy or DataHub GMS directly by providing +the generated Access Token as a Bearer token in the `Authorization` header: + +``` +Authorization: Bearer +``` + +For example, using a curl to the frontend proxy (preferred in production): + +```bash +curl 'http://localhost:9002/api/gms/entities/urn:li:corpuser:datahub' -H 'Authorization: Bearer +``` + +or to Metadata Service directly: + +```bash +curl 'http://localhost:8080/entities/urn:li:corpuser:datahub' -H 'Authorization: Bearer +``` + +Since authorization happens at the GMS level, this means that ingestion is also protected behind access tokens, to use them simply add a `token` to the sink config property as seen below: + +

+ +

+ +:::note + +Without an access token, making programmatic requests will result in a 401 result from the server if Metadata Service Authentication +is enabled. + +::: + +## Additional Resources + +- Learn more about how this feature is by DataHub [Authentication Metadata Service](introducing-metadata-service-authentication.md). +- Check out our [Authorization Policies](../authorization/policies.md) to see what permissions can be programmatically used. + +### GraphQL + +- Have a look at [Token Management in GraphQL](../api/graphql/token-management.md) to learn how to manage tokens programmatically! + +## FAQ and Troubleshooting + +**The button to create tokens is greyed out - why can’t I click on it?** + +This means that the user currently logged in DataHub does not have either `Generate Personal Access Tokens` or `Manage All Access Tokens` permissions. +Please ask your DataHub administrator to grant you those permissions. + +**When using a token, I get 401 unauthorized - why?** + +A PAT represents a user in DataHub, if that user does not have permissions for a given action, neither will the token. + +**Can I create a PAT that represents some other user?** + +Yes, although not through the UI correctly, you will have to use the [token management graphQL API](../api/graphql/token-management.md) and the user making the request must have `Manage All Access Tokens` permissions. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authorization/README.md b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/README.md new file mode 100644 index 00000000..c8c92f33 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/README.md @@ -0,0 +1,26 @@ +--- +title: DataHub Authorization Overview +sidebar_label: DataHub Authorization Overview +slug: /authorization +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authorization/README.md +--- + +# Authorization Overview + +Authorization specifies _what_ accesses an _authenticated_ user has within a system. +This section is all about how DataHub authorizes a given user/service that wants to interact with the system. + +:::note + +Authorization only makes sense in the context of an **Authenticated** DataHub deployment. To use DataHub's authorization features +please first make sure that the system has been configured from an authentication perspective as you intend. + +::: + +Once the identity of a user or service has been established, DataHub determines what accesses the authenticated request has. + +This is done by checking what operation a given user/service wants to perform within DataHub & whether it is allowed to do so. +The set of operations that are allowed in DataHub are what we call **Policies**. + +Policies specify fine-grain access control for _who_ can do _what_ to _which_ resources, for more details on the set of Policies that DataHub provides please see the [Policies Guide](../authorization/policies.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authorization/access-policies-guide.md b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/access-policies-guide.md new file mode 100644 index 00000000..a7a98bba --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/access-policies-guide.md @@ -0,0 +1,278 @@ +--- +title: Access Policies +slug: /authorization/access-policies-guide +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authorization/access-policies-guide.md +--- +# Access Policies + + + +Access Policies define who can do what to which resources. In conjunction with [Roles](./roles.md), Access Policies determine what users are allowed to do on DataHub. + +## Policy Types + +There are 2 types of Access Policy within DataHub: + +1. **Platform** Policies +2. **Metadata** Policies + +

+ +

+ +## Platform + +Policies determine who has platform-level Privileges on DataHub. These include: + +- Managing Users & Groups +- Viewing the DataHub Analytics Page +- Managing Policies themselves + +Platform policies can be broken down into 2 parts: + +1. **Privileges**: Which privileges should be assigned to the Actors (e.g. "View Analytics") +2. **Actors**: Who the should be granted the privileges (Users, or Groups) + +A few Platform Policies in plain English include: + +- The Data Platform team should be allowed to manage users & groups, view platform analytics, & manage policies themselves +- John from IT should be able to invite new users + +## Metadata + +Metadata policies determine who can do what to which Metadata Entities. For example: + +- Who can edit Dataset Documentation & Links? +- Who can add Owners to a Chart? +- Who can add Tags to a Dashboard? + +Metadata policies can be broken down into 3 parts: + +1. **Privileges**: The 'what'. What actions are being permitted by a Policy, e.g. "Add Tags". +2. **Resources**: The 'which'. Resources that the Policy applies to, e.g. "All Datasets". +3. **Actors**: The 'who'. Specific users, groups, & roles that the Policy applies to. + +A few **Metadata** Policies in plain English include: + +- Dataset Owners should be allowed to edit documentation, but not Tags. +- Jenny, our Data Steward, should be allowed to edit Tags for any Dashboard, but no other metadata. +- James, a Data Analyst, should be allowed to edit the Links for a specific Data Pipeline he is a downstream consumer of. + +Each of these can be implemented by constructing DataHub Access Policies. + +## Using Access Policies + +:::note Required Access + +- **Manage Policies** Privilege + +This Platform Privilege allows users to create, edit, and remove all Access Policies on DataHub. Therefore, it should only be +given to those users who will be serving as Admins of the platform. The default `Admin` role has this Privilege. +::: + +Policies can be created by first navigating to **Settings > Permissions > Policies**. + +To begin building a new Policy, click **Create new Policy**. + +

+ +

+ +### Creating a Platform Policy + +#### Step 1. Provide a Name & Description + +In the first step, we select the **Platform** Policy type, and define a name and description for the new Policy. + +Good Policy names describe the high-level purpose of the Policy. For example, a Policy named +"View DataHub Analytics - Data Governance Team" would be a great way to describe a Platform +Policy which grants abilities to view DataHub's Analytics view to anyone on the Data Governance team. + +You can optionally provide a text description to add richer details about the purpose of the Policy. + +#### Step 2: Configure Privileges + +In the second step, we can simply select the Privileges that this Platform Policy will grant. + +

+ +

+ +**Platform** Privileges most often provide access to perform administrative functions on the Platform. +Refer to the [Policies Guide](./policies.md#platform-level-privileges) for a complete list of these privileges. + +#### Step 3: Choose Policy Actors + +In this step, we can select the actors who should be granted Privileges appearing on this Policy. + +To do so, simply search and select the Users or Groups that the Policy should apply to. + +**Assigning a Policy to a User** + +

+ +

+ +**Assigning a Policy to a Group** + +

+ +

+ +### Creating a Metadata Policy + +#### Step 1. Provide a Name & Description + +In the first step, we select the **Metadata** Policy, and define a name and description for the new Policy. + +Good Policy names describe the high-level purpose of the Policy. For example, a Policy named +"Full Dataset Edit Privileges - Data Platform Engineering" would be a great way to describe a Metadata +Policy which grants all abilities to edit Dataset Metadata to anyone in the "Data Platform" group. + +You can optionally provide a text description to add richer detail about the purpose of the Policy. + +#### Step 2: Configure Privileges + +In the second step, we can simply select the Privileges that this Metadata Policy will grant. +To begin, we should first determine which assets that the Privileges should be granted for (i.e. the _scope_), then +select the appropriate Privileges to grant. + +Using the `Resource Type` selector, we can narrow down the _type_ of the assets that the Policy applies to. If left blank, +all entity types will be in scope. + +For example, if we only want to grant access for `Datasets` on DataHub, we can select +`Datasets`. + +

+ +

+ +Next, we can search for specific Entities of the that the Policy should grant privileges on. +If left blank, all entities of the selected types are in scope. + +For example, if we only want to grant access for a specific sample dataset, we can search and +select it directly. + +

+ +

+ +We can also limit the scope of the Policy to assets that live in a specific **Domain**. If left blank, +entities from all Domains will be in scope. + +For example, if we only want to grant access for assets part of a "Marketing" Domain, we can search and +select it. + +

+ +

+ +Finally, we will choose the Privileges to grant when the selected entities fall into the defined +scope. + +

+ +

+ +**Metadata** Privileges grant access to change specific _entities_ (i.e. data assets) on DataHub. +These include [**common metadata privileges**](./policies.md#platform-level-privileges) that span across entity types, as well as [**specific entity-level privileges**](./policies.md#specific-entity-level-privileges). + +#### Step 3: Choose Policy Actors + +In this step, we can select the actors who should be granted the Privileges on this Policy. Metadata Policies +can target specific Users & Groups, or the _owners_ of the Entities that are included in the scope of the Policy. + +To do so, simply search and select the Users or Groups that the Policy should apply to. + +

+ +

+ +

+ +

+ +We can also grant the Privileges to the _owners_ of Entities (or _Resources_) that are in scope for the Policy. +This advanced functionality allows of Admins of DataHub to closely control which actions can or cannot be performed by owners. + +

+ +

+ +### Updating an Existing Policy + +To update an existing Policy, simply click the **Edit** on the Policy you wish to change. + +

+ +

+ +Then, make the changes required and click **Save**. When you save a Policy, it may take up to 2 minutes for changes +to be reflected. + +### Removing a Policy + +To remove a Policy, simply click on the trashcan icon located on the Policies list. This will remove the Policy and +deactivate it so that it no longer applies. + +When you delete a Policy, it may take up to 2 minutes for changes to be reflected. + +### Deactivating a Policy + +In addition to deletion, DataHub also supports "deactivating" a Policy. This is useful if you need to temporarily disable +a particular Policy, but do not want to remove it altogether. + +To deactivate a Policy, simply click the **Deactivate** button on the Policy you wish to deactivate. When you change +the state of a Policy, it may take up to 2 minutes for the changes to be reflected. + +

+ +

+ +After deactivating, you can re-enable a Policy by clicking **Activate**. + +### Default Policies + +Out of the box, DataHub is deployed with a set of pre-baked Policies. This set of policies serves the +following purposes: + +1. Assigns immutable super-user privileges for the root `datahub` user account (Immutable) +2. Assigns all Platform Privileges for all Users by default (Editable) + +The reason for #1 is to prevent people from accidentally deleting all policies and getting locked out (`datahub` super user account can be a backup) +The reason for #2 is to permit administrators to log in via OIDC or another means outside of the `datahub` root account +when they are bootstrapping with DataHub. This way, those setting up DataHub can start managing Access Policies without friction. +Note that these Privileges _can_ and likely _should_ be changed inside the **Policies** page before onboarding +your company's users. + +### REST API Authorization + +Policies only affect REST APIs when the environment variable `REST_API_AUTHORIZATION` is set to `true` for GMS. Some policies only apply when this setting is enabled, marked above, and other Metadata and Platform policies apply to the APIs where relevant, also specified in the table above. + +## Additional Resources + +- [Authorization Overview](./README.md) +- [Roles Overview](./roles.md) +- [Authorization using Groups](./groups.md) + +### Videos + +- [Introducing DataHub Access Policies](https://youtu.be/19zQCznqhMI?t=282) + +### GraphQL + +- [listPolicies](../../graphql/queries.md#listpolicies) +- [createPolicy](../../graphql/mutations.md#createpolicy) +- [updatePolicy](../../graphql/mutations.md#updatepolicy) +- [deletePolicy](../../graphql/mutations.md#deletepolicy) + +## FAQ and Troubleshooting + +**How do Policies relate to Roles?** + +Policies are the lowest level primitive for granting Privileges to users on DataHub. + +Roles are built for convenience on top of Policies. Roles grant Privileges to actors indirectly, driven by Policies +behind the scenes. Both can be used in conjunction to grant Privileges to end users. For more information on roles +please refer to [Authorization > Roles](./roles.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authorization/groups.md b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/groups.md new file mode 100644 index 00000000..101b50ce --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/groups.md @@ -0,0 +1,38 @@ +--- +title: Authorization using Groups +slug: /authorization/groups +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authorization/groups.md +--- +# Authorization using Groups + +## Introduction + +DataHub provides the ability to use **Groups** to manage policies. + +## Why do we need groups for authorization? + +### Easily Applying Access Privileges + +Groups are useful for managing user privileges in DataHub. If you want a set of Admin users, +or you want to define a set of users that are only able to view metadata assets but not make changes to them, you could +create groups for each of these use cases and apply the appropriate policies at the group-level rather than the +user-level. + +### Syncing with Existing Enterprise Groups (via IdP) + +If you work with an Identity Provider like Okta or Azure AD, it's likely you already have groups defined there. DataHub +allows you to import the groups you have from OIDC for [Okta](../generated/ingestion/sources/okta.md) and +[Azure AD](../generated/ingestion/sources/azure-ad.md) using the DataHub ingestion framework. + +If you routinely ingest groups from these providers, you will also be able to keep groups synced. New groups will +be created in DataHub, stale groups will be deleted, and group membership will be updated! + +## Custom Groups + +DataHub admins can create custom groups by going to the **Settings > Users & Groups > Groups > Create Group**. +Members can be added to Groups via the Group profile page. + +## Feedback / Questions / Concerns + +We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on Slack! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authorization/policies.md b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/policies.md new file mode 100644 index 00000000..fd59f6e4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/policies.md @@ -0,0 +1,448 @@ +--- +title: Policies Guide +slug: /authorization/policies +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authorization/policies.md +--- +# Policies Guide + +## Introduction + +DataHub provides the ability to declare fine-grained access control Policies via the UI & GraphQL API. +Access policies in DataHub define _who_ can _do what_ to _which resources_. A few policies in plain English include + +- Dataset Owners should be allowed to edit documentation, but not Tags. +- Jenny, our Data Steward, should be allowed to edit Tags for any Dashboard, but no other metadata. +- James, a Data Analyst, should be allowed to edit the Links for a specific Data Pipeline he is a downstream consumer of. +- The Data Platform team should be allowed to manage users & groups, view platform analytics, & manage policies themselves. + +In this document, we'll take a deeper look at DataHub Policies & how to use them effectively. + +## What is a Policy? + +There are 2 types of Policy within DataHub: + +1. Platform Policies +2. Metadata Policies + +We'll briefly describe each. + +### Platform Policies + +**Platform** policies determine who has platform-level privileges on DataHub. These privileges include + +- Managing Users & Groups +- Viewing the DataHub Analytics Page +- Managing Policies themselves + +Platform policies can be broken down into 2 parts: + +1. **Actors**: Who the policy applies to (Users or Groups) +2. **Privileges**: Which privileges should be assigned to the Actors (e.g. "View Analytics") + +Note that platform policies do not include a specific "target resource" against which the Policies apply. Instead, +they simply serve to assign specific privileges to DataHub users and groups. + +### Metadata Policies + +**Metadata** policies determine who can do what to which Metadata Entities. For example, + +- Who can edit Dataset Documentation & Links? +- Who can add Owners to a Chart? +- Who can add Tags to a Dashboard? + +and so on. + +A Metadata Policy can be broken down into 3 parts: + +1. **Resources**: The 'which'. Resources that the policy applies to, e.g. "All Datasets". +2. **Privileges**: The 'what'. What actions are being permitted by a policy, e.g. "Add Tags". +3. **Actors**: The 'who'. Specific users, groups that the policy applies to. + +#### Resources + +Resources can be associated with the policy in a number of ways: + +1. **Resource types** - The entity's type, for example: dataset, chart, dashboard +2. **Resource URNs** - Specific entity URNs to target +3. **Tags** - Assets tagged with specific tags +4. **Domains** - Assets within specific domains +5. **Containers** - Assets within specific containers (e.g., databases, schemas) +6. **Glossary Terms or Term Groups** - Assets annotated with specific glossary terms or any term within a term group (see [Glossary-Based Policy Targeting](#glossary-based-policy-targeting) below) + +:::note Important Note +The associations in the list above are an _intersection_ or an _AND_ operation. For example, if the policy targets +`1. resource type: dataset` and `3. resources tagged: 'myTag'`, it will apply to datasets that are tagged with tag 'myTag'. +::: + +##### Domain-Based Policy Targeting + +Policies can be targeted to assets based on **Domains**. This allows you to apply permissions to all assets within a specific business domain. + +When you target a policy by domain, the policy applies recursively to any asset that belongs to that domain as well as any assets in nested child domains. + +**Example**: A policy targeting the "Marketing" domain will apply to all datasets, dashboards, and other assets assigned to that domain, as well as assets in child domains like "Marketing Analytics" or "Marketing Campaigns". + +##### Container-Based Policy Targeting + +Policies can be targeted to assets based on **Containers** (e.g., databases, schemas, projects). This allows you to apply permissions based on technical organization. + +When you target a policy by container, the policy applies recursively to all assets within that container as well as any assets in nested child containers. + +**Example**: A policy targeting a "production" database container will apply to all schemas within that database and all tables within those schemas. + +##### Glossary-Based Policy Targeting + +Policies can be targeted to assets based on **Glossary Terms** or **Glossary Term Groups**. This allows you to apply permissions based on business vocabulary rather than technical properties. + +When you target a policy by glossary terms or groups: + +- **Individual Terms**: The policy applies to any asset annotated with that specific glossary term +- **Term Groups**: The policy applies recursively to assets annotated with any term within that group, including all nested child terms and groups + +This works similarly to domain and container-based policies, automatically covering assets as your glossary evolves. + +**Example**: A policy targeting the "Sensitive Data" term group will apply to all assets tagged with child terms like "PII", "PHI", or any terms in nested groups, without requiring policy updates when new terms are added to the hierarchy. + +#### Privileges + +Check out the list of +privileges [here](https://github.com/datahub-project/datahub/blob/master/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java) +. Note, the privileges are semantic by nature, and does not tie in 1-to-1 with the aspect model. + +All edits on the UI are covered by a privilege, to make sure we have the ability to restrict write access. See the +[Reference](#Reference) section below. + +#### Actors + +We currently support 3 ways to define the set of actors the policy applies to: + +1. list of users (or all users) +2. list of groups (or all groups) +3. owners of the entity + +:::note Important Note +Unlike resources, the definitions for actors are a union of the actors. For example, if user `1. Alice` is associated +with the policy as well as `3. owners of the entity`. This means that Alice _OR_ any owner of +the targeted resource(s) will be included in the policy. +::: + +## Managing Policies + +Policies can be managed on the page **Settings > Permissions > Policies** page. The `Policies` tab will only +be visible to those users having the `Manage Policies` privilege. + +Out of the box, DataHub is deployed with a set of pre-baked Policies. The set of default policies are created at deploy +time and can be found inside the `policies.json` file within `metadata-service/war/src/main/resources/boot`. This set of policies serves the +following purposes: + +1. Assigns immutable super-user privileges for the root `datahub` user account (Immutable) +2. Assigns all Platform privileges for all Users by default (Editable) + +The reason for #1 is to prevent people from accidentally deleting all policies and getting locked out (`datahub` super user account can be a backup) +The reason for #2 is to permit administrators to log in via OIDC or another means outside of the `datahub` root account +when they are bootstrapping with DataHub. This way, those setting up DataHub can start managing policies without friction. +Note that these privilege _can_ and likely _should_ be altered inside the **Policies** page of the UI. + +:::note Pro-Tip +To login using the `datahub` account, simply navigate to `/login` and enter `datahub`, `datahub`. Note that the password can be customized for your +deployment by changing the `user.props` file within the `datahub-frontend` module. Notice that JaaS authentication must be enabled. +:::note + +## Configuration + +By default, the Policies feature is _enabled_. This means that the deployment will support creating, editing, removing, and +most importantly enforcing fine-grained access policies. + +In some cases, these capabilities are not desirable. For example, if your company's users are already used to having free reign, you +may want to keep it that way. Or perhaps it is only your Data Platform team who actively uses DataHub, in which case Policies may be overkill. + +For these scenarios, we've provided a back door to disable Policies in your deployment of DataHub. This will completely hide +the policies management UI and by default will allow all actions on the platform. It will be as though +each user has _all_ privileges, both of the **Platform** & **Metadata** flavor. + +To disable Policies, you can simply set the `AUTH_POLICIES_ENABLED` environment variable for the `datahub-gms` service container +to `false`. For example in your `docker/datahub-gms/docker.env`, you'd place + +``` +AUTH_POLICIES_ENABLED=false +``` + +### REST API Authorization + +Policies only affect REST APIs when the environment variable `REST_API_AUTHORIZATION` is set to `true` for GMS. Some policies only apply when this setting is enabled, marked above, and other Metadata and Platform policies apply to the APIs where relevant, also specified in the table above. + +## Reference + +For a complete list of privileges see the +privileges [here](https://github.com/datahub-project/datahub/blob/master/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java). + +### Platform-level privileges + +These privileges are for DataHub operators to access & manage the administrative functionality of the system. + +#### Access & Credentials + +| Platform Privileges | Description | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --- | +| Generate Personal Access Tokens | Allow actor to generate personal access tokens for use with DataHub APIs. | +| Manage Policies | Allow actor to create and remove access control policies. Be careful - Actors with this privilege are effectively super users. | +| Manage Secrets | Allow actor to create & remove Secrets stored inside DataHub. | +| Manage Users & Groups | Allow actor to create, remove, and update users and groups on DataHub. | +| Manage All Access Tokens | Allow actor to create, list and revoke access tokens on behalf of users in DataHub. Be careful - Actors with this privilege are effectively super users that can impersonate other users. | +| Manage User Credentials | Allow actor to manage credentials for native DataHub users, including inviting new users and resetting passwords | | +| Manage Connections | Allow actor to manage connections to external DataHub platforms. | + +#### Product Features + +| Platform Privileges | Description | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------ | +| Manage Home Page Posts | Allow actor to create and delete home page posts | +| Manage Business Attribute | Allow actor to create, update, delete Business Attribute | +| Manage Documentation Forms | Allow actor to manage forms assigned to assets to assist in documentation efforts. | +| Manage Metadata Ingestion | Allow actor to create, remove, and update Metadata Ingestion sources. | +| Manage Features | Umbrella privilege to manage all features. | +| View Analytics | Allow actor to view the DataHub analytics dashboard. | +| Manage Public Views | Allow actor to create, update, and delete any Public (shared) Views. | +| Manage Ownership Types | Allow actor to create, update and delete Ownership Types. | +| Create Business Attribute | Allow actor to create new Business Attribute. | +| Manage Structured Properties | Manage structured properties in your instance. | +| View Tests | View Asset Tests. | +| Manage Tests[^1] | Allow actor to create and remove Asset Tests. | +| View Metadata Proposals[^1] | Allow actor to view the requests tab for viewing metadata proposals. | +| Create metadata constraints[^2] | Allow actor to create metadata constraints. | +| Manage Platform Settings[^1] | Allow actor to view and change platform-level settings, like integrations & notifications. | +| Manage Monitors[^1] | Allow actor to create, update, and delete any data asset monitors, including Custom SQL monitors. Grant with care. | +| View Manage Tags | Allow the actor to view the Manage Tags page. | + +#### Entity Management + +| Platform Privileges | Description | +| ------------------- | --------------------------------------------------------- | +| Manage Domains | Allow actor to create and remove Asset Domains. | +| Manage Glossaries | Allow actor to create, edit, and remove Glossary Entities | +| Manage Tags | Allow actor to create and remove Tags. | + +#### System Management + +| Platform Privileges | Description | +| --------------------------------------------- | -------------------------------------------------------------------------------------------------------- | --- | +| Restore Indices API[^3] | Allow actor to use the Restore Indices API. | | +| Get Timeseries index sizes API[^3] | Allow actor to use the get Timeseries indices size API. | +| Truncate timeseries aspect index size API[^3] | Allow actor to use the API to truncate a timeseries index. | +| Get ES task status API[^3] | Allow actor to use the get task status API for an ElasticSearch task. | +| Enable/Disable Writeability API[^3] | Allow actor to enable or disable GMS writeability for data migrations. | +| Apply Retention API[^3] | Allow actor to apply retention using the API. | +| Analytics API access[^3] | Allow actor to use API read access to raw analytics data. | +| Explain ElasticSearch Query API[^3] | Allow actor to use the Operations API explain endpoint. | +| Produce Platform Event API[^3] | Allow actor to produce Platform Events using the API. | +| Manage System Operations | Allow actor to manage system operation controls. This setting includes all System Management privileges. | + +### Common Metadata Privileges + +These privileges are to view & modify any entity within DataHub. + +#### Entity Privileges + +| Entity Privileges | Description | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| View Entity Page | Allow actor to view the entity page. | +| Edit Entity | Allow actor to edit any information about an entity. Super user privileges for the entity. | +| Delete | Allow actor to delete this entity. | +| Create Entity | Allow actor to create an entity if it doesn't exist. | +| Entity Exists | Allow actor to determine whether the entity exists. | +| Execute Entity | Allow actor to execute entity ingestion. | +| Get Timeline API[^3] | Allow actor to use the GET Timeline API. | +| Get Entity + Relationships API[^3] | Allow actor to use the GET Entity and Relationships API. | +| Get Aspect/Entity Count APIs[^3] | Allow actor to use the GET Aspect/Entity Count APIs. | +| View Entity[^1] | Allow actor to view the entity in search results. This privilege can be explicitly granted, but is also implied by the `View Entity Page` privilege. | +| Share Entity[^1] | Allow actor to share an entity with another DataHub Cloud instance. | + +#### Aspect Privileges + +| Aspect Privileges | Description | +| ----------------------------- | ------------------------------------------------------------------ | +| Edit Tags | Allow actor to add and remove tags to an asset. | +| Edit Glossary Terms | Allow actor to add and remove glossary terms to an asset. | +| Edit Description | Allow actor to edit the description (documentation) of an entity. | +| Edit Links | Allow actor to edit links associated with an entity. | +| Edit Status | Allow actor to edit the status of an entity (soft deleted or not). | +| Edit Domain | Allow actor to edit the Domain of an entity. | +| Edit Data Product | Allow actor to edit the Data Product of an entity. | +| Edit Deprecation | Allow actor to edit the Deprecation status of an entity. | +| Edit Incidents | Allow actor to create and remove incidents for an entity. | +| Edit Lineage | Allow actor to add and remove lineage edges for this entity. | +| Edit Properties | Allow actor to edit the properties for an entity. | +| Edit Owners | Allow actor to add and remove owners of an entity. | +| Get Timeseries Aspect API[^3] | Allow actor to use the GET Timeseries Aspect API. | + +#### Proposals + +| Proposals Privileges | Description | +| ------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| Propose Tags[^1] | Allow actor to propose adding a tag to an asset. | +| Propose Glossary Terms[^1] | Allow actor to propose adding a glossary term to an asset. | +| Propose Owners[^1] | Allow actor to propose adding an owner to an asset. | +| Propose Domains[^1] | Allow actor to propose adding a domain to an asset. | +| Propose Data Contract[^1] | Allow actor to propose adding a data contract to a dataset. | +| Propose Structured properties[^1] | Allow actor to propose adding a structured property to an asset. | +| Propose Documentation[^1] | Allow actor to propose updates to an asset's documentation. | +| Propose Dataset Column Glossary Terms[^1] | Allow actor to propose a glossary term to a dataset schema column (field). | +| Propose Dataset Column Tags[^1] | Allow actor to propose a tag to a dataset schema column (field). | +| Propose Dataset Column Descriptions[^1] | Allow actor to propose a updates to dataset's schema column (field) description | +| Propose Dataset Column Structured Properties[^1] | Allow actor to propose a structured property to a dataset schema column (field). | +| Propose Create Glossary Term[^1] | Allow actor to propose creation of a new glossary term. | +| Propose Create Glossary Node[^1] | Allow actor to propose creation of a new glossary node. | +| Manage Tag Proposals[^1] | Allow actor to manage a proposal to add a tag to an asset. | +| Manage Glossary Term Proposals[^1] | Allow actor to manage a proposal to add a glossary term to an asset. | +| Manage Domain Proposals[^1] | Allow actor to manage a proposal to add a domain to an asset. | +| Manage Owner Proposals[^1] | Allow actor to manage a proposal to add an owner to an asset. | +| Manage Property Proposals[^1] | Allow actor to manage a proposal to add a structured property to an asset. | +| Manage Data Contract Proposals[^1] | Allow actor to manage a proposal to add a data contract to a dataset. | +| Manage Documentation Proposals[^1] | Allow actor to manage updates to asset's documentation. | +| Manage Dataset Column Tag Proposals[^1] | Allow actor to manage a proposal to add a tag to dataset schema field (column). | +| Manage Dataset Column Glossary Term Proposals[^1] | Allow actor to manage a proposal to add a glossary term to dataset schema field (column). | +| Manage Dataset Column Property Proposals[^1] | Allow actor to manage a proposal to add a structured property to dataset schema field (column). | + +### Specific Entity-level Privileges + +These privileges are not generalizable. + +#### Users & Groups + +| Entity | Privilege | Description | +| ------ | -------------------------------------- | ------------------------------------------------------------------------------------------------ | +| Group | Edit Group Members | Allow actor to add and remove members to a group. | +| Group | Manage Group Notification Settings[^1] | Allow actor to manage notification settings for a group. | +| Group | Manage Group Subscriptions[^1] | Allow actor to manage subscriptions for a group. | +| Group | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | +| User | Edit Contact Information | Allow actor to change the contact information such as email & chat handles. | +| User | Edit User Profile | Allow actor to change the user's profile including display name, bio, title, profile image, etc. | + +#### Dataset + +| Entity | Privilege | Description | +| ------------ | ----------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Dataset | View Dataset Usage | Allow actor to access dataset usage information (includes usage statistics and queries). | +| Dataset | View Dataset Profile | Allow actor to access dataset profile (snapshot statistics) | +| Dataset | Edit Dataset Column Descriptions | Allow actor to edit the column (field) descriptions associated with a dataset schema. | +| Dataset | Edit Dataset Column Tags | Allow actor to edit the column (field) tags associated with a dataset schema. | +| Dataset | Edit Dataset Column Glossary Terms | Allow actor to edit the column (field) glossary terms associated with a dataset schema. | +| Dataset | Edit Dataset Column Properties | Allow actor to edit the column (field) properties associated with a dataset schema. | +| Dataset | Propose Dataset Column Glossary Terms[^1] | Allow actor to propose column (field) glossary terms associated with a dataset schema. | +| Dataset | Propose Dataset Column Tags[^1] | Allow actor to propose new column (field) tags associated with a dataset schema. | +| Dataset | Manage Dataset Column Glossary Terms[^1] | Allow actor to manage column (field) glossary term proposals associated with a dataset schema. | +| Dataset | Propose Dataset Column Descriptions[^1] | Allow actor to propose new descriptions associated with a dataset schema. | +| Dataset | Manage Dataset Column Tag Proposals[^1] | Allow actor to manage column (field) tag proposals associated with a dataset schema. | +| Dataset | Edit Assertions | Allow actor to add and remove assertions from an entity. | +| Dataset | Edit Dataset Queries | Allow actor to edit the Queries for a Dataset. | +| Dataset | View Dataset Operations | Allow actor to view operations on a Dataset. | +| Dataset | Create erModelRelationship | Allow actor to add erModelRelationship on a dataset. | +| Dataset | Edit Monitors[^1] | Allow actor to edit monitors for the entity. | +| Dataset | Edit SQL Assertion Monitors[^1] | Allow actor to edit custom SQL assertion monitors for the entity. Note that this gives read query access to users with through the Custom SQL assertion builder. Grant with care. | +| Dataset | Edit Data Contract[^1] | Allow actor to edit the Data Contract for an entity. | +| Dataset | Manage Data Contract Proposals[^1] | Allow actor to manage a proposal for a Data Contract | +| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | +| Domain | Manage Data Products | Allow actor to create, edit, and delete Data Products within a Domain | +| GlossaryNode | Manage Direct Glossary Children | Allow actor to create and delete the direct children of this entity. | +| GlossaryNode | Manage All Glossary Children | Allow actor to create and delete everything underneath this entity. | + +#### Misc + +| Entity | Privilege | Description | +| ------------ | ------------------------------- | --------------------------------------------------------------------- | +| Tag | Edit Tag Color | Allow actor to change the color of a Tag. | +| Domain | Manage Data Products | Allow actor to create, edit, and delete Data Products within a Domain | +| GlossaryNode | Manage Direct Glossary Children | Allow actor to create and delete the direct children of this entity. | +| GlossaryNode | Manage All Glossary Children | Allow actor to create and delete everything underneath this entity. | + +## Coming Soon + +### Experimental + +Support for Policy Constraints based on entity sub-resources (tags, glossary terms, domains, containers, etc.) is currently in development and in an experimental phase. + +Currently the only supported sub-resources are tags. These are supported through an additional parameter in DataHubPolicyInfo which is currently only modifiable via API, there is no UI option to configure it. Specifically the +option is `privilegeConstraints` which takes a `PolicyMatchFilter` within the existing `DataHubResourceFilter` for a policy. This works similarly to the existing resource filter, but instead of applying to the main entity being acted on +it applies to the subResource targeted in the action. For example, if the policy specifies it is constrained to tags that equal `urn:li:tag:tag1` or `urn:li:tag:tag2` for `EDIT_DATASET_TAGS` privilege, then assuming no other policies match, +a user would only be able to apply those tags to the dataset. This is also supported with the `NOT_EQUALS` condition for preventing certain tags from being added/removed. These policies apply by default in the UI and can be configured to apply +to API operations as well through the `MCP_VALIDATION_PRIVILEGE_CONSTRAINTS` environment variable which should be applied globally (GMS, MCE Consumer, and DataHub Upgrade specifically), which is enabled by default. + +Example JSON of a policy with constraints: + +```json +{ + "actors": { + "resourceOwners": false, + "groups": [], + "allGroups": false, + "allUsers": false, + "users": ["urn:li:corpuser:ryan@email.com"] + }, + "lastUpdatedTimestamp": 0, + "privileges": ["EDIT_ENTITY_TAGS", "EDIT_DATASET_COL_TAGS"], + "editable": true, + "displayName": "Ryan Policy", + "resources": { + "filter": { "criteria": [] }, + "allResources": false, + "privilegeConstraints": { + "criteria": [ + { + "field": "URN", + "condition": "EQUALS", + "values": ["urn:li:tag:PII", "urn:li:tag:Business Critical"] + } + ] + } + }, + "description": "", + "state": "ACTIVE", + "type": "METADATA" +} +``` + +```graphql +mutation { + createPolicy( + input: { + type: METADATA + name: "my-policy" + state: ACTIVE + description: "My policy" + privileges: ["EDIT_ENTITY_TAGS"] + actors: { + allUsers: true + users: [] + groups: [] + resourceOwners: true + allGroups: true + } + resources: { + allResources: true + resources: [] + filter: { criteria: [] } + policyConstraints: { + criteria: [ + { + field: "URN" + values: ["urn:li:tag:PII", "urn:li:tag:Business Critical"] + condition: EQUALS + } + ] + } + } + } + ) +} +``` + +## Feedback / Questions / Concerns + +We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on Slack! + +[^3]: Only active if REST_API_AUTHORIZATION_ENABLED is true + +[^1]: DataHub Cloud only + +[^2]: Deprecated feature diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/authorization/roles.md b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/roles.md new file mode 100644 index 00000000..c3884d7c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/authorization/roles.md @@ -0,0 +1,208 @@ +--- +title: Roles +slug: /authorization/roles +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/authorization/roles.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Roles + + + +DataHub provides the ability to use **Roles** to manage permissions. + +:::tip **Roles** are the recommended way to manage permissions on DataHub. This should suffice for most use cases, but advanced users can use **Policies** if needed. + +## Roles Setup, Prerequisites, and Permissions + +The out-of-the-box Roles represent the most common types of DataHub users. Currently, the supported Roles are **Admin**, **Editor** and **Reader**. + +| Role Name | Description | +| --------- | --------------------------------------------------------------------------------------- | +| Admin | Can do everything on the platform. | +| Editor | Can read and edit all metadata. Cannot take administrative actions. | +| Reader | Can read all metadata. Cannot edit anything by default, or take administrative actions. | + +:::note To manage roles, including viewing roles, or editing a user's role, you must either be an **Admin**, or have the **Manage Policies** privilege. + +## Using Roles + +### Viewing Roles + +You can view the list of existing Roles under **Settings > Permissions > Roles**. You can click into a Role to see details about +it, like which users have that Role, and which Policies correspond to that Role. + +

+ +

+ +### Assigning Roles + +Roles can be assigned in two different ways. + +#### Assigning a New Role to a Single User + +If you go to **Settings > Users & Groups > Users**, you will be able to view your full list of users, as well as which Role they are currently +assigned to, including if they don't have a Role. + +

+ +

+ +You can simply assign a new Role to a user by clicking on the drop-down that appears on their row and selecting the desired Role. + +

+ +

+ +#### Batch Assigning a Role + +When viewing the full list of roles at **Settings > Permissions > Roles**, you will notice that each role has an `Add Users` button next to it. Clicking this button will +lead you to a search box where you can search through your users, and select which users you would like to assign this role to. + +

+ +

+ +### How do Roles interact with Policies? + +Roles actually use Policies under-the-hood, and come prepackaged with corresponding policies to control what a Role can do, which you can view in the +Policies tab. Note that these Role-specific policies **cannot** be changed. You can find the full list of policies corresponding to each Role at the bottom of this +[file](https://github.com/datahub-project/datahub/blob/master/metadata-service/war/src/main/resources/boot/policies.json). + +If you would like to have finer control over what a user on your DataHub instance can do, the Roles system interfaces cleanly +with the Policies system. For example, if you would like to give a user a **Reader** role, but also allow them to edit metadata +for certain domains, you can add a policy that will allow them to do. Note that adding a policy like this will only add to what a user can do +in DataHub. + +### Role Privileges + +#### Self-Hosted DataHub and DataHub Cloud + +These privileges are common to both Self-Hosted DataHub and DataHub Cloud. + +##### Platform Privileges + +| Privilege | Admin | Editor | Reader | Description | +| ----------------------------------------- | ------------------ | ------------------ | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Generate Personal Access Tokens | :heavy_check_mark: | :heavy_check_mark: | :x: | Generate personal access tokens for use with DataHub APIs. | +| Manage Domains | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and remove Asset Domains. | +| Manage Home Page Posts | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and delete home page posts | +| Manage Glossaries | :heavy_check_mark: | :heavy_check_mark: | :x: | Create, edit, and remove Glossary Entities | +| Manage Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and remove Tags. | +| Manage Business Attribute | :heavy_check_mark: | :heavy_check_mark: | :x: | Create, update, delete Business Attribute | +| Manage Documentation Forms | :heavy_check_mark: | :heavy_check_mark: | :x: | Manage forms assigned to assets to assist in documentation efforts. | +| Manage Policies | :heavy_check_mark: | :x: | :x: | Create and remove access control policies. Be careful - Actors with this privilege are effectively super users. | +| Manage Metadata Ingestion | :heavy_check_mark: | :x: | :x: | Create, remove, and update Metadata Ingestion sources. | +| Manage Secrets | :heavy_check_mark: | :x: | :x: | Create & remove Secrets stored inside DataHub. | +| Manage Users & Groups | :heavy_check_mark: | :x: | :x: | Create, remove, and update users and groups on DataHub. | +| View Analytics | :heavy_check_mark: | :x: | :x: | View the DataHub analytics dashboard. | +| Manage All Access Tokens | :heavy_check_mark: | :x: | :x: | Create, list and revoke access tokens on behalf of users in DataHub. Be careful - Actors with this privilege are effectively super users that can impersonate other users. | +| Manage User Credentials | :heavy_check_mark: | :x: | :x: | Manage credentials for native DataHub users, including inviting new users and resetting passwords | +| Manage Public Views | :heavy_check_mark: | :x: | :x: | Create, update, and delete any Public (shared) Views. | +| Manage Ownership Types | :heavy_check_mark: | :x: | :x: | Create, update and delete Ownership Types. | +| Create Business Attribute | :heavy_check_mark: | :x: | :x: | Create new Business Attribute. | +| Manage Connections | :heavy_check_mark: | :x: | :x: | Manage connections to external DataHub platforms. | +| Restore Indices API | :heavy_check_mark: | :x: | :x: | The ability to use the Restore Indices API. | +| Get Timeseries index sizes API | :heavy_check_mark: | :x: | :x: | The ability to use the get Timeseries indices size API. | +| Truncate timeseries aspect index size API | :heavy_check_mark: | :x: | :x: | The ability to use the API to truncate a timeseries index. | +| Get ES task status API | :heavy_check_mark: | :x: | :x: | The ability to use the get task status API for an ElasticSearch task. | +| Enable/Disable Writeability API | :heavy_check_mark: | :x: | :x: | The ability to enable or disable GMS writeability for data migrations. | +| Apply Retention API | :heavy_check_mark: | :x: | :x: | The ability to apply retention using the API. | +| Analytics API access | :heavy_check_mark: | :x: | :x: | API read access to raw analytics data. | + +##### Metadata Privileges + +| Privilege | Admin | Editor | Reader | Description | +| ---------------------------------- | ------------------ | ------------------ | ------------------ | ------------------------------------------------------------------------------------------------ | +| View Entity Page | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to view the entity page. | +| View Dataset Usage | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to access dataset usage information (includes usage statistics and queries). | +| View Dataset Profile | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to access dataset profile (snapshot statistics) | +| Edit Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove tags to an asset. | +| Edit Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove glossary terms to an asset. | +| Edit Description | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the description (documentation) of an entity. | +| Edit Links | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit links associated with an entity. | +| Edit Status | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the status of an entity (soft deleted or not). | +| Edit Domain | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Domain of an entity. | +| Edit Data Product | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Data Product of an entity. | +| Edit Deprecation | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Deprecation status of an entity. | +| Edit Assertions | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove assertions from an entity. | +| Edit Incidents | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to create and remove incidents for an entity. | +| Edit Entity | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit any information about an entity. Super user privileges for the entity. | +| Edit Dataset Column Tags | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the column (field) tags associated with a dataset schema. | +| Edit Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the column (field) glossary terms associated with a dataset schema. | +| Edit Dataset Column Descriptions | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the column (field) descriptions associated with a dataset schema. | +| Edit Tag Color | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to change the color of a Tag. | +| Edit Lineage | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to add and remove lineage edges for this entity. | +| Edit Dataset Queries | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the Queries for a Dataset. | +| Execute Entity | :heavy_check_mark: | :x: | :x: | The ability to execute ingestion for an Entity. | +| Manage Data Products | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to create, edit, and delete Data Products within a Domain | +| Edit Properties | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to edit the properties for an entity. | +| Edit Owners | :heavy_check_mark: | :x: | :x: | The ability to add and remove owners of an entity. | +| Edit Group Members | :heavy_check_mark: | :x: | :x: | The ability to add and remove members to a group. | +| Edit User Profile | :heavy_check_mark: | :x: | :x: | The ability to change the user's profile including display name, bio, title, profile image, etc. | +| Edit Contact Information | :heavy_check_mark: | :x: | :x: | The ability to change the contact information such as email & chat handles. | +| Delete | :heavy_check_mark: | :x: | :x: | The ability to delete this entity. | +| Search API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to access search APIs. | +| Get Aspect/Entity Count APIs | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Aspect/Entity Count APIs. | +| Get Timeseries Aspect API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Timeseries Aspect API. | +| Get Entity + Relationships API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Entity and Relationships API. | +| Get Timeline API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the GET Timeline API. | +| Explain ElasticSearch Query API | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to use the Operations API explain endpoint. | +| Produce Platform Event API | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to produce Platform Events using the API. | + +#### DataHub Cloud + +These privileges are only relevant to DataHub Cloud. + +##### Platform Privileges + +| Privilege | Admin | Editor | Reader | Description | +| ------------------------------- | ------------------ | ------------------ | ------ | --------------------------------------------------------------------------------------------------- | +| Manage Tests | :heavy_check_mark: | :heavy_check_mark: | :x: | Create and remove Asset Tests. | +| View Metadata Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | View the requests tab for viewing metadata proposals. | +| Create metadata constraints[^1] | :heavy_check_mark: | :heavy_check_mark: | :x: | Create metadata constraints. | +| Manage Platform Settings | :heavy_check_mark: | :x: | :x: | View and change platform-level settings, like integrations & notifications. | +| Manage Monitors | :heavy_check_mark: | :x: | :x: | Create, update, and delete any data asset monitors, including Custom SQL monitors. Grant with care. | + +[^1]: Deprecated feature + +##### Metadata Privileges + +| Privilege | Admin | Editor | Reader | Description | +| ------------------------------------- | ------------------ | ------------------ | ------------------ | ---------------------------------------------------------------------------------------------- | +| View Entity | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to view the entity in search results. | +| Propose Tags | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose adding a tag to an asset. | +| Propose Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose adding a glossary term to an asset. | +| Propose Documentation | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose updates to an asset's documentation. | +| Propose Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose column (field) glossary terms associated with a dataset schema. | +| Propose Dataset Column Tags | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | The ability to propose new column (field) tags associated with a dataset schema. | +| Manage Tag Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal to add a tag to an asset. | +| Manage Glossary Term Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal to add a glossary term to an asset. | +| Manage Dataset Column Glossary Terms | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage column (field) glossary term proposals associated with a dataset schema. | +| Manage Dataset Column Tag Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage column (field) tag proposals associated with a dataset schema. | +| Manage Documentation Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal update an asset's documentation | +| Manage Group Notification Settings | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage notification settings for a group. | +| Manage Group Subscriptions | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage subscriptions for a group. | +| Manage User Subscriptions | :heavy_check_mark: | :x: | :x: | The ability to manage subscriptions for another user. | +| Manage Data Contract Proposals | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to manage a proposal for a Data Contract | +| Share Entity | :heavy_check_mark: | :heavy_check_mark: | :x: | The ability to share an entity with another DataHub Cloud instance. | + +## Additional Resources + +### GraphQL + +- [acceptRole](../../graphql/mutations.md#acceptrole) +- [batchAssignRole](../../graphql/mutations.md#batchassignrole) +- [listRoles](../../graphql/queries.md#listroles) + +## FAQ and Troubleshooting + +## What updates are planned for Roles? + +In the future, the DataHub team is looking into adding the following features to Roles. + +- Defining a role mapping from OIDC identity providers to DataHub that will grant users a DataHub role based on their IdP role +- Allowing Admins to set a default role on DataHub so all users are assigned a role +- Building custom roles diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-docs.md b/docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-docs.md new file mode 100644 index 00000000..a6bb508c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-docs.md @@ -0,0 +1,63 @@ +--- +title: AI Documentation +slug: /automations/ai-docs +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/automations/ai-docs.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# AI Documentation + + + +With AI-powered documentation, you can automatically generate documentation for tables and columns. + +

+ +

+ +## Prerequisites + +As of DataHub Cloud v0.3.17, AI documentation is **Generally Available**. Admins (or users with the "Manage Platform Settings" privilege) can enable it from settings. + +

+ +

+ +## Usage + +Ensure you have permissions to edit the dataset description. No other configuration is required - just hit "Generate" on any table or column in the UI. + +All AI-generated documentation that has not been reviewed by a human will be marked as such with the sparkle icon. + +

+ +

+ +### Customize Documentation Generation + +As of v0.3.15, you can customize how documentation is generated by providing custom instructions that are passed to the underlying AI model when generating dcumentation for any Table or Column. This is useful if you want AI-generated documentation to follow specific guidelines or standards set by your organization. + +To provide custom instructions for documentation generation, start by navigating to **Settings > AI**. Then simply provide custom instructions in the **AI Documentation > Instructions** input. + +

+ +

+ +Note that after updating instructions, it may take up to 5 minutes for the new instructions to take effect. + +## How it works + +Generating good documentation requires a holistic understanding of the data. Information we take into account includes, but is not limited to: + +- Dataset name and any existing documentation +- Column name, type, description, and sample values +- Lineage relationships to upstream and downstream assets +- Metadata about other related assets + +Data privacy: Your metadata is not sent to any third-party LLMs. We use AWS Bedrock internally, which means all metadata remains within the DataHub Cloud AWS account. We do not fine-tune on customer data. + +## Limitations + +- AI documentation is not available for tables with more than 3000 columns (in v0.3.12 the limit was 1000 columns; prior to v0.3.12, it was 100 columns). +- This feature is powered by LLMs, which can produce inaccurate results. While we've taken steps to reduce the likelihood of hallucinations, they may still occur. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-term-suggestion.md b/docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-term-suggestion.md new file mode 100644 index 00000000..ad1f577a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/automations/ai-term-suggestion.md @@ -0,0 +1,90 @@ +--- +title: AI Glossary Term Suggestions +slug: /automations/ai-term-suggestion +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/automations/ai-term-suggestion.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# AI Glossary Term Suggestions + + + +:::info + +This feature is currently in closed beta. Reach out to your DataHub Cloud representative to get access. + +::: + +The AI Glossary Term Suggestion automation uses LLMs to suggest [Glossary Terms](../glossary/business-glossary.md) for tables and columns in your data. + +This is useful for improving coverage of glossary terms across your organization, which is important for compliance and governance efforts. + +This automation can: + +- Automatically suggests glossary terms for tables and columns. +- Goes beyond a predefined set of terms and works with your business glossary. +- Generates [proposals](../managed-datahub/change-proposals.md) for owners to review, or can automatically add terms to tables/columns. +- Automatically adjusts to human-provided feedback and curation (coming soon). + +## Prerequisites + +- A business glossary with terms defined. Additional metadata, like documentation and existing term assignments, will improve the accuracy of our suggestions. + +## Usage + +1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create the Automation**: Click on 'Create' and select 'AI Glossary Term Suggestions'. + +

+ +

+ +3. **Configure the Automation**: Fill in the required fields to configure the automation. + The main fields to configure are (1) what terms to use for suggestions and (2) what entities to generate suggestions for. + +

+ +

+ +4. Once it's enabled, that's it! You'll start to see terms show up in the UI, either on assets or in the proposals page. + +

+ +

+ +### Customize AI Classification Generation + +As of v0.3.15, you can customize how classification recommendations are generated by providing custom instructions that are passed to the underlying AI model when generating classification decisions for any Table or Column. This is useful if you want AI-generated classification to follow specific guidelines or standards set by your organization. + +To provide custom instructions for AI classification generation, simply provide them in the Custom Instruction input when creating the AI classification automation. + +

+ +

+ +Note that after updating instructions, it may take up to 5 minutes for the new instructions to take effect. + +## How it works + +The automation will scan through all the datasets matched by the configured filters. For each one, it will generate suggestions. +If new entities are added that match the configured filters, those will also be classified within 24 hours. + +We take into account the following metadata when generating suggestions: + +- Dataset name and description +- Column name, type, description, and sample values +- Glossary term name, documentation, and hierarchy +- Feedback loop: existing assignments and accepted/rejected proposals (coming soon) + +Data privacy: Your metadata is not sent to any third-party LLMs. We use AWS Bedrock internally, which means all metadata remains within the DataHub Cloud AWS account. We do not fine-tune on customer data. + +## Limitations + +- A single configured automation can classify at most 10k entities. +- We cannot do partial reclassification. If you add a new column to an existing table, we won't regenerate suggestions for that table. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/automations/bigquery-metadata-sync.md b/docs-archive/versioned_docs/version-1.5.0/docs/automations/bigquery-metadata-sync.md new file mode 100644 index 00000000..f930a804 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/automations/bigquery-metadata-sync.md @@ -0,0 +1,198 @@ +--- +title: BigQuery Metadata Sync Automation +slug: /automations/bigquery-metadata-sync +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/automations/bigquery-metadata-sync.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# BigQuery Metadata Sync Automation + + + +:::info + +This feature is currently in open beta in DataHub Cloud. Reach out to your DataHub Cloud representative to get access. + +::: + +## Introduction + +BigQuery Metadata Sync is an automation that synchronizes DataHub Tags, Table and Column descriptions, and Column Glossary Terms with +BigQuery. This automation is exclusively available in DataHub Cloud. + +## Use-Cases + +- Maintain consistent metadata across DataHub and BigQuery +- Improve data discovery by propagating rich descriptions back to BigQuery +- Enhance data governance by applying Policy Tags based on DataHub Glossary Terms +- Streamline data classification by syncing DataHub Tags to BigQuery Labels +- Facilitate compliance efforts by automatically tagging sensitive data columns +- Support data lineage tracking by keeping metadata aligned across platforms + +## Sync Capabilities + +| DataHub Source | BigQuery Target | Sync Direction | Notes | +| --------------------- | ------------------- | ------------------ | -------------------------------------------------------------------------------------------------------- | +| Table Tags | Table Labels | Bi-directional | Changes in either system reflect in both | +| Table Descriptions | Table Descriptions | Bi-directional | Changes in either system reflect in both | +| Column Descriptions | Column Descriptions | Bi-directional | Changes in either system reflect in both.
Thes sync doesn't delete table description from BigQuery | +| Column Glossary Terms | Column Policy Tags | DataHub → BigQuery | Created under DataHub taxonomy | + +## Setup Instructions + +### 1. Verify Permissions + +Ensure your service account has the following permissions: + +| Task | Required Permissions | Available Role | +| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| Policy Tag Management | • `datacatalog.taxonomies.create`
• `datacatalog.taxonomies.update`
• `datacatalog.taxonomies.list`
• `datacatalog.taxonomies.get`
• `bigquery.tables.createTagBinding` | Policy Tag Admin | +| Policy Tag Assignment | • `bigquery.tables.updateTag` | - | +| Description Management | • `bigquery.tables.update` | - | +| Label Management | • `bigquery.tables.update` | - | + +**Note**: `bigquery.tables` permissions must be granted in every project where metadata sync is needed. + +### 2. Enable the Automation + +1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create An Automation**: Click on 'Create' and select 'BigQuery Tag Propagation'. + +

+ +

+ +3. **Configure Automation**: + + 1. **Select a Propagation Action** + +

+ +

+ + | Propagation Type | DataHub Entity | BigQuery Entity | Note | + | ------------------------------------ | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | Table Tags as Labels | [Table Tag](/docs/tags/) | [BigQuery Label](https://cloud.google.com/bigquery/docs/labels-intro) | - | + | Column Glossary Terms as Policy Tags | [Glossary Term on Table Column](/docs/glossary/business-glossary/) | [Policy Tag](https://cloud.google.com/bigquery/docs/best-practices-policy-tags) |
  • Assigned Policy tags are created under DataHub taxonomy.
  • Only the latest assigned glossary term set as policy tag. BigQuery only supports one assigned policy tag.
  • Policy Tags are not synced to DataHub as glossary term from BigQuery.
| + | Table Descriptions | [Table Description](/docs/api/tutorials/descriptions/) | Table Description | - | + | Column Descriptions | [Column Description](/docs/api/tutorials/descriptions/) | Column Description | - | + + :::note + + You can limit propagation based on specific Tags and Glossary Terms. If none are selected, ALL Tags or Glossary Terms will be automatically propagated to BigQuery tables and columns. (The recommended approach is to not specify a filter to avoid inconsistent states.) + + ::: + + :::note + + - BigQuery supports only one Policy Tag per table field. Consequently, the most recently assigned Glossary Term will be set as the Policy Tag for that field. + - Policy Tags cannot be applied to fields in External tables. Therefore, if a Glossary Term is assigned to a field in an External table, it will not be applied. + + ::: + + 2. **Fill in the required fields to connect to BigQuery, along with the name, description, and category** + +

+ +

+ + 3. **Finally, click 'Save and Run' to start the automation** + +## 3. Propagating for Existing Assets (Optional) + +To ensure that all existing table Tags and Column Glossary Terms are propagated to BigQuery, you can back-fill historical data for existing assets. Note that the initial back-filling process may take some time, depending on the number of BigQuery assets you have. + +To do so, follow these steps: + +1. Navigate to the Automation you created in Step 3 above +2. Click the 3-dot "More" menu + +

+ +

+ +3. Click "Initialize" + +

+ +

+ +This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating descriptions going forward, you can skip this step. + +## Viewing Propagated Tags + +You can view propagated Tags inside the BigQuery UI to confirm the automation is working as expected. + +

+ +

+ +## Troubleshooting BigQuery Propagation + +### Q: What metadata elements support bi-directional syncing between DataHub and BigQuery? + +A: The following metadata elements support bi-directional syncing: + +- Tags (via BigQuery Labels): Changes made in either DataHub Table Tags or BigQuery Table Labels will be reflected in the other system. +- Descriptions: Both table and column descriptions are synced bi-directionally. + +### Q: Are Policy Tags bi-directionally synced? + +A: No, BigQuery Policy Tags are only propagated from DataHub to BigQuery, not vice versa. This means that Policy Tags should be mastered in DataHub using the [Business Glossary](/docs/glossary/business-glossary/). + +It is recommended to avoid enabling `extract_policy_tags_from_catalog` during +ingestion, as this will ingest policy tags as BigQuery labels. Our sync process +propagates Glossary Term assignments to BigQuery as Policy Tags. + +In a future release, we plan to remove this restriction to support full bi-directional syncing. + +### Q: What metadata is synced from BigQuery to DataHub during ingestion? + +A: During ingestion from BigQuery: + +- Tags and descriptions from BigQuery will be ingested into DataHub. +- Existing Policy Tags in BigQuery will not overwrite or create Business Glossary Terms in DataHub. It only syncs assigned column Glossary Terms from DataHub to BigQuery. + +### Q: Where should I manage my Business Glossary? + +A: The expectation is that you author and manage the glossary in DataHub. Policy tags in BigQuery should be treated as a reflection of the DataHub glossary, not as the primary source of truth. + +### Q: Are there any limitations with Policy Tags in BigQuery? + +A: Yes, BigQuery only supports one Policy Tag per column. If multiple glossary +terms are assigned to a column in DataHub, only the most recently assigned term +will be set as the policy tag in BigQuery. To reduce the scope of conflicts, you +can set up filters in the BigQuery Metadata Sync to only synchronize terms from +a specific area of the Business Glossary. + +### Q: How frequently are changes synced between DataHub and BigQuery? + +A: From DataHub to BigQuery, the sync happens instantly (within a few seconds) +when the change occurs in DataHub. + +From BigQuery to DataHub, changes are synced when ingestion occurs, and the frequency depends on your custom ingestion schedule. (Visible on the **Integrations** page) + +### Q: What happens if there's a conflict between DataHub and BigQuery metadata? + +A: In case of conflicts (e.g., a tag is modified in both systems between syncs), the DataHub version will typically take precedence. However, it's best to make changes in one system consistently to avoid potential conflicts. + +### Q: What permissions are required for bi-directional syncing? + +A: Ensure that the service account used for the automation has the necessary permissions in both DataHub and BigQuery to read and write metadata. See the required BigQuery permissions at the top of the page. + +### Q: Can table description removed? + +No, the sync can only modify table description but it won't remove or clear a description from a table. + +## Related Documentation + +- [DataHub Tags Documentation](/docs/tags/) +- [DataHub Glossary Documentation](/docs/glossary/business-glossary/) +- [BigQuery Labels Documentation](https://cloud.google.com/bigquery/docs/labels-intro) +- [BigQuery Policy Tags Documentation](https://cloud.google.com/bigquery/docs/best-practices-policy-tags) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/automations/databricks-metadata-sync.md b/docs-archive/versioned_docs/version-1.5.0/docs/automations/databricks-metadata-sync.md new file mode 100644 index 00000000..ba76aea6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/automations/databricks-metadata-sync.md @@ -0,0 +1,286 @@ +--- +title: Databricks Metadata Sync Automation +slug: /automations/databricks-metadata-sync +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/automations/databricks-metadata-sync.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Databricks Metadata Sync Automation + + + +:::info + +This feature is currently in **Public Beta** in DataHub Cloud. Reach out to your DataHub Cloud representative if you face any issues configuring or validating the capabilities outlined below. + +::: + +## Overview + +Databricks Metadata Sync is an automation feature that enables seamless synchronization of DataHub Tags and Descriptions with Databricks Unity Catalog. This automation ensures consistent metadata governance across both platforms, automatically propagating DataHub governance artifacts to Unity Catalog tables, columns, catalogs, and schemas. Typically, this will be used in conjunction with the [Databricks ingestion source](/docs/generated/ingestion/sources/databricks), which enables ingesting Tags & descriptions from Databricks into DataHub. + +This automation is exclusively available in DataHub Cloud. + +## Use Cases + +- Maintain consistent metadata across DataHub and Databricks +- Improve data discovery by propagating descriptions back to Databricks +- Unity data governance by managing Tag application directly within DataHub + +## Sync Capabilities + +The Databricks Metadata Sync automation provides comprehensive metadata synchronization with the following features: + +- **Automated Tag Propagation**: Seamlessly sync DataHub Tags to Unity Catalog tables, columns, catalogs, and schemas +- **Description Synchronization**: Automatically propagate DataHub descriptions to Unity Catalog objects as comments +- **Bidirectional Updates**: Maintain consistency by automatically removing Tags and descriptions from Unity Catalog when they are removed in DataHub +- **Selective Propagation**: Configure specific Tags for propagation, or sync all Tags +- **Historical Backfill**: Initialize Tags and Descriptions for assets on Databricks with current DataHub Tags & Descriptions. + +> **A note about legacy Hive Metastore**: Bi-directional sync for _descriptions_ is supported for Hive Metastore Schemas & Tables, but Tag sync is _not_. This is because Databricks does not support applying of Tags to these assets on Hive Metastore. + +## Prerequisites + +Before enabling Databricks Metadata Sync, ensure the following permissions and configurations are in place: + +### Required Unity Catalog Permissions + +#### Basic Access Permissions (Required for Both Tags and Descriptions) + +- **USE CATALOG**: Access to the Unity Catalog containing target objects +- **USE SCHEMA**: Permission to access schemas within the catalog + +#### Permissions for Tag Synchronization + +Based on Unity Catalog requirements, to add tags to objects you need: + +- **APPLY TAG**: Required on each object where tags will be applied (catalogs, schemas, tables, columns) +- **USE SCHEMA**: Required on the object's parent schema (already covered in basic access) +- **USE CATALOG**: Required on the object's parent catalog (already covered in basic access) + +_Note: If using governed tags, you may also need ASSIGN permission on the tag policy._ + +#### Permissions for Description Synchronization + +- **MODIFY**: Required to update comments/descriptions on Unity Catalog objects (catalogs, schemas, tables, columns) + +### Example Permission Configuration + +Configure the necessary permissions for your DataHub automation service principal based on your sync requirements: + +#### For Tags Only + +```sql +-- Basic access permissions +GRANT USE CATALOG ON CATALOG your_catalog TO `datahub-automation@your-domain.com`; +GRANT USE SCHEMA ON SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; + +-- Tag application permissions +GRANT APPLY TAG ON CATALOG your_catalog TO `datahub-automation@your-domain.com`; +GRANT APPLY TAG ON SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; +GRANT APPLY TAG ON ALL TABLES IN SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; +``` + +#### For Descriptions Only + +```sql +-- Basic access permissions +GRANT USE CATALOG ON CATALOG your_catalog TO `datahub-automation@your-domain.com`; +GRANT USE SCHEMA ON SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; + +-- Description modification permissions +GRANT MODIFY ON CATALOG your_catalog TO `datahub-automation@your-domain.com`; +GRANT MODIFY ON SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; +GRANT MODIFY ON ALL TABLES IN SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; +``` + +#### For Both Tags and Descriptions + +```sql +-- Basic access permissions +GRANT USE CATALOG ON CATALOG your_catalog TO `datahub-automation@your-domain.com`; +GRANT USE SCHEMA ON SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; + +-- Tag application permissions +GRANT APPLY TAG ON CATALOG your_catalog TO `datahub-automation@your-domain.com`; +GRANT APPLY TAG ON SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; +GRANT APPLY TAG ON ALL TABLES IN SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; + +-- Description modification permissions +GRANT MODIFY ON CATALOG your_catalog TO `datahub-automation@your-domain.com`; +GRANT MODIFY ON SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; +GRANT MODIFY ON ALL TABLES IN SCHEMA your_catalog.your_schema TO `datahub-automation@your-domain.com`; +``` + +### Connection Requirements + +Ensure your DataHub instance has: + +- Valid Databricks workspace credentials +- Network connectivity to your Databricks Unity Catalog environment +- Appropriate service principal or user authentication configured +- Databricks warehouse id for executing operations + +## Configuration Guide + +### Step 1: Access Automations + +Navigate to the Automations section in your DataHub Cloud interface: + +1. Select **Automations** from the dropdown menu, Under the **Govern** section + +

+ Navigate to Automations +

+ +### Step 2: Create Databricks Automation + +Configure the automation: + +1. Click the **Create** button +2. Select **Databricks Metadata Sync** from the available automation types + +

+ Select Databricks Metadata Sync +

+ +### Step 3: Configure Sync Options + +Choose the types of information to synchronize: + +#### Select Action + +Choose between: + +- **Tags**: Sync Tags for Tables, Columns, Catalogs, & Schemas (Unity Catalog only) +- **Descriptions**: Sync descriptions for Tables, Columns, Catalogs & Schemas as comments (Unity Catalog & legacy Hive Metastore) + +

+ Select Sync Action +

+ +#### Configure Tag Selection (if Tags selected) + +When syncing Tags, you can choose: + +- **All Tags**: Propagate all DataHub Tags to Unity Catalog +- **Specific Tags**: Select only specific Tags for synchronization + +### Step 4: Configure Connection Settings + +Complete the Databricks connection configuration: + +#### Required Connection Details + +- **Workspace URL**: Your Databricks workspace URL (e.g., `https://abcsales.cloud.databricks.com`) +- **Warehouse ID**: The SQL warehouse ID used for executing for metadata operations (e.g., `fab3e5fg0bcbfc56`) +- **Token**: Databricks personal access token or service principal token + +

+ Connection Configuration +

+ +#### Test Connection + +Click **Test Connection** to verify your configuration before proceeding. + +### Step 5: Configure Automation Details + +Provide automation metadata: + +- **Name**: Descriptive name for your automation (e.g., "Databricks Metadata Sync") +- **Description**: Details about the automation's purpose and scope +- **Category**: Select an appropriate category for organization + +Click **Save and Run** to activate the automation and begin real-time synchronization. + +## Historical Data Synchronization + +### Initializing Existing Assets + +For environments with existing DataHub metadata, you can perform a one-time backfill to ensure all current Tags and Descriptions from DataHub are propagated to Unity Catalog. Depending on the number of assets, this might take a while! + +#### Initialization Process + +1. Navigate to your created Databricks Metadata Sync automation +2. Click the three-dot **More** menu next to the automation + +

+ Automation More Menu +

+ +3. Select **Initialize** from the dropdown menu + +

+ Initialize Automation +

+ +:::note Initialization Timeline + +The initialization process duration depends on the volume of Unity Catalog assets in your environment. Large catalogs with extensive metadata may require significant processing time. + +::: + +## Validating the Integration + +### Viewing Synced Metadata + +Confirm successful metadata syncing by examining Unity Catalog objects: + +1. **Access Databricks UI**: Navigate to your Databricks workspace +2. **Browse Catalog**: Open the Unity Catalog explorer +3. **Inspect Objects**: Select tables or columns to view applied tags and comments + +## Troubleshooting + +### Common Issues and Solutions + +#### Permission Errors + +- Verify service principal has all required Unity Catalog permissions +- Confirm catalog and schema access rights +- Check tag creation and application privileges + +#### Connection Issues + +- Validate Databricks workspace URL format +- Ensure access token is valid and not expired +- Verify warehouse ID is correct and accessible +- Check network connectivity between DataHub and Databricks + +#### Synchronization Failures + +- Check Unity Catalog object permissions +- Verify target objects exist and are accessible +- Ensure warehouse is running and available + +### Support Resources + +For additional assistance with Databricks Metadata Sync, contact your DataHub Cloud representative. + +## FAQ + +1. **Where should I manage Tags & Descriptions?** + +In general, we recommend centrally authoring Tags and Descriptions within DataHub. This allows you to maintain a clear and consistent governance posture across _all_ of your data sources and data products - there is always data outside of Databricks! Authoring this critical information in DataHub also improves the experience for your data practicioners trying to find the right data. + +This automation is intended to enable this style of management, allowing you to "push down" metadata from the central catalog into Databricks, where your data is stored and queried. + +2. **How does DataHub represent key-value tags from Databricks?** + +During ingestion from [Databricks](/docs/generated/ingestion/sources/databricks), DataHub can ingest tags and descriptions that were originally authored within Databricks. DataHub converts key-value formatted tags in Databricks into DataHub tags of the format: `key:value`. For example, if you have a tag with key `has_pii` and value `true` in Databricks, this will be ingested as a single combined tag named `has_pii: true` in DataHub. + +After ingestion into DataHub, you can apply this tag to tables or columns and sync it back to Databricks using this automation. Any tag with the format `key:value` that is applied on DataHub will be synced back to Databricks in proper key, value form. + +If you apply a tag without a separator colon in DataHub (e.g. `has_pii`), it will be synced back to Databricks with the key being `has_pii` and value being empty. + +3. **I updated a table description in _Databricks_, but I don't see it reflecting after ingestion into DataHub. Why not?** + +This is usually because you've already overridden the description inside DataHub for this table. DataHub assumes that _it_ will be the source of truth for documentation, which means that any edits that have taken place in the DataHub UI (or via API) will take precedent over changes provided in Databricks. When you change the description in DataHub, the description change will overwrite the latest description in Databricks if this automation is enabled. + +But fear not - you can always view the original underlying Databricks description underneath the DataHub description in the DataHub UI, even when it changes. + +4. **Can I sync DataHub Structured Properties or Glossary Terms back to Databricks as Tags?** + +Currently, no. Sync back is limited to Tags, to keep the concepts aligned more simply across both platforms. Reach out if you'd benefit from this capability! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/automations/docs-propagation.md b/docs-archive/versioned_docs/version-1.5.0/docs/automations/docs-propagation.md new file mode 100644 index 00000000..8bb0b32d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/automations/docs-propagation.md @@ -0,0 +1,119 @@ +--- +title: Documentation Propagation Automation +slug: /automations/docs-propagation +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/automations/docs-propagation.md +--- +# Documentation Propagation Automation + +:::info + +This feature is currently in open beta in DataHub Cloud. Reach out to your DataHub Cloud representative to get access. + +::: + +## Introduction + +Documentation Propagation is an automation automatically propagates column and asset (coming soon) descriptions based on downstream column-level lineage and sibling relationships. +It simplifies metadata management by ensuring consistency and reducing the manual effort required for documenting data assets to aid +in Data Governance & Compliance along with Data Discovery. + +This feature is enabled by default in Open Source DataHub. + +## Capabilities + +### DataHub Core (Open Source) + +- **Column-Level Docs Propagation**: Automatically propagate documentation to downstream columns and sibling columns that are derived or dependent on the source column. +- **(Coming Soon) Asset-Level Docs Propagation**: Propagate descriptions to sibling assets. + +### DataHub Cloud + +- Includes all the features of Open Source. +- **Propagation Rollback (Undo)**: Offers the ability to undo any propagation changes, providing a safety net against accidental updates. +- **Historical Backfilling**: Automatically backfills historical data for newly documented columns to maintain consistency across time. + +### Comparison of Features + +| Feature | Open Source | DataHub Cloud | +| ----------------------------- | ----------- | ------------- | +| Column-Level Docs Propagation | ✔️ | ✔️ | +| Asset-Level Docs Propagation | ✔️ | ✔️ | +| Downstream Lineage + Siblings | ✔️ | ✔️ | +| Historical Backfilling | ❌ | ✔️ | + +## Enabling Documentation Propagation + +### In Open Source + +Notice that the user must have the `Manage Ingestion` permission to view and enable the feature. + +1. **Navigate to Settings**: Click on the 'Settings' gear in top navigation bar. + +

+ +

+ +2. **Navigate to Features**: Click on the 'Features' tab in the left-hand navigation bar. + +

+ +

+ +3. **Enable Documentation Propagation**: Locate the 'Documentation Propagation' section and toggle the feature to enable it for column-level and asset-level propagation. + Currently, Column Level propagation is supported, with asset level propagation coming soon. + +

+ +

+ +### In DataHub Cloud + +1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create An Automation**: Click on 'Create' and select 'Column Documentation Propagation'. + +

+ +

+ +3. **Configure Automation**: Fill in the required fields, such as the name, description, and category. Finally, click 'Save and Run' to start the automation + +

+ +

+ +## Propagating for Existing Assets (DataHub Cloud Only) + +In DataHub Cloud, you can back-fill historical data for existing assets to ensure that all existing column descriptions are propagated to downstreams +when you start the automation. Note that it may take some time to complete the initial back-filling process, depending on the number of assets and the complexity of your lineage. + +To do this, navigate to the Automation you created in Step 3 above, click the 3-dot "more" menu: + +

+ +

+ +and then click "Initialize". + +

+ +

+ +This one-time step will kick off the back-filling process for existing descriptions. If you only want to begin propagating +descriptions going forward, you can skip this step. + +## Viewing Propagated Descriptions + +Once the automation is enabled, you'll be able to recognize propagated descriptions as those with the thunderbolt icon next to them: + +The tooltip will provide additional information, including where the description originated and any intermediate hops that were +used to propagate the description. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/automations/glossary-term-propagation.md b/docs-archive/versioned_docs/version-1.5.0/docs/automations/glossary-term-propagation.md new file mode 100644 index 00000000..e50dcb05 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/automations/glossary-term-propagation.md @@ -0,0 +1,71 @@ +--- +title: Glossary Term Propagation Automation +slug: /automations/glossary-term-propagation +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/automations/glossary-term-propagation.md +--- +# Glossary Term Propagation Automation + + + +:::info + +This feature is currently in open beta in DataHub Cloud. Reach out to your DataHub Cloud representative to get access. + +::: + +## Introduction + +Glossary Term Propagation is an automation feature that propagates classification labels (Glossary Terms) across column and assets based on downstream lineage and sibling relationships. +This automation simplifies metadata management by ensuring consistent term classification and reducing manual effort in categorizing data assets, aiding Data Governance & Compliance, and enhancing Data Discovery. + +## Capabilities + +- **Column-Level Glossary Term Propagation**: Automatically propagate Glossary Terms to all downstream lineage columns and sibling columns. +- **Asset-Level Glossary Term Propagation**: Automatically propagate Glossary Terms to all downstream lineage assets & sibling assets. +- **Select Terms & Term Groups**: Select specific Glossary Terms & Term Groups to propagate, e.g. to propagate only sensitive or important labels. + +Note that Asset-level propagation is currently only support for **Datasets** (Tables, Views, Topics, etc), and not for other asset types including +Charts, Dashboards, Data Pipelines, Data Tasks. + +## Enabling Glossary Term Propagation + +1. **Navigate to Automations**: Go to 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create An Automation**: Select 'Glossary Term Propagation' from the automation types. + +

+ +

+ +3. **Configure Automation**: Complete the required fields and select 'Save and Run' to activate the automation. + +

+ +

+ +## Propagating for Existing Assets + +In DataHub Cloud, you can back-fill historical data to ensure existing Glossary Terms are consistently propagated across downstream relationships. To begin, access the Automation created in Step 3, click the 3-dot "more" menu, and choose "Initialize." This will kick off the backfill process. + +

+ +

+ +and then click "Initialize". + +

+ +

+ +## Viewing Propagated Glossary Terms + +Once enabled, propagated Glossary Terms will display a thunderbolt icon, indicating the origin of the term and any intermediate lineage hops used in propagation. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/automations/snowflake-tag-propagation.md b/docs-archive/versioned_docs/version-1.5.0/docs/automations/snowflake-tag-propagation.md new file mode 100644 index 00000000..27af3014 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/automations/snowflake-tag-propagation.md @@ -0,0 +1,117 @@ +--- +title: Snowflake Tag Propagation Automation +slug: /automations/snowflake-tag-propagation +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/automations/snowflake-tag-propagation.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Snowflake Tag Propagation Automation + + + +:::info + +This feature is currently in open beta in DataHub Cloud. Reach out to your DataHub Cloud representative to get access. + +::: + +## Introduction + +Snowflake Tag Propagation is an automation that allows you to sync DataHub Glossary Terms and Tags on +both columns and tables back to Snowflake. This automation is available in DataHub Cloud only. + +## Capabilities + +- Automatically Add DataHub Glossary Terms to Snowflake Tables and Columns +- Automatically Add DataHub Tags to Snowflake Tables and Columns +- Automatically Remove DataHub Glossary Terms and Tags from Snowflake Tables and Columns when they are removed in DataHub + +## Prerequisites + +### Permissions Required for Tag Management + +- `CREATE TAG`: Required to create new tags in Snowflake. + Ensure the user or role has this privilege on the specific schema or database where tags will be created. +- `APPLY TAG`: Required to assign tags to Snowflake objects such as tables, columns, or other database objects. + This permission must be granted at the database, schema, or object level depending on the scope. + +### Permissions Required for Object Access + +- `USAGE` on the database and schema: Allows access to the database and schema to view and apply changes. +- `SELECT` on the objects (tables, views, etc.): Enables the automation to read metadata and verify existing tags. + +### Example Permission Grant Statements + +To grant the necessary permissions for a specific role (DATAHUB_AUTOMATION_ROLE), you can use the following SQL commands: + +```sql +-- Tag management permissions +GRANT CREATE TAG ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT APPLY TAG ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; + +-- Object access for metadata operations +GRANT USAGE ON DATABASE your_database TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT USAGE ON SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT SELECT ON ALL TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; + +-- Future privileges for tagging +GRANT SELECT ON FUTURE TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +GRANT APPLY TAG ON FUTURE TABLES IN SCHEMA your_database.your_schema TO ROLE DATAHUB_AUTOMATION_ROLE; +``` + +## Enabling Snowflake Tag Sync + +1. **Navigate to Automations**: Click on 'Govern' > 'Automations' in the navigation bar. + +

+ +

+ +2. **Create An Automation**: Click on 'Create' and select 'Snowflake Tag Propagation'. + +

+ +

+ +3. **Configure Automation**: Fill in the required fields to connect to Snowflake, along with the name, description, and category. + Note that you can limit propagation based on specific Tags and Glossary Terms. If none are selected, then ALL Tags or Glossary Terms will be automatically + propagated to Snowflake tables and columns. Finally, click 'Save and Run' to start the automation + +

+ +

+ +## Propagating for Existing Assets + +You can back-fill historical data for existing assets to ensure that all current column and table Glossary Terms are propagated to Snowflake. +Note that it may take some time to complete the initial back-filling process, depending on the number of Snowflake assets you have. + +To do so, navigate to the Automation you created in Step 3 above, click the 3-dot "More" menu + +

+ +

+ +and then click "Initialize". + +

+ +

+ +This one-time step will kick off the back-filling process for existing terms. If you only want to begin propagating +terms going forward, you can skip this step. + +:::info + +The back-filling of tags will be available in a future release. + +::: + +## Viewing Propagated Tags + +You can view propagated Tags (and corresponding DataHub URNs) inside the Snowflake UI to confirm the automation is working as expected. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/browseV2/browse-paths-v2.md b/docs-archive/versioned_docs/version-1.5.0/docs/browseV2/browse-paths-v2.md new file mode 100644 index 00000000..a4db64d4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/browseV2/browse-paths-v2.md @@ -0,0 +1,57 @@ +--- +title: Generating Browse Paths (V2) +slug: /browsev2/browse-paths-v2 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/browseV2/browse-paths-v2.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Generating Browse Paths (V2) + + + +## Introduction + +Browse (V2) is a way for users to explore and dive deeper into their data. Its integration with the search experience allows users to combine search queries and filters with entity type and platform nested folders. + +Most entities should have a browse path that allows users to navigate the left side panel on the search page to find groups of entities under different folders that come from these browse paths. Below, you can see an example of the sidebar with some new browse paths. + +

+ +

+ +This new browse sidebar always starts with Entity Type, then optionally shows Environment (PROD, DEV, etc.) if there are 2 or more Environments, then Platform. Below the Platform level, we render out folders that come directly from entity's [browsePathsV2](/docs/generated/metamodel/entities/dataset#browsepathsv2) aspects. + +## Generating Custom Browse Paths + +A `browsePathsV2` aspect has a field called `path` which contains a list of `BrowsePathEntry` objects. Each object in the path represents one level of the entity's browse path where the first entry is the highest level and the last entry is the lowest level. + +If an entity has this aspect filled out, their browse path will show up in the browse sidebar so that you can navigate its folders and select one to filter search results down. + +For example, in the browse sidebar on the left of the image above, there are 10 Dataset entities from the BigQuery Platform that have `browsePathsV2` aspects that look like the following: + +``` +[ { id: "bigquery-public-data" }, { id: "covid19_public_forecasts" } ] +``` + +The `id` in a `BrowsePathEntry` is required and is what will be shown in the UI unless the optional `urn` field is populated. If the `urn` field is populated, we will try to resolve this path entry into an entity object and display that entity's name. We will also show a link to allow you to open up the entity profile. + +The `urn` field should only be populated if there is an entity in your DataHub instance that belongs in that entity's browse path. This makes most sense for Datasets to have Container entities in the browse paths as well as some other cases such as a DataFlow being part of a DataJob's browse path. For any other situation, feel free to leave `urn` empty and populate `id` with the text you want to be shown in the UI for your entity's path. + +## Additional Resources + +### GraphQL + +- [browseV2](../../graphql/queries.md#browsev2) + +## FAQ and Troubleshooting + +**How are browsePathsV2 aspects created?** + +We create `browsePathsV2` aspects for all entities that should have one by default when you ingest your data if this aspect is not already provided. This happens based on separator characters that appear within an Urn. + +Our ingestion sources are also producing `browsePathsV2` aspects since CLI version v0.10.5. + +### Related Features + +- [Search](../how/search.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/businessattributes.md b/docs-archive/versioned_docs/version-1.5.0/docs/businessattributes.md new file mode 100644 index 00000000..78d553d4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/businessattributes.md @@ -0,0 +1,108 @@ +--- +title: Business Attributes +slug: /businessattributes +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/businessattributes.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Business Attributes + + + +> **Note:** This is BETA feature + +## What are Business Attributes + +A Business Attribute, as its name implies, is an attribute with a business focus. It embodies the traits or properties of an entity within a business framework. This attribute is a crucial piece of data for a business, utilised to define or control the entity throughout the organisation. If a business process or concept is depicted as a comprehensive logical model, then each Business Attribute can be considered as an individual component within that model. While business names and descriptions are generally managed through glossary terms, Business Attributes encompass additional characteristics such as data quality rules/assertions, data privacy markers, data usage protocols, standard tags, and supplementary documentation, alongside Names and Descriptions. + +For instance, "United States - Social Security Number" comes with a Name and definition. However, it also includes an abbreviation, a Personal Identifiable Information (PII) classification tag, a set of data rules, and possibly some additional references. + +## Benefits of Business Attributes + +The principle of "Define Once; Use in Many Places" applies to Business Attributes. Information Stewards can establish these attributes once with all their associated characteristics in an enterprise environment. Subsequently, individual applications or data owners can link their dataset attributes with these Business Attributes. This process allows the complete metadata structure built for a Business Attribute to be inherited. Application owners can also use these attributes to check if their applications align with the organisation-wide standard descriptions and data policies. This approach aids in centralised management for enhanced control and enforcement of metadata standards. + +This standardised metadata can be employed to facilitate data quality, data governance, and data discovery use cases within the organisation. + +A collection of 'related' Business Attributes can create a logical business model. + +With Business Attributes users have the ability to search associated datasets using business description/tags/glossary attached to business attribute + +## How can you use Business Attributes + +Business Attributes can be utilised in any of the following scenario: +Attributes that are frequently used across multiple domains, data products, projects, and applications. +Attributes requiring standardisation and inheritance of their characteristics, including name and descriptions, to be propagated. +Attributes that need centralised management for improved control and standard enforcement. + +A Business Attribute could be used to accelerate and standardise business definition management at entity / fields a field across various datasets. This ensures consistent application of the characteristics across all datasets using the Business Attribute. Any change in the them requires a change at only one place (i.e., business attributes) and change can then be inherited across all the application & datasets in the organisation + +Taking the example of "United States- Social Security Number", if an application or data owner has multiple instances of the social security number within their datasets, they can link all these dataset attributes with a Business Attribute to inherit all the aforementioned characteristics. Additionally, users can search for associated datasets using the business description, tags, or glossary linked to the Business Attribute. + +## Business Attributes Setup, Prerequisites, and Permissions + +What you need to create/update and associate business attributes to dataset schema field + +- **Manage Business Attributes** platform privilege to create/update/delete business attributes. + +## Using Business Attributes + +As of now Business Attributes can only be created through UI + +### Creating a Business Attribute (UI) + +To create a Business Attribute, first navigate to the Business Attributes tab on the home page. + +

+ +

+ +Then click on '+ Create Business Attribute'. +This will open a new modal where you can configure the settings for your business attribute. Inside the form, you can choose a name for Business Attribute. Most often, this will align with the logical purpose of the Business Attribute, +for example 'Social Security Number'. You can also add documentation for your Business Attribute to help other users easily discover it. This can be changed later. + +We can also add datatype for Business Attribute. It has String as a default value. + +

+ +

+ +Once you've chosen a name and a description, click 'Create' to create the new Business Attribute. + +Then we can attach tags and glossary terms to it to make it more discoverable. + +### Assigning Business Attribute to a Dataset Schema Field (UI) + +You can associate the business attribute to a dataset schema field using the Dataset's schema page as the starting point. As per design, there is one to one mapping between business attribute and dataset schema field. + +On a Dataset's schema page, click the 'Add Attribute' to add business attribute to the dataset schema field. + +

+ +

+ +After association, dataset schema field gets its description, tags and glossary inherited from Business attribute. +Description inherited from business attribute is greyed out to differentiate between original description of schema field. Similarly, tags and glossary terms inherited can't be removed directly. + +

+ +

+ +### Enable Business Attributes Feature + +By default, business attribute is disabled. To enable Business Attributes feature, export environmental variable +(may be done via `extraEnvs` for GMS deployment): + +```shell +BUSINESS_ATTRIBUTE_ENTITY_ENABLED=true +``` + +### What updates are planned for the Business Attributes feature? + +- Ingestion of Business attributes through recipe file (YAML) +- AutoTagging of Business attributes to child datasets through lineage + +### Related Features + +- [Glossary Terms](./glossary/business-glossary.md) +- [Tags](./tags.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/dataset.md b/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/dataset.md new file mode 100644 index 00000000..55e7da7b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/dataset.md @@ -0,0 +1,317 @@ +--- +title: DataHub Dataset Command +sidebar_label: Dataset Command +slug: /cli-commands/dataset +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/cli-commands/dataset.md +--- +# DataHub Dataset Command + +The `dataset` command allows you to interact with Dataset entities in DataHub. This includes creating, updating, retrieving, validating, and synchronizing Dataset metadata. + +## Commands + +### sync + +Synchronize Dataset metadata between YAML files and DataHub. + +```shell +datahub dataset sync -f PATH_TO_YAML_FILE --to-datahub|--from-datahub +``` + +**Options:** + +- `-f, --file` - Path to the YAML file (required) +- `--to-datahub` - Push metadata from YAML file to DataHub +- `--from-datahub` - Pull metadata from DataHub to YAML file + +**Example:** + +```shell +# Push to DataHub +datahub dataset sync -f dataset.yaml --to-datahub + +# Pull from DataHub +datahub dataset sync -f dataset.yaml --from-datahub +``` + +The `sync` command offers bidirectional synchronization, allowing you to keep your local YAML files in sync with the DataHub platform. The `upsert` command actually uses `sync` with the `--to-datahub` flag internally. + +For details on the supported YAML format, see the [Dataset YAML Format](#dataset-yaml-format) section. + +### file + +Operate on a Dataset YAML file for validation or linting. + +```shell +datahub dataset file [--lintCheck] [--lintFix] PATH_TO_YAML_FILE +``` + +**Options:** + +- `--lintCheck` - Check the YAML file for formatting issues (optional) +- `--lintFix` - Fix formatting issues in the YAML file (optional) + +**Example:** + +```shell +# Check for linting issues +datahub dataset file --lintCheck dataset.yaml + +# Fix linting issues +datahub dataset file --lintFix dataset.yaml +``` + +This command helps maintain consistent formatting of your Dataset YAML files. For more information on the expected format, refer to the [Dataset YAML Format](#dataset-yaml-format) section. + +### upsert + +Create or update Dataset metadata in DataHub. + +```shell +datahub dataset upsert -f PATH_TO_YAML_FILE +``` + +**Options:** + +- `-f, --file` - Path to the YAML file containing Dataset metadata (required) + +**Example:** + +```shell +datahub dataset upsert -f dataset.yaml +``` + +This command will parse the YAML file, validate that any entity references exist in DataHub, and then emit the corresponding metadata change proposals to update or create the Dataset. + +For details on the required structure of your YAML file, see the [Dataset YAML Format](#dataset-yaml-format) section. + +### get + +Retrieve Dataset metadata from DataHub and optionally write it to a file. + +```shell +datahub dataset get --urn DATASET_URN [--to-file OUTPUT_FILE] +``` + +**Options:** + +- `--urn` - The Dataset URN to retrieve (required) +- `--to-file` - Path to write the Dataset metadata as YAML (optional) + +**Example:** + +```shell +datahub dataset get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,example_table,PROD)" --to-file my_dataset.yaml +``` + +If the URN does not start with `urn:li:dataset:`, it will be automatically prefixed. + +The output file will be formatted according to the [Dataset YAML Format](#dataset-yaml-format) section. + +### add_sibling + +Add sibling relationships between Datasets. + +```shell +datahub dataset add_sibling --urn PRIMARY_URN --sibling-urns SECONDARY_URN [--sibling-urns ANOTHER_URN ...] +``` + +**Options:** + +- `--urn` - URN of the primary Dataset (required) +- `--sibling-urns` - URNs of secondary sibling Datasets (required, multiple allowed) + +**Example:** + +```shell +datahub dataset add_sibling --urn "urn:li:dataset:(urn:li:dataPlatform:hive,example_table,PROD)" --sibling-urns "urn:li:dataset:(urn:li:dataPlatform:snowflake,example_table,PROD)" +``` + +Siblings are semantically equivalent datasets, typically representing the same data across different platforms or environments. + +## Dataset YAML Format + +The Dataset YAML file follows a structured format with various supported fields: + +```yaml +# Basic identification (required) +id: "example_table" # Dataset identifier +platform: "hive" # Platform name +env: "PROD" # Environment (PROD by default) + +# Metadata (optional) +name: "Example Table" # Display name (defaults to id if not specified) +description: "This is an example table" + +# Schema definition (optional) +schema: + fields: + - id: "field1" # Field identifier + type: "string" # Data type + description: "First field" # Field description + doc: "First field" # Alias for description + nativeDataType: "VARCHAR" # Native platform type (defaults to type if not specified) + nullable: false # Whether field can be null (default: false) + label: "Field One" # Display label (optional business label for the field) + isPartOfKey: true # Whether field is part of primary key + isPartitioningKey: false # Whether field is a partitioning key + jsonProps: { "customProp": "value" } # Custom JSON properties + + - id: "field2" + type: "number" + description: "Second field" + nullable: true + globalTags: ["PII", "Sensitive"] + glossaryTerms: ["urn:li:glossaryTerm:Revenue"] + structured_properties: + property1: "value1" + property2: 42 + file: example.schema.avsc # Optional schema file (required if defining tables with nested fields) + +# Additional metadata (all optional) +properties: # Custom properties as key-value pairs + origin: "external" + pipeline: "etl_daily" + +subtype: "View" # Dataset subtype +subtypes: ["View", "Materialized"] # Multiple subtypes (if only one, use subtype field instead) + +downstreams: # Downstream Dataset URNs + - "urn:li:dataset:(urn:li:dataPlatform:hive,downstream_table,PROD)" + +tags: # Tags + - "Tier1" + - "Verified" + +glossary_terms: # Associated glossary terms + - "urn:li:glossaryTerm:Revenue" + +owners: # Dataset owners + - "jdoe" # Simple format (defaults to TECHNICAL_OWNER) + - id: "alice" # Extended format with ownership type + type: "BUSINESS_OWNER" + +structured_properties: # Structured properties + priority: "P1" + cost_center: 123 + +external_url: "https://example.com/datasets/example_table" +``` + +You can also define multiple datasets in a single YAML file by using a list format: + +```yaml +- id: "dataset1" + platform: "hive" + description: "First dataset" + # other properties... + +- id: "dataset2" + platform: "snowflake" + description: "Second dataset" + # other properties... +``` + +### Schema Definition + +You can define Dataset schema in two ways: + +1. **Direct field definitions** as shown above + + > **Important limitation**: When using inline schema field definitions, only non-nested (flat) fields are currently supported. For nested or complex schemas, you must use the Avro file approach described below. + +2. **Reference to an Avro schema file**: + ```yaml + schema: + file: "path/to/schema.avsc" + ``` + +Even when using the Avro file approach for the basic schema structure, you can still use the `fields` section to provide additional metadata like structured properties, tags, and glossary terms for your schema fields. + +#### Schema Field Properties + +The Schema Field object supports the following properties: + +| Property | Type | Description | +| ----------------------- | ------- | ----------------------------------------------------------------------------- | +| `id` | string | Field identifier/path (required if `urn` not provided) | +| `urn` | string | URN of the schema field (required if `id` not provided) | +| `type` | string | Data type (one of the supported [Field Types](#field-types)) | +| `nativeDataType` | string | Native data type in the source platform (defaults to `type` if not specified) | +| `description` | string | Field description | +| `doc` | string | Alias for description | +| `nullable` | boolean | Whether the field can be null (default: false) | +| `label` | string | Display label for the field | +| `recursive` | boolean | Whether the field is recursive (default: false) | +| `isPartOfKey` | boolean | Whether the field is part of the primary key | +| `isPartitioningKey` | boolean | Whether the field is a partitioning key | +| `jsonProps` | object | Custom JSON properties | +| `globalTags` | array | List of tags associated with the field | +| `glossaryTerms` | array | List of glossary terms associated with the field | +| `structured_properties` | object | Structured properties for the field | + +**Important Note on Schema Field Types**: +When specifying fields in the YAML file, you must follow an all-or-nothing approach with the `type` field: + +- If you want the command to generate the schema for you, specify the `type` field for ALL fields. +- If you only want to add field-level metadata (like tags, glossary terms, or structured properties), do NOT specify the `type` field for ANY field. + +Example of fields with only metadata (no types): + +```yaml +schema: + fields: + - id: "field1" # Field identifier + structured_properties: + prop1: prop_value + - id: "field2" + structured_properties: + prop1: prop_value +``` + +### Ownership Types + +When specifying owners, the following ownership types are supported: + +- `TECHNICAL_OWNER` (default) +- `BUSINESS_OWNER` +- `DATA_STEWARD` + +Custom ownership types can be specified using the URN format. + +### Field Types + +When defining schema fields, the following types are supported: +Primitive + +- `string` +- `number` +- `int` +- `long` +- `float` +- `double` +- `boolean` +- `bytes` +- `fixed` + +Complex + +- `array` +- `map` +- `union` +- `record` +- `date` +- `time` +- `timestamp` + +## Implementation Notes + +- URNs are generated automatically if not provided, based on the platform, id, and env values +- The command performs validation to ensure referenced entities (like structured properties) exist +- When updating schema fields, changes are propagated correctly to maintain consistent metadata +- The Dataset object will check for existence of entity references and will skip datasets with missing references +- When using the `sync` command with `--from-datahub`, existing YAML files will be updated with metadata from DataHub while preserving comments and structure +- For structured properties, single values are simplified (not wrapped in lists) when appropriate +- Field paths are simplified for better readability +- When specifying field types, all fields must have type information or none of them should diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/graphql.md b/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/graphql.md new file mode 100644 index 00000000..507d5649 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/graphql.md @@ -0,0 +1,509 @@ +--- +title: DataHub GraphQL CLI +sidebar_label: GraphQL CLI +slug: /cli-commands/graphql +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/cli-commands/graphql.md +--- +# DataHub GraphQL CLI + +The `datahub graphql` command provides a powerful interface to interact with DataHub's GraphQL API directly from the command line. This enables you to query metadata, perform mutations, and explore the GraphQL schema without writing custom applications. + +## Quick Start + +```shell +# Get current user info +datahub graphql --operation me + +# Search for datasets +datahub graphql --operation searchAcrossEntities --variables '{"input": {"query": "users", "types": ["DATASET"]}}' + +# Execute raw GraphQL +datahub graphql --query "query { me { username } }" +``` + +## Core Features + +### 1. Schema Discovery + +Discover available operations and understand their structure: + +```shell +# List all available operations +datahub graphql --list-operations + +# List only queries or mutations +datahub graphql --list-queries +datahub graphql --list-mutations +``` + +### 2. Smart Description + +The `--describe` command intelligently searches for both operations and types: + +```shell +# Describe an operation +datahub graphql --describe searchAcrossEntities + +# Describe a GraphQL type +datahub graphql --describe SearchInput + +# Describe enum types to see allowed values +datahub graphql --describe FilterOperator +``` + +**When both operation and type exist with same name:** + +```shell +datahub graphql --describe someConflictingName +# Output: +# === OPERATION === +# Operation: someConflictingName +# Type: Query +# ... +# +# === TYPE === +# Type: someConflictingName +# Kind: INPUT_OBJECT +# ... +``` + +### 3. Recursive Type Exploration + +Use `--recurse` with `--describe` to explore all nested types: + +```shell +# Explore operation with all its input types +datahub graphql --describe searchAcrossEntities --recurse + +# Explore type with all nested dependencies +datahub graphql --describe SearchInput --recurse +``` + +**Example recursive output:** + +``` +Operation: searchAcrossEntities +Type: Query +Description: Search across all entity types +Arguments: + - input: SearchInput! + +Input Type Details: + +SearchInput: + query: String + types: [EntityType!] + filters: SearchFilter + +SearchFilter: + criteria: [FacetFilterInput!] + +FacetFilterInput: + field: String! - Name of field to filter by + values: [String!]! - Values, one of which the intended field should match + condition: FilterOperator - Condition for the values + +FilterOperator: + EQUAL - Represents the relation: field = value + GREATER_THAN - Represents the relation: field > value + LESS_THAN - Represents the relation: field < value +``` + +### 4. Operation Execution + +Execute operations by name without writing full GraphQL: + +```shell +# Execute operation by name +datahub graphql --operation me + +# Execute with variables +datahub graphql --operation searchAcrossEntities --variables '{"input": {"query": "datasets", "types": ["DATASET"]}}' + +# Execute with variables from file +datahub graphql --operation createGroup --variables ./group-data.json +``` + +### 5. Raw GraphQL Execution + +Execute any custom GraphQL query or mutation: + +```shell +# Simple query +datahub graphql --query "query { me { username } }" + +# Query with variables +datahub graphql --query "query GetUser($urn: String!) { corpUser(urn: $urn) { info { email } } }" --variables '{"urn": "urn:li:corpuser:john"}' + +# Query from file +datahub graphql --query ./complex-query.graphql --variables ./variables.json + +# Mutation +datahub graphql --query "mutation { addTag(input: {resourceUrn: \"urn:li:dataset:...\", tagUrn: \"urn:li:tag:Important\"}) }" +``` + +### 6. File Support + +Both queries and variables can be loaded from files: + +```shell +# Load query from file +datahub graphql --query ./queries/search-datasets.graphql + +# Load variables from file +datahub graphql --operation searchAcrossEntities --variables ./variables/search-params.json + +# Both from files +datahub graphql --query ./query.graphql --variables ./vars.json +``` + +### 7. LLM-Friendly JSON Output + +Use `--format json` to get structured JSON output perfect for LLM consumption: + +```shell +# Get operations as JSON for LLM processing +datahub graphql --list-operations --format json + +# Describe operation with complete type information +datahub graphql --describe searchAcrossEntities --recurse --format json + +# Get type details in structured format +datahub graphql --describe SearchInput --format json +``` + +**Example JSON output for `--list-operations --format json`:** + +```json +{ + "schema": { + "queries": [ + { + "name": "me", + "type": "Query", + "description": "Get current user information", + "arguments": [] + }, + { + "name": "searchAcrossEntities", + "type": "Query", + "description": "Search across all entity types", + "arguments": [ + { + "name": "input", + "type": { + "kind": "NON_NULL", + "ofType": { + "name": "SearchInput", + "kind": "INPUT_OBJECT" + } + }, + "required": true, + "description": "Search input parameters" + } + ] + } + ], + "mutations": [...] + } +} +``` + +**Example JSON output for `--describe searchAcrossEntities --recurse --format json`:** + +```json +{ + "operation": { + "name": "searchAcrossEntities", + "type": "Query", + "description": "Search across all entity types", + "arguments": [...] + }, + "relatedTypes": { + "SearchInput": { + "name": "SearchInput", + "kind": "INPUT_OBJECT", + "fields": [ + { + "name": "query", + "type": {"name": "String", "kind": "SCALAR"}, + "description": "Search query string" + }, + { + "name": "filters", + "type": {"name": "SearchFilter", "kind": "INPUT_OBJECT"}, + "description": "Optional filters" + } + ] + }, + "SearchFilter": {...}, + "FilterOperator": { + "name": "FilterOperator", + "kind": "ENUM", + "values": [ + { + "name": "EQUAL", + "description": "Represents the relation: field = value", + "deprecated": false + } + ] + } + }, + "meta": { + "query": "searchAcrossEntities", + "recursive": true + } +} +``` + +### 8. Custom Schema Path + +When introspection is disabled or for local development: + +```shell +# Use local GraphQL schema files +datahub graphql --list-operations --schema-path ./local-schemas/ + +# Describe with custom schema +datahub graphql --describe searchAcrossEntities --schema-path ./graphql-schemas/ + +# Get JSON format with custom schema +datahub graphql --list-operations --schema-path ./schemas/ --format json +``` + +## Command Reference + +### Global Options + +| Option | Type | Description | +| ------------------- | ------ | -------------------------------------------------------------- | +| `--query` | string | GraphQL query/mutation string or path to .graphql file | +| `--variables` | string | Variables as JSON string or path to .json file | +| `--operation` | string | Execute named operation from DataHub's schema | +| `--describe` | string | Describe operation or type (searches both) | +| `--recurse` | flag | Recursively explore nested types with --describe | +| `--list-operations` | flag | List all available operations | +| `--list-queries` | flag | List available query operations | +| `--list-mutations` | flag | List available mutation operations | +| `--schema-path` | string | Path to GraphQL schema files directory | +| `--no-pretty` | flag | Disable pretty-printing of JSON output (default: pretty-print) | +| `--format` | choice | Output format: `human` (default) or `json` for LLM consumption | + +### Usage Patterns + +```shell +# Discovery +datahub graphql --list-operations +datahub graphql --describe [--recurse] + +# Execution +datahub graphql --operation [--variables ] +datahub graphql --query [--variables ] +``` + +## Advanced Examples + +### Complex Search with Filters + +```shell +datahub graphql --operation searchAcrossEntities --variables '{ + "input": { + "query": "customer", + "types": ["DATASET", "DASHBOARD"], + "filters": [{ + "field": "platform", + "values": ["mysql", "postgres"] + }], + "start": 0, + "count": 20 + } +}' +``` + +### Adding Tags to Multiple Entities + +```shell +# Add Important tag to a dataset +datahub graphql --query 'mutation AddTag($input: TagAssociationInput!) { + addTag(input: $input) +}' --variables '{ + "input": { + "resourceUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,db.users,PROD)", + "tagUrn": "urn:li:tag:Important" + } +}' +``` + +### Batch User Queries + +```shell +# Get multiple users using raw GraphQL +datahub graphql --query 'query GetUsers($urns: [String!]!) { + users: batchGet(urns: $urns) { + ... on CorpUser { + urn + username + properties { + email + displayName + } + } + } +}' --variables '{"urns": ["urn:li:corpuser:alice", "urn:li:corpuser:bob"]}' +``` + +## Schema Introspection + +DataHub's GraphQL CLI provides two modes for schema discovery: + +### Schema Discovery Modes + +1. **Live Introspection** (default): Queries the live GraphQL endpoint when no `--schema-path` is provided +2. **Local Schema Files**: Uses `.graphql` files from the specified directory when `--schema-path` is provided + +**Note:** These modes are mutually exclusive with no fallback between them. If introspection fails, the command will fail with an error. If local schema files are invalid, the command will fail with an error. + +### Schema File Structure + +When using `--schema-path`, the directory should contain `.graphql` files with: + +```graphql +# queries.graphql +extend type Query { + me: AuthenticatedUser + searchAcrossEntities(input: SearchInput!): SearchResults +} + +# mutations.graphql +extend type Mutation { + addTag(input: TagAssociationInput!): String + deleteEntity(urn: String!): String +} +``` + +## Error Handling + +The CLI provides clear error messages for common issues: + +```shell +# Operation not found +datahub graphql --describe nonExistentOp +# Error: 'nonExistentOp' not found as an operation or type. Use --list-operations to see available operations or try a specific type name. + +# Missing required arguments +datahub graphql --operation searchAcrossEntities +# Error: Operation 'searchAcrossEntities' requires arguments: input. Provide them using --variables '{"input": "value", ...}' + +# Invalid JSON variables +datahub graphql --operation me --variables '{invalid json}' +# Error: Invalid JSON in variables: Expecting property name enclosed in double quotes +``` + +## Output Formats + +### Pretty Printing (Default) + +```json +{ + "me": { + "corpUser": { + "urn": "urn:li:corpuser:datahub", + "username": "datahub" + } + } +} +``` + +### Compact Output + +```shell +datahub graphql --operation me --no-pretty +{"me":{"corpUser":{"urn":"urn:li:corpuser:datahub","username":"datahub"}}} +``` + +## Integration Examples + +### Shell Scripts + +```bash +#!/bin/bash +# Get all datasets for a platform +PLATFORM="mysql" +RESULTS=$(datahub graphql --operation searchAcrossEntities --variables "{ + \"input\": { + \"query\": \"*\", + \"types\": [\"DATASET\"], + \"filters\": [{\"field\": \"platform\", \"values\": [\"$PLATFORM\"]}] + } +}" --no-pretty) + +echo "Found $(echo "$RESULTS" | jq '.searchAcrossEntities.total') datasets" +``` + +### CI/CD Pipelines + +```yaml +# GitHub Actions example +- name: Tag Important Datasets + run: | + datahub graphql --operation addTag --variables '{ + "input": { + "resourceUrn": "${{ env.DATASET_URN }}", + "tagUrn": "urn:li:tag:Production" + } + }' +``` + +## LLM Integration + +The `--format json` option makes the CLI perfect for LLM integration: + +### Benefits for AI Assistants + +1. **Schema Understanding**: LLMs can parse the complete GraphQL schema structure +2. **Query Generation**: AI can generate accurate GraphQL queries based on available operations +3. **Type Validation**: LLMs understand required vs optional arguments and their types +4. **Documentation**: Rich descriptions and examples help AI provide better user assistance + +### Use Cases + +```shell +# AI assistant gets complete schema knowledge +datahub graphql --list-operations --format json | ai-assistant process-schema + +# Generate queries for user requests +datahub graphql --describe searchAcrossEntities --recurse --format json | ai-helper generate-query --user-intent "find mysql tables" + +# Validate user input against schema +datahub graphql --describe createGroup --format json | validate-user-input +``` + +### JSON Schema Benefits + +- **Structured data**: No parsing of human-readable text required +- **Complete type information**: Includes GraphQL type wrappers (NON_NULL, LIST) +- **Rich metadata**: Descriptions, deprecation info, argument requirements +- **Consistent format**: Predictable structure across all operations and types +- **Recursive exploration**: Complete dependency graphs for complex types + +## Tips and Best Practices + +1. **Start with Discovery**: Use `--list-operations` and `--describe` to understand available operations +2. **Use --recurse**: When learning about complex operations, `--describe --recurse` shows the complete type structure +3. **LLM Integration**: Use `--format json` when building AI assistants or automation tools +4. **File-based Variables**: For complex variables, use JSON files instead of inline JSON +5. **Error Handling**: The CLI provides detailed error messages - read them carefully for debugging +6. **Schema Evolution**: Operations and types can change between DataHub versions - use discovery commands to stay current + +## Troubleshooting + +### Common Issues + +**"Introspection not available"**: Use `--schema-path` to point to local GraphQL schema files + +**"Operation not found"**: Check spelling and use `--list-operations` to see available operations + +**"Type not found"**: Verify type name casing (GraphQL types are case-sensitive) + +**Environment issues**: Ensure DataHub server is running and accessible at the configured endpoint diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/search.md b/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/search.md new file mode 100644 index 00000000..e8a07072 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/cli-commands/search.md @@ -0,0 +1,786 @@ +--- +title: DataHub Search Command +sidebar_label: Search Command +slug: /cli-commands/search +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/cli-commands/search.md +--- +# DataHub Search Command + +The `datahub search` command provides a powerful interface to search across all DataHub entities directly from the command line. It supports both keyword search (default) and semantic search with flexible filtering, multiple output formats, and comprehensive discovery features. + +The search command is a **command group** with multiple subcommands: + +- `query` (default) - Execute search queries across DataHub entities +- `diagnose` - Diagnose search configuration and availability + +## Quick Start + +```shell +# Basic keyword search (query is the default subcommand) +datahub search "users" +datahub search query "users" # Explicit subcommand + +# Search with filters +datahub search "*" --filter platform=snowflake --filter entity_type=dataset + +# Semantic search +datahub search --semantic "financial reports" + +# Get results in table format +datahub search "dashboard" --table + +# Get just URNs for piping +datahub search "dataset" --urns-only + +# Preview query without executing +datahub search "users" --filter platform=snowflake --dry-run + +# Custom field projection (reduce response size) +datahub search "users" --projection "urn type platform { name }" + +# Diagnose search configuration +datahub search diagnose +datahub search diagnose --format json +``` + +## Core Features + +### 1. Dual Search Modes + +**Keyword Search (Default)**: Traditional text-based search using DataHub's search index. + +```shell +datahub search "users" +datahub search "customer_data" +``` + +**Semantic Search**: AI-powered search that understands meaning and context. + +```shell +datahub search --semantic "financial reports" +datahub search --semantic "user analytics dashboards" +``` + +### 2. Flexible Filtering + +**Simple Filters**: Easy key=value syntax with implicit AND logic. + +```shell +# Single filter +datahub search "*" --filter platform=snowflake + +# Multiple filters (AND) +datahub search "*" --filter platform=snowflake --filter entity_type=dataset + +# OR logic with comma-separated values +datahub search "*" --filter platform=snowflake,bigquery +``` + +**Complex Filters**: JSON-based filters with AND/OR/NOT logic. + +```shell +# AND logic +datahub search "*" --filters '{"and": [{"platform": ["snowflake"]}, {"env": ["PROD"]}]}' + +# OR logic +datahub search "*" --filters '{"or": [{"entity_type": ["chart"]}, {"entity_type": ["dashboard"]}]}' + +# NOT logic +datahub search "*" --filters '{"not": {"platform": ["mysql"]}}' + +# Complex nested logic +datahub search "*" --filters '{ + "and": [ + {"platform": ["snowflake", "bigquery"]}, + {"env": ["PROD"]}, + {"not": {"tag": ["urn:li:tag:Deprecated"]}} + ] +}' +``` + +### 3. Multiple Output Formats + +**JSON (Default)**: Structured output with full metadata. + +```shell +datahub search "users" --format json +# or simply +datahub search "users" +``` + +**Table**: Human-readable table format. + +```shell +datahub search "users" --table +# or +datahub search "users" --format table +``` + +**URNs**: List of URNs only, perfect for piping. + +```shell +datahub search "dataset" --urns-only +# or +datahub search "dataset" --format urns +``` + +### 4. Filter Discovery + +**List Available Filters**: See all filter types and their descriptions. + +```shell +datahub search --list-filters +``` + +Output includes: + +- entity_type, entity_subtype +- platform, domain, container +- tag, glossary_term, owner +- env, status +- custom, and, or, not + +**Describe Specific Filter**: Get detailed information about a filter. + +```shell +datahub search --describe-filter platform +datahub search --describe-filter entity_type +datahub search --describe-filter custom +``` + +### 5. Faceted Search + +**Facets-Only Mode**: Get aggregation counts without search results. + +```shell +datahub search "*" --facets-only + +# With filters +datahub search "*" --filter entity_type=dataset --facets-only --format table +``` + +Useful for: + +- Understanding data distribution +- Building filter UIs +- Data exploration + +### 6. Pagination + +```shell +# Get first 20 results +datahub search "*" --limit 20 + +# Get next page (results 20-40) +datahub search "*" --limit 20 --offset 20 + +# Maximum limit is 50 +datahub search "*" --limit 50 +``` + +### 7. Sorting + +```shell +# Sort by field +datahub search "*" --sort-by lastModified + +# Sort ascending +datahub search "*" --sort-by name --sort-order asc + +# Sort descending (default) +datahub search "*" --sort-by name --sort-order desc +``` + +### 8. DataHub Views + +Apply saved DataHub Views to searches: + +```shell +datahub search "*" --view "urn:li:dataHubView:12345678" +``` + +### 9. Dry Run + +Preview the compiled GraphQL query and variables without connecting to DataHub: + +```shell +# See what query would be sent +datahub search "users" --dry-run + +# Dry run with filters and sorting +datahub search "*" --filter platform=snowflake --sort-by lastModified --dry-run + +# Dry run with semantic search +datahub search --semantic "financial data" --dry-run + +# Dry run with projection +datahub search "users" --projection "urn type" --dry-run +``` + +Output is a JSON object containing `operation_name`, `graphql_field`, `variables`, and optionally `projection` and `query` (when using `--projection`). + +### 10. Field Projection + +Control which GraphQL fields are returned per entity to reduce response size: + +```shell +# Return only URN and type +datahub search "users" --projection "urn type" + +# Return URN with platform info +datahub search "users" --projection "urn platform { ...PlatformFields }" + +# Load projection from a file +datahub search "users" --projection @my_fields.gql + +# Braces are optional — these are equivalent +datahub search "users" --projection "{ urn type }" +datahub search "users" --projection "urn type" +``` + +When `--projection` is omitted, the full default selection set from `search_queries.gql` is used. Projections have a 5000-character limit and must not contain mutations, introspection queries, variables, or directives. + +### 11. Agent Context + +Print best-practice guidance for AI agents consuming the search CLI: + +```shell +# Print agent context and exit +datahub search --agent-context +``` + +When stdout is not a TTY (e.g., piped to an AI agent), `--help` automatically appends the agent context. + +## Command Reference + +### Main Command + +```shell +datahub search [OPTIONS] COMMAND [ARGS] +``` + +The `search` command is a command group. The default subcommand is `query`, so `datahub search "text"` is equivalent to `datahub search query "text"`. + +### Subcommands + +#### `query` (default) + +Execute search queries across DataHub entities. + +```shell +datahub search [QUERY] [OPTIONS] +datahub search query [QUERY] [OPTIONS] +``` + +**Options:** + +| Option | Type | Description | +| ------------------- | ------ | ----------------------------------------------------------- | +| `QUERY` | string | Search query string (default: "\*" for all entities) | +| `--semantic` | flag | Use semantic search instead of keyword search | +| `-f, --filter` | string | Simple filter: key=value (repeatable, comma for OR) | +| `--filters` | string | Complex filters as JSON string (AND/OR/NOT logic) | +| `-n, --limit` | int | Number of results to return (default: 10, max: 50) | +| `--offset` | int | Starting position for pagination (default: 0) | +| `--sort-by` | string | Field name to sort by | +| `--sort-order` | choice | Sort order: `asc` or `desc` (default: desc) | +| `--format` | choice | Output format: `json`, `table`, or `urns` (default: json) | +| `--table` | flag | Shortcut for --format table | +| `--urns-only` | flag | Shortcut for --format urns | +| `--list-filters` | flag | List all available filter fields | +| `--describe-filter` | string | Describe a specific filter field | +| `--facets-only` | flag | Return only facets, no search results | +| `--view` | string | DataHub View URN to apply | +| `--dry-run` | flag | Show compiled GraphQL query and variables without executing | +| `--projection` | string | Custom GraphQL selection set for entity (inline or @file) | +| `--agent-context` | flag | Print agent skill context (best practices for AI agents) | + +#### `diagnose` + +Diagnose search configuration and availability. Checks semantic search configuration, backend connectivity, embedding provider, model details, and enabled entity types. + +```shell +datahub search diagnose [OPTIONS] +``` + +**Options:** + +| Option | Type | Description | +| ---------- | ------ | ----------------------------------------------- | +| `--format` | choice | Output format: `text` or `json` (default: text) | + +**Examples:** + +```shell +# Text output (default) +datahub search diagnose + +# JSON output +datahub search diagnose --format json +``` + +### Available Filters + +#### Entity Filters + +- **entity_type**: Filter by entity type (dataset, dashboard, chart, corpuser, etc.) +- **entity_subtype**: Filter by entity subtype (Table, View, Model, etc.) + +#### Platform & Location + +- **platform**: Filter by data platform (snowflake, bigquery, looker, etc.) +- **container**: Filter by container URN (database, schema, etc.) +- **domain**: Filter by domain URN + +#### Metadata + +- **tag**: Filter by tag URN +- **glossary_term**: Filter by glossary term URN +- **owner**: Filter by owner URN (user or group) + +#### Environment + +- **env**: Filter by environment (PROD, DEV, STAGING, QA) +- **status**: Filter by deletion status (NOT_SOFT_DELETED, SOFT_DELETED) + +#### Advanced + +- **custom**: Custom field filter with flexible conditions +- **and**: Logical AND of filters +- **or**: Logical OR of filters +- **not**: Logical NOT of a filter + +## Usage Examples + +### Basic Searches + +```shell +# Search for everything +datahub search "*" + +# Search for specific term +datahub search "users" + +# Search with limit +datahub search "dashboard" --limit 20 +``` + +### Semantic Search + +```shell +# Find financial datasets +datahub search --semantic "financial reports and revenue data" + +# Find user analytics +datahub search --semantic "user behavior analysis dashboards" + +# Find ML models +datahub search --semantic "machine learning models for prediction" +``` + +### Simple Filtering + +```shell +# Platform filter +datahub search "*" --filter platform=snowflake + +# Entity type filter +datahub search "*" --filter entity_type=dataset + +# Multiple filters (AND) +datahub search "*" --filter platform=snowflake --filter env=PROD + +# OR with comma separation +datahub search "*" --filter platform=snowflake,bigquery + +# Combine different filters +datahub search "customer" --filter platform=mysql --filter entity_type=dataset,dashboard +``` + +### Complex Filtering + +```shell +# Find Snowflake PROD datasets +datahub search "*" --filters '{ + "and": [ + {"platform": ["snowflake"]}, + {"entity_type": ["dataset"]}, + {"env": ["PROD"]} + ] +}' + +# Find charts OR dashboards in Looker +datahub search "*" --filters '{ + "and": [ + {"platform": ["looker"]}, + {"or": [ + {"entity_type": ["chart"]}, + {"entity_type": ["dashboard"]} + ]} + ] +}' + +# Exclude deprecated datasets +datahub search "*" --filters '{ + "and": [ + {"entity_type": ["dataset"]}, + {"not": {"tag": ["urn:li:tag:Deprecated"]}} + ] +}' + +# Custom field filter +datahub search "*" --filters '{ + "custom": { + "field": "name", + "condition": "CONTAIN", + "values": ["customer"] + } +}' +``` + +### Output Formats + +```shell +# JSON (default) - full metadata +datahub search "users" --format json + +# Table - human readable +datahub search "users" --table + +# URNs only - for piping to other commands +datahub search "*" --filter entity_type=dataset --urns-only | while read urn; do + datahub get --urn "$urn" --aspect ownership +done +``` + +### Discovery & Exploration + +```shell +# List all available filters +datahub search --list-filters + +# Learn about a specific filter +datahub search --describe-filter platform +datahub search --describe-filter entity_type +datahub search --describe-filter custom + +# Explore data distribution with facets +datahub search "*" --facets-only --format table +datahub search "*" --filter entity_type=dataset --facets-only + +# Diagnose search configuration +datahub search diagnose +datahub search diagnose --format json +``` + +### Pagination + +```shell +# First page (10 results) +datahub search "dataset" + +# Second page +datahub search "dataset" --limit 10 --offset 10 + +# Large page +datahub search "*" --limit 50 --offset 100 +``` + +### Sorting + +```shell +# Sort by last modified (newest first) +datahub search "*" --sort-by lastModified --sort-order desc + +# Sort by name (alphabetically) +datahub search "*" --sort-by name --sort-order asc +``` + +## Advanced Use Cases + +### Finding Datasets in Multiple Platforms + +```shell +# Quick version with simple filters +datahub search "*" --filter platform=snowflake,bigquery,postgres --filter entity_type=dataset --table + +# Complex version with AND/OR +datahub search "*" --filters '{ + "and": [ + {"entity_type": ["dataset"]}, + {"or": [ + {"platform": ["snowflake"]}, + {"platform": ["bigquery"]}, + {"platform": ["postgres"]} + ]} + ] +}' --table +``` + +### Finding Production Data with Specific Tags + +```shell +datahub search "*" --filters '{ + "and": [ + {"env": ["PROD"]}, + {"tag": ["urn:li:tag:PII", "urn:li:tag:Sensitive"]} + ] +}' --table +``` + +### Finding Datasets Owned by Specific Team + +```shell +datahub search "*" --filters '{ + "and": [ + {"entity_type": ["dataset"]}, + {"owner": ["urn:li:corpGroup:data-eng"]} + ] +}' --table +``` + +### Pipeline: Search and Process + +```shell +# Find all PROD datasets and check their ownership +datahub search "*" \ + --filter entity_type=dataset \ + --filter env=PROD \ + --urns-only | \ +while read urn; do + echo "Checking $urn" + datahub get --urn "$urn" --aspect ownership +done + +# Find datasets and export to file +datahub search "*" \ + --filter platform=snowflake \ + --format json > snowflake_datasets.json +``` + +### Semantic Search for Data Discovery + +```shell +# Find datasets about customers +datahub search --semantic "customer information and profiles" + +# Find ML models for forecasting +datahub search --semantic "predictive models and forecasting algorithms" + +# Find compliance-related documentation +datahub search --semantic "GDPR compliance and privacy documentation" +``` + +## Output Examples + +### JSON Output (Default) + +```json +{ + "count": 2, + "facets": [ + { + "aggregations": [ + { "count": 10, "value": "DATASET" }, + { "count": 5, "value": "DASHBOARD" } + ], + "displayName": "Type", + "field": "_entityType" + } + ], + "searchResults": [ + { + "entity": { + "platform": { "name": "snowflake" }, + "properties": { + "description": "User data table", + "name": "users" + }, + "urn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,db.users,PROD)" + }, + "matchedFields": [{ "name": "name", "value": "users" }] + } + ], + "start": 0, + "total": 15 +} +``` + +### Table Output + +``` ++--------------------------------------------------------------+------------+-----------+-------------------------+ +| URN | Name | Platform | Description | ++==============================================================+============+===========+=========================+ +| urn:li:dataset:(urn:li:dataPlatform:snowflake,db.users,PROD)| users | snowflake | User data table | ++--------------------------------------------------------------+------------+-----------+-------------------------+ +| urn:li:dashboard:(looker,dashboards.19) | User Stats | looker | User analytics dash... | ++--------------------------------------------------------------+------------+-----------+-------------------------+ + +Showing 1-2 of 15 results +``` + +### URNs Output + +``` +urn:li:dataset:(urn:li:dataPlatform:snowflake,db.users,PROD) +urn:li:dashboard:(looker,dashboards.19) +urn:li:chart:(looker,chart.123) +``` + +## Integration Examples + +### Shell Scripts + +```bash +#!/bin/bash +# Find and tag all PROD datasets in Snowflake + +DATASETS=$(datahub search "*" \ + --filter platform=snowflake \ + --filter entity_type=dataset \ + --filter env=PROD \ + --urns-only) + +for urn in $DATASETS; do + echo "Tagging $urn" + datahub put --urn "$urn" --aspect globalTags -d '{ + "tags": [{"tag": "urn:li:tag:Production"}] + }' +done +``` + +### Data Quality Checks + +```bash +#!/bin/bash +# Find datasets without ownership + +DATASETS=$(datahub search "*" \ + --filter entity_type=dataset \ + --limit 50 \ + --format json) + +echo "$DATASETS" | jq -r '.searchResults[] | + select(.entity.ownership == null) | + .entity.urn' +``` + +### Monitoring + +```bash +#!/bin/bash +# Monitor new datasets added today + +COUNT=$(datahub search "*" \ + --filter entity_type=dataset \ + --format json | jq '.total') + +echo "Total datasets: $COUNT" +``` + +## Error Handling + +The CLI provides clear error messages for common issues: + +```shell +# Semantic search not enabled +datahub search --semantic "test" +# Error: Semantic search is not enabled on this DataHub instance. +# Use keyword search (default) or contact your DataHub administrator. + +# Invalid filter format +datahub search "*" --filter invalid_format +# Error: Invalid filter: invalid_format. Expected key=value format. + +# Conflicting filter options +datahub search "*" --filter platform=snowflake --filters '{"env": ["PROD"]}' +# Error: Cannot use both --filter and --filters. +# Use --filter for simple filters or --filters for complex logic. + +# Invalid limit +datahub search "*" --limit 0 +# Error: --limit must be at least 1 + +# Limit too high +datahub search "*" --limit 100 +# Warning: limit capped at 50 +``` + +## Diagnostics + +The `diagnose` subcommand helps troubleshoot search configuration issues, particularly for semantic search: + +```shell +# Check semantic search configuration +datahub search diagnose + +# Get detailed JSON output +datahub search diagnose --format json +``` + +The diagnostic output includes: + +- Semantic search availability and status +- Embedding provider configuration +- Model ID and embedding key +- AWS region (if applicable) +- Enabled entity types for semantic search +- Recommendations for resolving issues + +## Tips and Best Practices + +1. **Start Broad**: Begin with `datahub search "*"` and add filters progressively +2. **Use Discovery**: Run `--list-filters` and `--describe-filter` to learn available options +3. **Test with `--table`**: Use table format for quick exploration before JSON processing +4. **Semantic Search**: Use semantic search for natural language queries and concept-based discovery +5. **Facets First**: Use `--facets-only` to understand data distribution before filtering +6. **Pipe URNs**: Combine `--urns-only` with other `datahub` commands for bulk operations +7. **Limit Wisely**: Start with small limits during development, increase for production +8. **Save Complex Filters**: Store complex JSON filters in files for reusability +9. **Check Pagination**: Use `total` field in JSON output to determine if pagination is needed +10. **Environment Variables**: Configure `DATAHUB_GMS_URL` and `DATAHUB_GMS_TOKEN` for easier access + +## Troubleshooting + +### Common Issues + +**"Semantic search is not enabled"**: Your DataHub instance doesn't have semantic search configured. Use keyword search instead or contact your administrator. + +**"No results found"**: Try broadening your search query or removing filters. Use `datahub search "*"` to verify connectivity. + +**"Invalid filter"**: Check filter syntax. Use `--list-filters` to see available filters and `--describe-filter ` for details. + +**Slow searches**: Use more specific filters to narrow results. Check network latency to DataHub server. + +**Permission denied**: Ensure you've run `datahub init` and have valid credentials configured. + +### Getting Help + +```shell +# Command help +datahub search --help + +# Subcommand help +datahub search query --help +datahub search diagnose --help + +# List available filters +datahub search --list-filters + +# Learn about specific filter +datahub search --describe-filter platform + +# Diagnose search configuration +datahub search diagnose + +# Check DataHub connection +datahub check plugins +``` + +## See Also + +- [DataHub CLI](../cli.md) - Main CLI documentation +- [GraphQL Command](./graphql.md) - GraphQL query interface +- [Dataset Command](./dataset.md) - Dataset-specific operations diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/cli.md b/docs-archive/versioned_docs/version-1.5.0/docs/cli.md new file mode 100644 index 00000000..a9ba3d7b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/cli.md @@ -0,0 +1,1203 @@ +--- +toc_max_heading_level: 4 +title: DataHub CLI +sidebar_label: CLI +slug: /cli +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/cli.md' +--- + +# DataHub CLI + +DataHub comes with a friendly cli called `datahub` that allows you to perform a lot of common operations using just the command line. [DataHub](https://datahub.com) maintains the [pypi package](https://pypi.org/project/acryl-datahub/) for `datahub`. + +## Installation + +### Using pip + +We recommend Python virtual environments (venv-s) to namespace pip modules. Here's an example setup: + +```shell +python3 -m venv venv # create the environment +source venv/bin/activate # activate the environment +``` + +**_NOTE:_** If you install `datahub` in a virtual environment, that same virtual environment must be re-activated each time a shell window or session is created. + +Once inside the virtual environment, install `datahub` using the following commands + +```shell +# Requires Python 3.10+ +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub +# validate that the install was successful +datahub version +# If you see "command not found", try running this instead: python3 -m datahub version +datahub init +# authenticate your datahub CLI with your datahub instance +``` + +If you run into an error, try checking the [_common setup issues_](../metadata-ingestion/developing.md#common-setup-issues). + +Other installation options such as installation from source and running the cli inside a container are available further below in the guide [here](#alternate-installation-options). + +## Starter Commands + +The `datahub` cli allows you to do many things, such as quick-starting a DataHub docker instance locally, ingesting metadata from your sources into a DataHub server or a DataHub lite instance, as well as retrieving, modifying and exploring metadata. +Like most command line tools, `--help` is your best friend. Use it to discover the capabilities of the cli and the different commands and sub-commands that are supported. + +```console +datahub --help +Usage: datahub [OPTIONS] COMMAND [ARGS]... + +Options: + --debug / --no-debug Enable debug logging. + --log-file FILE Enable debug logging. + --debug-vars / --no-debug-vars Show variable values in stack traces. Implies --debug. While we try to avoid + printing sensitive information like passwords, this may still happen. + --version Show the version and exit. + -dl, --detect-memory-leaks Run memory leak detection. + --help Show this message and exit. + +Commands: + actions + check Helper commands for checking various aspects of DataHub. + container A group of commands to interact with containers in DataHub. + dataproduct A group of commands to interact with the DataProduct entity in DataHub. + dataset A group of commands to interact with the Dataset entity in DataHub. + delete Delete metadata from DataHub. + docker Helper commands for setting up and interacting with a local DataHub instance using Docker. + exists A group of commands to check existence of entities in DataHub. + forms A group of commands to interact with forms in DataHub. + get A group of commands to get metadata from DataHub. + graphql Execute GraphQL queries and mutations against DataHub. + group A group of commands to interact with the Group entity in DataHub. + ingest Ingest metadata into DataHub. + init Configure which datahub instance to connect to + lite A group of commands to work with a DataHub Lite instance + migrate Helper commands for migrating metadata within DataHub. + properties A group of commands to interact with structured properties in DataHub. + put A group of commands to put metadata in DataHub. + state Managed state stored in DataHub by stateful ingestion. + telemetry Toggle telemetry. + timeline Get timeline for an entity based on certain categories + user A group of commands to interact with the User entity in DataHub. + version Print version number and exit. +``` + +The following top-level commands listed below are here mainly to give the reader a high-level picture of what are the kinds of things you can accomplish with the cli. +We've ordered them roughly in the order we expect you to interact with these commands as you get deeper into the `datahub`-verse. + +### docker + +The `docker` command allows you to start up a local DataHub instance using `datahub docker quickstart`. You can also check if the docker cluster is healthy using `datahub docker check`. + +### version + +The `version` command allows you to get version number of CLI that you are using. + +```console +datahub version +DataHub CLI version: 1.2.0.9 +Models: bundled +Python version: 3.11.11 (main, Mar 17 2025, 21:33:08) [Clang 20.1.0 ] +``` + +You can pass `--include-server` flag to include server information too. This is helpful to share cli and server version at once for debugging purposes. This does require `datahub init` to be run before that so there is a server that CLI is able to connect to. + +```console +datahub version --include-server +DataHub CLI version: 1.2.0.9 +Models: bundled +Python version: 3.11.11 (main, Mar 17 2025, 21:33:08) [Clang 20.1.0 ] +Server config: {'models': {}, 'managedIngestion': {'defaultCliVersion': '1.2.0.9', 'enabled': True}, 'timeZone': 'GMT', 'datasetUrnNameCasing': False, 'datahub': {'serverEnv': 'cloud', 'serverType': 'prod'}, 'baseUrl': 'https://xyz.acryl.io', 'patchCapable': True, 'versions': {'acryldata/datahub': {'version': 'v0.3.14rc0', 'commit': '464d94926bb21a596f3d9d81164b272c74872ce7'}}, 'statefulIngestionCapable': True, 'remoteExecutorBackend': {'revision': 2}, 'supportsImpactAnalysis': True, 'telemetry': {'enabledCli': False, 'enabledMixpanel': False, 'enabledServer': True, 'enabledIngestion': True}, 'retention': 'true', 'noCode': 'true'} +``` + +### ingest + +The `ingest` command allows you to ingest metadata from your sources using ingestion configuration files, which we call recipes. +Source specific crawlers are provided by plugins and might sometimes need additional extras to be installed. See [installing plugins](#installing-plugins) for more information. +[Removing Metadata from DataHub](./how/delete-metadata.md) contains detailed instructions about how you can use the ingest command to perform operations like rolling-back previously ingested metadata through the `rollback` sub-command and listing all runs that happened through `list-runs` sub-command. + +```console +Usage: datahub [datahub-options] ingest [command-options] + +Command Options: + -c / --config Config file in .toml or .yaml format + -n / --dry-run Perform a dry run of the ingestion, essentially skipping writing to sink + --preview Perform limited ingestion from the source to the sink to get a quick preview + --preview-workunits The number of workunits to produce for preview + --strict-warnings If enabled, ingestion runs with warnings will yield a non-zero error code + --test-source-connection When set, ingestion will only test the source connection details from the recipe + --no-progress If enabled, mute intermediate progress ingestion reports +``` + +#### ingest --dry-run + +The `--dry-run` option of the `ingest` command performs all of the ingestion steps, except writing to the sink. This is useful to validate that the +ingestion recipe is producing the desired metadata events before ingesting them into datahub. + +```shell +# Dry run +datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --dry-run +# Short-form +datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n +``` + +#### ingest list-source-runs + +The `list-source-runs` option of the `ingest` command lists the previous runs, displaying their run ID, source name, +start time, status, and source URN. This command allows you to filter results using the --urn option for URN-based +filtering or the --source option to filter by source name (partial or complete matches are supported). + +```shell +# List all ingestion runs +datahub ingest list-source-runs +# Filter runs by a source name containing "demo" +datahub ingest list-source-runs --source "demo" +``` + +#### ingest --preview + +The `--preview` option of the `ingest` command performs all of the ingestion steps, but limits the processing to only the first 10 workunits produced by the source. +This option helps with quick end-to-end smoke testing of the ingestion recipe. + +```shell +# Preview +datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --preview +# Preview with dry-run +datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n --preview +``` + +By default `--preview` creates 10 workunits. But if you wish to try producing more workunits you can use another option `--preview-workunits` + +```shell +# Preview 20 workunits without sending anything to sink +datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml -n --preview --preview-workunits=20 +``` + +#### ingest --no-default-report + +By default, the cli sends an ingestion report to DataHub, which allows you to see the result of all cli-based ingestion in the UI. This can be turned off with the `--no-default-report` flag. + +```shell +# Running ingestion with reporting to DataHub turned off +datahub ingest -c ./examples/recipes/example_to_datahub_rest.dhub.yaml --no-default-report +``` + +The reports include the recipe that was used for ingestion. This can be turned off by adding an additional section to the ingestion recipe. + +```yaml +source: + # source configs + +sink: + # sink configs + +# Add configuration for the datahub reporter +reporting: + - type: datahub + config: + report_recipe: false + +# Optional log to put failed JSONs into a file +# Helpful in case you are trying to debug some issue with specific ingestion failing +failure_log: + enabled: false + log_config: + filename: ./path/to/failure.json +``` + +#### ingest --record (Beta) + +:::note Beta Feature +Recording and replay is currently in beta. The feature is stable for debugging purposes but the archive format may change in future releases. +::: + +The `--record` option enables recording of all HTTP requests and database queries during ingestion. This creates an encrypted archive that can be replayed offline for debugging. + +```shell +# Record an ingestion run with password protection +datahub ingest -c ./recipe.yaml --record --record-password mysecret + +# Record to a specific local directory +export INGESTION_ARTIFACT_DIR=/path/to/recordings +datahub ingest -c ./recipe.yaml --record --record-password mysecret --no-s3-upload + +# Record and upload directly to S3 +datahub ingest -c ./recipe.yaml --record --record-password mysecret \ + --record-output-path s3://my-bucket/recordings/my-run.zip +``` + +Recording options: + +| Option | Description | +| ----------------------- | ---------------------------------------------------------------------------------------------------------- | +| `--record` | Enable recording of the ingestion run | +| `--record-password` | Password for encrypting the archive. Can also be set via `DATAHUB_RECORDING_PASSWORD` environment variable | +| `--record-output-path` | Path to save the recording archive. Use local path or S3 URL (`s3://bucket/path/file.zip`) | +| `--no-s3-upload` | Disable S3 upload (save locally only) | +| `--no-secret-redaction` | Keep actual credentials in recording (use with caution, for local debugging only) | + +The recording creates an encrypted ZIP archive containing HTTP cassettes, database query recordings, and a redacted recipe. This archive can be replayed using `datahub ingest replay`. + +**Installation:** Recording requires the `debug-recording` plugin: + +```shell +pip install 'acryl-datahub[debug-recording]' +``` + +➡️ [Learn more about recording and debugging ingestion](./how/debug-ingestion-recording.md) + +### ingest replay (Beta) + +The `ingest replay` command replays a recorded ingestion run for debugging. This allows you to reproduce issues in an air-gapped environment without network access. + +```shell +# Replay from local file +datahub ingest replay ./recording.zip --password mysecret + +# Replay from S3 +datahub ingest replay s3://bucket/recordings/run-id.zip --password mysecret + +# Replay with live sink (emit to real DataHub instance) +datahub ingest replay ./recording.zip --password mysecret --live-sink --server http://localhost:8080 +``` + +Replay options: + +| Option | Description | +| ------------- | ---------------------------------------------------------------------------------------------------------- | +| `--password` | Password for decrypting the archive. Can also be set via `DATAHUB_RECORDING_PASSWORD` environment variable | +| `--live-sink` | Emit to real GMS server instead of using recorded responses | +| `--server` | GMS server URL when using `--live-sink` | +| `--report-to` | Path to write the report file | + +### recording (Beta) + +The `recording` command group provides utilities for working with recording archives. + +```shell +# View archive metadata +datahub recording info recording.zip --password mysecret + +# Extract archive contents +datahub recording extract recording.zip --password mysecret --output-dir ./extracted + +# List archive contents +datahub recording list recording.zip --password mysecret +``` + +#### recording info + +Display metadata about a recording archive including run ID, source type, creation time, and whether an exception was captured. + +```shell +datahub recording info recording.zip --password mysecret +``` + +Use `--json` for machine-readable output. + +#### recording extract + +Extract a recording archive to inspect its contents: + +```shell +datahub recording extract recording.zip --password mysecret --output-dir ./extracted +``` + +#### recording list + +List the files contained in a recording archive: + +```shell +datahub recording list recording.zip --password mysecret +``` + +### ingest deploy + +The `ingest deploy` command instructs the cli to upload an ingestion recipe to DataHub to be run by DataHub's [UI Ingestion](./ui-ingestion.md). +This command can also be used to schedule the ingestion while uploading or even to update existing sources. It will upload to the remote instance the +CLI is connected to, not the sink of the recipe. Use `datahub init` to set the remote if not already set. + +This command will automatically create a new recipe if it doesn't exist, or update it if it does. +Note that this is a complete update, and will remove any options that were previously set. +I.e: Not specifying a schedule in the cli update command will remove the schedule from the recipe to be updated. + +#### Command Options + +```console +Usage: datahub ingest deploy [OPTIONS] + +Options: + -n, --name TEXT Recipe Name + -c, --config FILE Config file in .toml or .yaml format. [required] + --urn TEXT Urn of recipe to update. If not specified here or in the recipe's pipeline_name, + this will create a new ingestion source. + --executor-id TEXT Executor id to route execution requests to. Do not use this unless you have + configured a custom executor. + --cli-version TEXT Provide a custom CLI version to use for ingestion. By default will use server + default. + --schedule TEXT Cron definition for schedule. If none is provided, ingestion recipe will not be + scheduled + --time-zone TEXT Timezone for the schedule in 'America/New_York' format. Uses UTC by default. + --debug BOOLEAN Should we debug. + --extra-pip TEXT Extra pip packages. e.g. ["memray"] + --extra-env TEXT Environment variables as comma-separated KEY=VALUE pairs. + e.g. "VAR1=value1,VAR2=value2" +``` + +#### Examples + +**Schedule a recipe with default executor:** + +```shell +datahub ingest deploy --name "Snowflake Integration" --schedule "0 5 * * *" --time-zone "Europe/London" -c recipe.yaml +``` + +**Deploy to a specific remote executor:** + +```shell +datahub ingest deploy --name "Remote Snowflake Integration" --executor-id "remote-executor-pool-1" --schedule "0 5 * * *" -c recipe.yaml +``` + +**Update an existing recipe:** + +```shell +datahub ingest deploy --urn "urn:li:dataHubIngestionSource:deploy-12345678" --schedule "0 6 * * *" -c updated_recipe.yaml +``` + +**Deploy with environment variables:** + +```shell +datahub ingest deploy --name "Snowflake Integration" --extra-env "SNOWFLAKE_WAREHOUSE=COMPUTE_WH,SNOWFLAKE_ROLE=ANALYST" -c recipe.yaml +``` + +By default, the ingestion recipe's identifier is generated by hashing the name. +You can override the urn generation by passing the `--urn` flag to the CLI. + +#### Remote Executors + +The `--executor-id` option allows you to route ingestion execution to specific executors: + +- **Default executor** (`"default"`): Uses the managed executor provided by DataHub Cloud or your configured default executor +- **Remote executors**: Route to custom remote executors you've deployed in your environment + +:::note +Use executor IDs other than "default" only if you have configured custom remote executors. +::: + +**Examples with remote executors:** + +```shell +# Deploy to default executor (can be configured as remote) +datahub ingest deploy --name "My Integration" -c recipe.yaml + +# Deploy to specific remote executor pool +datahub ingest deploy --name "Private Network Integration" --executor-id "private-network-pool" -c recipe.yaml + +# Deploy to region-specific executor +datahub ingest deploy --name "EU Region Integration" --executor-id "eu-west-executor" -c recipe.yaml +``` + +For more information on setting up remote executors, see the [Remote Executor Setup Guide](managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md). + +#### Using deployment section + +As an alternative to configuring settings from the CLI, they can also be set in the `deployment` field of the recipe. + +```yml +# deployment_recipe.yml +deployment: + name: "Snowflake Integration" + schedule: "0 5 * * *" + time_zone: "Europe/London" + executor_id: "remote-executor-pool-1" # Optional: specify remote executor + cli_version: "0.15.0.1" # Optional: specify CLI version + extra_pip: '["polars==1.35.2"]' + extra_env: "VAR1=value1,VAR2=value2" + +source: ... +``` + +```shell +datahub ingest deploy -c deployment_recipe.yml +``` + +CLI options will override corresponding values in the deployment section. + +#### Deployment Configuration Options + +These deployment options that can be specified via CLI flags can also be configured in the `deployment` section: + +| Field | CLI Option | Description | Default | +| ------------- | --------------- | ------------------------------- | ------------------ | +| `name` | `--name` | Recipe name displayed in the UI | Required | +| `schedule` | `--schedule` | Cron expression for scheduling | None (manual only) | +| `time_zone` | `--time-zone` | Timezone for scheduled runs | `"UTC"` | +| `executor_id` | `--executor-id` | Target executor for ingestion | `"default"` | +| `cli_version` | `--cli-version` | CLI version for ingestion | Server default | +| `extra_pip` | `--extra-pip` | Extra pip packages | None | +| `extra_env` | `--extra-env` | Extra environment variables | None | + +#### Batch Deployment + +Deploy multiple recipes from version control: + +```shell +# Deploy every yml recipe in a directory +ls recipe_directory/*.yml | xargs -n 1 -I {} datahub ingest deploy -c {} + +# Deploy with consistent executor across all recipes +ls recipe_directory/*.yml | xargs -n 1 -I {} datahub ingest deploy --executor-id "production-executor" -c {} +``` + +### init + +The init command is used to tell `datahub` about where your DataHub instance is located. The CLI will point to localhost DataHub by default. + +**_Note_**: Provide your GMS instance's host when the prompt asks you for the DataHub host. + +#### Interactive Mode + +```shell +# Interactive mode - prompts for input +datahub init +/Users/user/.datahubenv already exists. Overwrite? [y/N]: y +Configure which datahub instance to connect to +Enter your DataHub host [http://localhost:8080]: http://localhost:8080 +Enter your DataHub access token []: +``` + +#### Non-Interactive Mode with Username/Password + +The CLI can automatically generate tokens from your username and password credentials. This is useful for quickstart instances, automation, and CI/CD pipelines. + +```shell +# Quickstart (local instance with default credentials) +datahub init --username datahub --password datahub + +# Custom credentials with longer token duration +datahub init --username alice --password secret --token-duration ONE_MONTH + +# For long-running jobs or CI/CD +datahub init --username alice --password secret --token-duration ONE_WEEK +``` + +#### DataHub Cloud Example + +For DataHub Cloud (Acryl-hosted) instances, you can use an existing token: + +```shell +# Interactive +datahub init +Enter your DataHub host [http://localhost:8080]: https://.acryl.io/gms +Enter your DataHub access token []: .acryl.io/settings/tokens> + +# Non-interactive +datahub init --host https://.acryl.io/gms --token +``` + +#### Environment variables supported + +The environment variables listed below take precedence over the DataHub CLI config created through the `init` command. + +- `DATAHUB_SKIP_CONFIG` (default `false`) - Set to `true` to skip creating the configuration file. +- `DATAHUB_GMS_URL` (default `http://localhost:8080`) - Set to a URL of GMS instance +- `DATAHUB_GMS_HOST` (default `localhost`) - Set to a host of GMS instance. Prefer using `DATAHUB_GMS_URL` to set the URL. +- `DATAHUB_GMS_PORT` (default `8080`) - Set to a port of GMS instance. Prefer using `DATAHUB_GMS_URL` to set the URL. +- `DATAHUB_GMS_PROTOCOL` (default `http`) - Set to a protocol like `http` or `https`. Prefer using `DATAHUB_GMS_URL` to set the URL. +- `DATAHUB_GMS_TOKEN` (default `None`) - Used for communicating with DataHub Cloud. +- `DATAHUB_USERNAME` (default `None`) - Username for generating access tokens via `datahub init`. Used with `DATAHUB_PASSWORD` for non-interactive authentication. +- `DATAHUB_PASSWORD` (default `None`) - Password for generating access tokens via `datahub init`. Used with `DATAHUB_USERNAME` for non-interactive authentication. +- `DATAHUB_TELEMETRY_ENABLED` (default `true`) - Set to `false` to disable telemetry. If CLI is being run in an environment with no access to public internet then this should be disabled. +- `DATAHUB_TELEMETRY_TIMEOUT` (default `10`) - Set to a custom integer value to specify timeout in secs when sending telemetry. +- `DATAHUB_DEBUG` (default `false`) - Set to `true` to enable debug logging for CLI. Can also be achieved through `--debug` option of the CLI. This exposes sensitive information in logs, enabling on production instances should be avoided especially if UI ingestion is in use as logs can be made available for runs through the UI. +- `DATAHUB_VERSION` (default `head`) - Set to a specific version to run quickstart with the particular version of docker images. +- `ACTIONS_VERSION` (default `head`) - Set to a specific version to run quickstart with that image tag of `datahub-actions` container. +- `DATAHUB_ACTIONS_IMAGE` (default `acryldata/datahub-actions`) - Set to `-slim` to run a slimmer actions container without pyspark/deequ features. +- `DATAHUB_RECORDING_PASSWORD` - Password for encrypting/decrypting recording archives. Used by `--record` and `--replay` commands. +- `INGESTION_ARTIFACT_DIR` - Directory to save recordings when S3 upload is disabled. If not set, recordings are saved to a temp directory. +- `DATAHUB_REPORT_FAILURE_SAMPLE_SIZE` (default `10`) - Maximum number of failure entries to include in the ingestion report. Increase to capture more failure samples when debugging. +- `DATAHUB_REPORT_WARNING_SAMPLE_SIZE` (default `10`) - Maximum number of warning entries to include in the ingestion report. Increase to capture more warning samples when debugging. + +```shell +DATAHUB_SKIP_CONFIG=false +DATAHUB_GMS_URL=http://localhost:8080 +DATAHUB_GMS_TOKEN= +DATAHUB_TELEMETRY_ENABLED=true +DATAHUB_TELEMETRY_TIMEOUT=10 +DATAHUB_DEBUG=false +DATAHUB_REPORT_FAILURE_SAMPLE_SIZE=10 +DATAHUB_REPORT_WARNING_SAMPLE_SIZE=10 +``` + +### container + +A group of commands to interact with containers in DataHub. + +e.g. You can use this to apply a tag to all datasets recursively in this container. + +```shell +datahub container tag --container-urn "urn:li:container:0e9e46bd6d5cf645f33d5a8f0254bc2d" --tag-urn "urn:li:tag:tag1" +datahub container domain --container-urn "urn:li:container:3f2effd1fbe154a4d60b597263a41e41" --domain-urn "urn:li:domain:ajsajo-b832-4ab3-8881-7ed5e991a44c" +datahub container owner --container-urn "urn:li:container:3f2effd1fbe154a4d60b597263a41e41" --owner-urn "urn:li:corpGroup:eng@example.com" +datahub container term --container-urn "urn:li:container:3f2effd1fbe154a4d60b597263a41e41" --term-urn "urn:li:term:PII" +``` + +### check + +The datahub package is composed of different plugins that allow you to connect to different metadata sources and ingest metadata from them. +The `check` command allows you to check if all plugins are loaded correctly as well as validate an individual MCE-file. + +#### restore-indices + +This command allows you to restore indices for one or more `urn`. + +```shell +datahub --debug check restore-indices --urn "URN" +``` + +It can also take `--file` argument that points to a file that has list of urns like + +```shell +datahub check restore-indices --file ./urn.txt +``` + +where urn.txt is like this + +```urn.txt +urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.schema1.test_all_nulls,PROD) +urn:li:dataset:(urn:li:dataPlatform:platform1,test_db.schema2.test_complex_types,PROD) +urn:li:dataset:(urn:li:dataPlatform:redshift,test_db.schema3.test_few_rows,PROD) +``` + +#### get-kafka-consumer-offsets + +This required DataHub Cloud `0.3.12.x` or above version. + +```shell +datahub check get-kafka-consumer-offsets +``` + +which can give can print out the details of lag in kafka topics. + +``` +{'mcl': {'generic-mae-consumer-job-client': {'MetadataChangeLog_Versioned_v1': {'metrics': {'avgLag': 36, + 'maxLag': 36, + 'medianLag': 36, + 'totalLag': 36}, + 'partitions': {'0': {'lag': 36, + 'offset': 257318}}}}}, + 'mcl-timeseries': {'generic-mae-consumer-job-client': {'MetadataChangeLog_Timeseries_v1': {'metrics': {'avgLag': 0, + 'maxLag': 0, + 'medianLag': 0, + 'totalLag': 0}, + 'partitions': {'0': {'lag': 0, + 'offset': 113}}}}}, + 'mcp': {'generic-mce-consumer-job-client': {'MetadataChangeProposal_v1': {'metrics': {'avgLag': 7222149, + 'maxLag': 7222149, + 'medianLag': 7222149, + 'totalLag': 7222149}, + 'partitions': {'0': {'lag': 7222149, + 'offset': 250254}}}}}} +``` + +### delete + +The `delete` command allows you to delete metadata from DataHub. + +The [metadata deletion guide](./how/delete-metadata.md) covers the various options for the delete command. + +### exists + +The exists command can be used to check if an entity exists in DataHub. + +```shell +> datahub exists --urn "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" +true +> datahub exists --urn "urn:li:dataset:(urn:li:dataPlatform:hive,NonExistentHiveDataset,PROD)" +false +``` + +### get + +The `get` command allows you to easily retrieve metadata from DataHub, by using the REST API. This works for both versioned aspects and timeseries aspects. For timeseries aspects, it fetches the latest value. +For example the following command gets the ownership aspect from the dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)` + +```shell-session +$ datahub get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" --aspect ownership +{ + "ownership": { + "lastModified": { + "actor": "urn:li:corpuser:jdoe", + "time": 1680210917580 + }, + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "source": { + "type": "SERVICE" + }, + "type": "NONE" + } + ] + } +} +``` + +### graphql + +The `graphql` command allows you to execute GraphQL queries and mutations against DataHub's GraphQL API. This provides full access to DataHub's metadata through its native GraphQL interface. + +```shell +# Execute a GraphQL query +datahub graphql --query "query { me { username } }" + +# Use named operations from DataHub's schema +datahub graphql --operation searchAcrossEntities --variables '{"input": {"query": "users"}}' + +# List available operations +datahub graphql --list-operations + +# Get help for a specific operation +datahub graphql --describe searchAcrossEntities + +# Explore types recursively +datahub graphql --describe SearchInput --recurse + +# Load queries and variables from files +datahub graphql --query ./search-tags.graphql --variables ./search-params.json + +# Get JSON output for LLM integration +datahub graphql --list-operations --format json +``` + +The GraphQL command supports both raw GraphQL queries/mutations and operation-based execution using DataHub's introspected schema. It automatically detects whether `--query` and `--variables` arguments are file paths or literal content, enabling seamless use of both inline GraphQL and file-based queries. + +Key features: + +- **Schema discovery**: List and describe all available operations and types +- **File support**: Load queries and variables from `.graphql` and `.json` files +- **LLM-friendly output**: JSON format with complete type information +- **Recursive exploration**: Deep-dive into complex GraphQL types + +➡️ [Learn more about the GraphQL command](./cli-commands/graphql.md) + +### search + +The `search` command allows you to search across all DataHub entities from the command line with support for both keyword and semantic search. + +```shell +# Basic keyword search +datahub search "users" + +# Semantic search +datahub search --semantic "financial reports" + +# Search with filters +datahub search "*" --filter platform=snowflake --filter entity_type=dataset + +# Get results in table format +datahub search "*" --table +``` + +Key features: + +- **Dual search modes**: Keyword search (default) and semantic search +- **Flexible filtering**: Simple key=value filters and complex JSON filters with AND/OR/NOT logic +- **Multiple output formats**: JSON (default), table, and URNs-only +- **Discovery features**: List available filters and describe specific filters +- **Faceted search**: Explore data distribution with aggregation counts +- **Pagination & sorting**: Control result size and ordering + +➡️ [Learn more about the search command](./cli-commands/search.md) + +### put + +The `put` group of commands allows you to write metadata into DataHub. This is a flexible way for you to issue edits to metadata from the command line. + +#### put aspect + +The **put aspect** (also the default `put`) command instructs `datahub` to set a specific aspect for an entity to a specified value. +For example, the command shown below sets the `ownership` aspect of the dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)` to the value in the file `ownership.json`. +The JSON in the `ownership.json` file needs to conform to the [`Ownership`](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl) Aspect model as shown below. + +```json +{ + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DEVELOPER" + }, + { + "owner": "urn:li:corpuser:jdub", + "type": "DATAOWNER" + } + ] +} +``` + +```console +datahub --debug put --urn "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" --aspect ownership -d ownership.json + +[DATE_TIMESTAMP] DEBUG {datahub.cli.cli_utils:340} - Attempting to emit to DataHub GMS; using curl equivalent to: +curl -X POST -H 'User-Agent: python-requests/2.26.0' -H 'Accept-Encoding: gzip, deflate' -H 'Accept: */*' -H 'Connection: keep-alive' -H 'X-RestLi-Protocol-Version: 2.0.0' -H 'Content-Type: application/json' --data '{"proposal": {"entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", "aspectName": "ownership", "changeType": "UPSERT", "aspect": {"contentType": "application/json", "value": "{\"owners\": [{\"owner\": \"urn:li:corpuser:jdoe\", \"type\": \"DEVELOPER\"}, {\"owner\": \"urn:li:corpuser:jdub\", \"type\": \"DATAOWNER\"}]}"}}}' 'http://localhost:8080/aspects/?action=ingestProposal' +Update succeeded with status 200 +``` + +#### put platform + +**🤝 Version Compatibility:** `acryl-datahub>0.8.44.4` + +The **put platform** command instructs `datahub` to create or update metadata about a data platform. This is very useful if you are using a custom data platform, to set up its logo and display name for a native UI experience. + +```shell +datahub put platform --name longtail_schemas --display_name "Long Tail Schemas" --logo "https://flink.apache.org/img/logo/png/50/color_50.png" +✅ Successfully wrote data platform metadata for urn:li:dataPlatform:longtail_schemas to DataHub (DataHubRestEmitter: configured to talk to https://longtailcompanions.acryl.io/api/gms with token: eyJh**********Cics) +``` + +### timeline + +The `timeline` command allows you to view a version history for entities. Currently only supported for Datasets. For example, +the following command will show you the modifications to tags for a dataset for the past week. The output includes a computed semantic version, +relevant for schema changes only currently, the target of the modification, and a description of the change including a timestamp. +The default output is sanitized to be more readable, but the full API output can be obtained by passing the `--verbose` flag and +to get the raw JSON difference in addition to the API output you can add the `--raw` flag. For more details about the feature please see [the main feature page](dev-guides/timeline.md) + +```console +datahub timeline --urn "urn:li:dataset:(urn:li:dataPlatform:mysql,User.UserAccount,PROD)" --category TAG --start 7daysago +2022-02-17 14:03:42 - 0.0.0-computed + MODIFY TAG dataset:mysql:User.UserAccount : A change in aspect editableSchemaMetadata happened at time 2022-02-17 20:03:42.0 +2022-02-17 14:17:30 - 0.0.0-computed + MODIFY TAG dataset:mysql:User.UserAccount : A change in aspect editableSchemaMetadata happened at time 2022-02-17 20:17:30.118 +``` + +## Entity Specific Commands + +### dataset (Dataset Entity) + +The `dataset` command allows you to interact with Dataset entities in DataHub, including creating, updating, retrieving, and validating Dataset metadata. + +```shell +# Get a dataset and write to YAML file +datahub dataset get --urn "urn:li:dataset:(urn:li:dataPlatform:hive,example_table,PROD)" --to-file dataset.yaml + +# Create or update dataset from YAML file +datahub dataset upsert -f dataset.yaml +``` + +➡️ [Learn more about the dataset command](./cli-commands/dataset.md) + +### user (User Entity) + +The `user` command allows you to interact with the User entity in DataHub. It supports two main operations: + +#### upsert + +Create or update users from a YAML file. For detailed information, please refer to [Creating Users and Groups with DataHub CLI](/docs/api/tutorials/owners.md#upsert-users). + +```shell +datahub user upsert -f users.yaml +``` + +An example of `users.yaml` would look like as in [bar.user.dhub.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/user/bar.user.dhub.yaml) file for the complete code. + +```yaml +- id: bar@acryl.io + first_name: The + last_name: Bar + email: bar@acryl.io + slack: "@the_bar_raiser" + description: "I like raising the bar higher" + groups: + - foogroup@acryl.io +- id: datahub + slack: "@datahubproject" + phone: "1-800-GOT-META" + description: "The DataHub Project" + picture_link: "https://raw.githubusercontent.com/datahub-project/datahub/master/datahub-web-react/src/images/datahub-logo-color-stable.svg" +``` + +#### add + +Create a native DataHub user with email/password authentication. This command creates users who can log in directly to DataHub using their email and password. + +```shell +# Create a user with a role +datahub user add --email user@example.com --display-name "John Doe" --password --role Admin + +# Create a user without a role +datahub user add --email user@example.com --display-name "Jane Smith" --password +``` + +**Options:** + +- `--email` (required): User's email address, which will be used as their login ID +- `--display-name` (required): User's full display name +- `--password` (required): Flag to prompt for password input (password will be hidden during entry) +- `--role` (optional): Role to assign to the user. Valid values are `Admin`, `Editor`, or `Reader` (case-insensitive) + +**Notes:** + +- The command will check if a user with the specified email already exists and exit if found +- Passwords are entered securely via a hidden prompt and require confirmation +- If role assignment fails, the user will still be created but without the role +- Requires admin permissions to execute + +### group (Group Entity) + +The `group` command allows you to interact with the Group entity. +It currently supports the `upsert` operation, which can be used to create a new group or update an existing one with embedded Users. +For more information, please refer to [Creating Users and Groups with DataHub CLI](/docs/api/tutorials/owners.md#upsert-users). + +```shell +datahub group upsert -f group.yaml +``` + +An example of `group.yaml` would look like as in [foo.group.dhub.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/cli_usage/group/foo.group.dhub.yaml) file for the complete code. + +```yaml +id: foogroup@acryl.io +display_name: Foo Group +admins: + - datahub +members: + - bar@acryl.io # refer to a user either by id or by urn + - id: joe@acryl.io # inline specification of user + slack: "@joe_shmoe" + display_name: "Joe's Hub" +``` + +### dataproduct (Data Product Entity) + +**🤝 Version Compatibility:** `acryl-datahub>=0.10.2.4` + +The dataproduct group of commands allows you to manage the lifecycle of a DataProduct entity on DataHub. +See the [Data Products](./dataproducts.md) page for more details on what a Data Product is and how DataHub represents it. + +```shell +datahub dataproduct --help +Commands: + upsert* Upsert attributes to a Data Product in DataHub + update Create or Update a Data Product in DataHub. + add_asset Add an asset to a Data Product + add_owner Add an owner to a Data Product + delete Delete a Data Product in DataHub. + diff Diff a Data Product file with its twin in DataHub + get Get a Data Product from DataHub + remove_asset Add an asset to a Data Product + remove_owner Remove an owner from a Data Product + set_description Set description for a Data Product in DataHub +``` + +Here we detail the sub-commands available under the dataproduct group of commands: + +#### upsert + +Use this to upsert a data product yaml file into DataHub. This will create the data product if it doesn't exist already. Remember, this will upsert all the fields that are specified in the yaml file and will not touch the fields that are not specified. For example, if you do not specify the `description` field in the yaml file, then `upsert` will not modify the description field on the Data Product entity in DataHub. To keep this file sync-ed with the metadata on DataHub use the [diff](#diff) command. The format of the yaml file is available [here](./dataproducts.md#creating-a-data-product-yaml--git). + +```shell +# Usage +> datahub dataproduct upsert -f data_product.yaml + +``` + +#### update + +Use this to fully replace a data product's metadata in DataHub from a yaml file. This will create the data product if it doesn't exist already. Remember, this will update all the fields including ones that are not specified in the yaml file. For example, if you do not specify the `description` field in the yaml file, then `update` will set the description field on the Data Product entity in DataHub to empty. To keep this file sync-ed with the metadata on DataHub use the [diff](#diff) command. The format of the yaml file is available [here](./dataproducts.md#creating-a-data-product-yaml--git). + +```shell +# Usage +> datahub dataproduct upsert -f data_product.yaml + +``` + +:::note + +❗**Pro-Tip: upsert versus update** + +Wondering which command is right for you? Use `upsert` if there are certain elements of metadata that you don't want to manage using the yaml file (e.g. owners, assets or description). Use `update` if you want to manage the entire data product's metadata using the yaml file. + +::: + +#### diff + +Use this to keep a data product yaml file updated from its server-side version in DataHub. + +```shell +# Usage +> datahub dataproduct diff -f data_product.yaml --update +``` + +#### get + +Use this to get a data product entity from DataHub and optionally write it to a yaml file + +```shell +# Usage +> datahub dataproduct get --urn urn:li:dataProduct:pet_of_the_week --to-file pet_of_the_week_dataproduct.yaml +{ + "id": "urn:li:dataProduct:pet_of_the_week", + "domain": "urn:li:domain:dcadded3-2b70-4679-8b28-02ac9abc92eb", + "assets": [ + "urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.pet_details,PROD)", + "urn:li:dashboard:(looker,dashboards.19)", + "urn:li:dataFlow:(airflow,snowflake_load,prod)" + ], + "display_name": "Pet of the Week Campaign", + "owners": [ + { + "id": "urn:li:corpuser:jdoe", + "type": "BUSINESS_OWNER" + } + ], + "description": "This campaign includes Pet of the Week data.", + "tags": [ + "urn:li:tag:adoption" + ], + "terms": [ + "urn:li:glossaryTerm:ClientsAndAccounts.AccountBalance" + ], + "properties": { + "lifecycle": "production", + "sla": "7am every day" + } +} +Data Product yaml written to pet_of_the_week_dataproduct.yaml +``` + +#### add_asset + +Use this to add a data asset to a Data Product. + +```shell +# Usage +> datahub dataproduct add_asset --urn "urn:li:dataProduct:pet_of_the_week" --asset "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" +``` + +#### remove_asset + +Use this to remove a data asset from a Data Product. + +```shell +# Usage +> datahub dataproduct remove_asset --urn "urn:li:dataProduct:pet_of_the_week" --asset "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)" +``` + +#### add_owner + +Use this to add an owner to a Data Product. + +```shell +# Usage +> datahub dataproduct add_owner --urn "urn:li:dataProduct:pet_of_the_week" --owner "jdoe@longtail.com" --owner-type BUSINESS_OWNER +``` + +#### remove_owner + +Use this to remove an owner from a Data Product. + +```shell +# Usage +> datahub dataproduct remove_owner --urn "urn:li:dataProduct:pet_of_the_week" --owner "urn:li:corpUser:jdoe@longtail.com" +``` + +#### set_description + +Use this to attach rich documentation for a Data Product in DataHub. + +```shell +> datahub dataproduct set_description --urn "urn:li:dataProduct:pet_of_the_week" --description "This is the pet dataset" +# For uploading rich documentation from a markdown file, use the --md-file option +# > datahub dataproduct set_description --urn "urn:li:dataProduct:pet_of_the_week" --md-file ./pet_of_the_week.md +``` + +#### delete + +Use this to delete a Data Product from DataHub. Default to `--soft` which preserves metadata, use `--hard` to erase all metadata associated with this Data Product. + +```shell +> datahub dataproduct delete --urn "urn:li:dataProduct:pet_of_the_week" +# For Hard Delete see below: +# > datahub dataproduct delete --urn "urn:li:dataProduct:pet_of_the_week" --hard +``` + +## Miscellaneous Admin Commands + +### lite (experimental) + +The lite group of commands allow you to run an embedded, lightweight DataHub instance for command line exploration of your metadata. This is intended more for developer tool oriented usage rather than as a production server instance for DataHub. See [DataHub Lite](./datahub_lite.md) for more information about how you can ingest metadata into DataHub Lite and explore your metadata easily. + +### telemetry + +To help us understand how people are using DataHub, we collect anonymous usage statistics on actions such as command invocations via Mixpanel. +We do not collect private information such as IP addresses, contents of ingestions, or credentials. +The code responsible for collecting and broadcasting these events is open-source and can be found [within our GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/telemetry/telemetry.py). + +Telemetry is enabled by default, and the `telemetry` command lets you toggle the sending of these statistics via `telemetry enable/disable`. + +### migrate + +The `migrate` group of commands allows you to perform certain kinds of migrations. + +#### dataplatform2instance + +The `dataplatform2instance` migration command allows you to migrate your entities from an instance-agnostic platform identifier to an instance-specific platform identifier. If you have ingested metadata in the past for this platform and would like to transfer any important metadata over to the new instance-specific entities, then you should use this command. For example, if your users have added documentation or added tags or terms to your datasets, then you should run this command to transfer this metadata over to the new entities. For further context, read the Platform Instance Guide [here](./platform-instances.md). + +A few important options worth calling out: + +- --dry-run / -n : Use this to get a report for what will be migrated before running +- --force / -F : Use this if you know what you are doing and do not want to get a confirmation prompt before migration is started +- --keep : When enabled, will preserve the old entities and not delete them. Default behavior is to soft-delete old entities. +- --hard : When enabled, will hard-delete the old entities. + +**_Note_**: Timeseries aspects such as Usage Statistics and Dataset Profiles are not migrated over to the new entity instances, you will get new data points created when you re-run ingestion using the `usage` or sources with profiling turned on. + +##### Dry Run + +```console +datahub migrate dataplatform2instance --platform elasticsearch --instance prod_index --dry-run +Starting migration: platform:elasticsearch, instance=prod_index, force=False, dry-run=True +100% (25 of 25) |####################################################################################################################################################################################| Elapsed Time: 0:00:00 Time: 0:00:00 +[Dry Run] Migration Report: +-------------- +[Dry Run] Migration Run Id: migrate-5710349c-1ec7-4b83-a7d3-47d71b7e972e +[Dry Run] Num entities created = 25 +[Dry Run] Num entities affected = 0 +[Dry Run] Num entities migrated = 25 +[Dry Run] Details: +[Dry Run] New Entities Created: {'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datahubretentionindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.schemafieldindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.system_metadata_service_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.tagindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataset_datasetprofileaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlmodelindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlfeaturetableindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datajob_datahubingestioncheckpointaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datahub_usage_event,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataset_operationaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datajobindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataprocessindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.glossarytermindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataplatformindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlmodeldeploymentindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datajob_datahubingestionrunsummaryaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.graph_service_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.datahubpolicyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataset_datasetusagestatisticsaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dashboardindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.glossarynodeindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlfeatureindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.dataflowindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.mlprimarykeyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,prod_index.chartindex_v2,PROD)'} +[Dry Run] External Entities Affected: None +[Dry Run] Old Entities Migrated = {'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataset_datasetusagestatisticsaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlmodelindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlmodeldeploymentindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datajob_datahubingestionrunsummaryaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datahubretentionindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datahubpolicyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataset_datasetprofileaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,glossarynodeindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataset_operationaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,graph_service_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datajobindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlprimarykeyindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dashboardindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datajob_datahubingestioncheckpointaspect_v1,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,tagindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,datahub_usage_event,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,schemafieldindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlfeatureindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataprocessindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataplatformindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,mlfeaturetableindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,glossarytermindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,dataflowindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,chartindex_v2,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:elasticsearch,system_metadata_service_v1,PROD)'} +``` + +##### Real Migration (with soft-delete) + +``` +> datahub migrate dataplatform2instance --platform hive --instance +datahub migrate dataplatform2instance --platform hive --instance warehouse +Starting migration: platform:hive, instance=warehouse, force=False, dry-run=False +Will migrate 4 urns such as ['urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)'] +New urns will look like ['urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.logging_events,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_created,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_deleted,PROD)'] + +Ok to proceed? [y/N]: +... +Migration Report: +-------------- +Migration Run Id: migrate-f5ae7201-4548-4bee-aed4-35758bb78c89 +Num entities created = 4 +Num entities affected = 0 +Num entities migrated = 4 +Details: +New Entities Created: {'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_deleted,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.logging_events,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,warehouse.fct_users_created,PROD)'} +External Entities Affected: None +Old Entities Migrated = {'urn:li:dataset:(urn:li:dataPlatform:hive,logging_events,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_deleted,PROD)', 'urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)'} +``` + +## Alternate Installation Options + +### Using docker + +[![Docker Hub](https://img.shields.io/docker/pulls/acryldata/datahub-ingestion?style=plastic)](https://hub.docker.com/r/acryldata/datahub-ingestion) + +If you don't want to install locally, you can alternatively run metadata ingestion within a Docker container. +We have prebuilt images available on [Docker hub](https://hub.docker.com/r/acryldata/datahub-ingestion). All plugins will be installed and enabled automatically. + +You can use the `datahub-ingestion` docker image as explained in [Docker Images](../docker/README.md). In case you are using Kubernetes you can start a pod with the `datahub-ingestion` docker image, log onto a shell on the pod and you should have the access to datahub CLI in your kubernetes cluster. + +_Limitation: the datahub_docker.sh convenience script assumes that the recipe and any input/output files are accessible in the current working directory or its subdirectories. Files outside the current working directory will not be found, and you'll need to invoke the Docker image directly._ + +```shell +# Assumes the DataHub repo is cloned locally. +./metadata-ingestion/scripts/datahub_docker.sh ingest -c ./examples/recipes/example_to_datahub_rest.yml +``` + +### Install from source + +If you'd like to install from source, see the [developer guide](../metadata-ingestion/developing.md). + +## Installing Plugins + +We use a plugin architecture so that you can install only the dependencies you actually need. Click the plugin name to learn more about the specific source recipe and any FAQs! + +### Sources + +Please see our [Integrations page](/integrations) if you want to filter on the features offered by each source. + +| Plugin Name | Install Command | Provides | +| ---------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | --------------------------------------- | +| [metadata-file](./generated/ingestion/sources/metadata-file.md) | _included by default_ | File source and sink | +| [athena](./generated/ingestion/sources/athena.md) | `pip install 'acryl-datahub[athena]'` | AWS Athena source | +| [bigquery](./generated/ingestion/sources/bigquery.md) | `pip install 'acryl-datahub[bigquery]'` | BigQuery source | +| [datahub-lineage-file](./generated/ingestion/sources/file-based-lineage.md) | _no additional dependencies_ | Lineage File source | +| [datahub-business-glossary](./generated/ingestion/sources/business-glossary.md) | _no additional dependencies_ | Business Glossary File source | +| [dbt](./generated/ingestion/sources/dbt.md) | `pip install 'acryl-datahub[dbt]'` | dbt source | +| [dremio](./generated/ingestion/sources/dremio.md) | `pip install 'acryl-datahub[dremio]'` | Dremio Source | +| [druid](./generated/ingestion/sources/druid.md) | `pip install 'acryl-datahub[druid]'` | Druid Source | +| [feast](./generated/ingestion/sources/feast.md) | `pip install 'acryl-datahub[feast]'` | Feast source (0.26.0) | +| [glue](./generated/ingestion/sources/glue.md) | `pip install 'acryl-datahub[glue]'` | AWS Glue source | +| [hana](./generated/ingestion/sources/hana.md) | `pip install 'acryl-datahub[hana]'` | SAP HANA source | +| [hive](./generated/ingestion/sources/hive.md) | `pip install 'acryl-datahub[hive]'` | Hive source | +| [kafka](./generated/ingestion/sources/kafka.md) | `pip install 'acryl-datahub[kafka]'` | Kafka source | +| [kafka-connect](./generated/ingestion/sources/kafka-connect.md) | `pip install 'acryl-datahub[kafka-connect]'` | Kafka connect source | +| [ldap](./generated/ingestion/sources/ldap.md) | `pip install 'acryl-datahub[ldap]'` ([extra requirements]) | LDAP source | +| [looker](./generated/ingestion/sources/looker.md) | `pip install 'acryl-datahub[looker]'` | Looker source | +| [lookml](./generated/ingestion/sources/looker.md#module-lookml) | `pip install 'acryl-datahub[lookml]'` | LookML source | +| [metabase](./generated/ingestion/sources/metabase.md) | `pip install 'acryl-datahub[metabase]'` | Metabase source | +| [mode](./generated/ingestion/sources/mode.md) | `pip install 'acryl-datahub[mode]'` | Mode Analytics source | +| [mongodb](./generated/ingestion/sources/mongodb.md) | `pip install 'acryl-datahub[mongodb]'` | MongoDB source | +| [mssql](./generated/ingestion/sources/mssql.md) | `pip install 'acryl-datahub[mssql]'` | SQL Server source | +| [mysql](./generated/ingestion/sources/mysql.md) | `pip install 'acryl-datahub[mysql]'` | MySQL source | +| [mariadb](./generated/ingestion/sources/mariadb.md) | `pip install 'acryl-datahub[mariadb]'` | MariaDB source | +| [openapi](./generated/ingestion/sources/openapi.md) | `pip install 'acryl-datahub[openapi]'` | OpenApi Source | +| [oracle](./generated/ingestion/sources/oracle.md) | `pip install 'acryl-datahub[oracle]'` | Oracle source | +| [postgres](./generated/ingestion/sources/postgres.md) | `pip install 'acryl-datahub[postgres]'` | Postgres source | +| [redash](./generated/ingestion/sources/redash.md) | `pip install 'acryl-datahub[redash]'` | Redash source | +| [redshift](./generated/ingestion/sources/redshift.md) | `pip install 'acryl-datahub[redshift]'` | Redshift source | +| [sagemaker](./generated/ingestion/sources/sagemaker.md) | `pip install 'acryl-datahub[sagemaker]'` | AWS SageMaker source | +| [salesforce](./generated/ingestion/sources/salesforce.md) | `pip install 'acryl-datahub[salesforce]'` | Salesforce source | +| [snowflake](./generated/ingestion/sources/snowflake.md) | `pip install 'acryl-datahub[snowflake]'` | Snowflake source | +| [sqlalchemy](./generated/ingestion/sources/sqlalchemy.md) | `pip install 'acryl-datahub[sqlalchemy]'` | Generic SQLAlchemy source | +| [superset](./generated/ingestion/sources/superset.md) | `pip install 'acryl-datahub[superset]'` | Superset source | +| [tableau](./generated/ingestion/sources/tableau.md) | `pip install 'acryl-datahub[tableau]'` | Tableau source | +| [trino](./generated/ingestion/sources/trino.md) | `pip install 'acryl-datahub[trino]'` | Trino source | +| [starburst-trino-usage](./generated/ingestion/sources/trino.md) | `pip install 'acryl-datahub[starburst-trino-usage]'` | Starburst Trino usage statistics source | +| [nifi](./generated/ingestion/sources/nifi.md) | `pip install 'acryl-datahub[nifi]'` | NiFi source | +| [powerbi](./generated/ingestion/sources/powerbi.md#module-powerbi) | `pip install 'acryl-datahub[powerbi]'` | Microsoft Power BI source | +| [powerbi-report-server](./generated/ingestion/sources/powerbi.md#module-powerbi-report-server) | `pip install 'acryl-datahub[powerbi-report-server]'` | Microsoft Power BI Report Server source | + +### Debug/Utility Plugins + +| Plugin Name | Install Command | Provides | +| --------------- | ---------------------------------------------- | ----------------------------------------------------- | +| debug-recording | `pip install 'acryl-datahub[debug-recording]'` | Record and replay ingestion runs for debugging (Beta) | + +### Sinks + +| Plugin Name | Install Command | Provides | +| ----------------------------------------------------------------- | -------------------------------------------- | -------------------------- | +| [metadata-file](../metadata-ingestion/sink_docs/metadata-file.md) | _included by default_ | File source and sink | +| [console](../metadata-ingestion/sink_docs/console.md) | _included by default_ | Console sink | +| [datahub-rest](../metadata-ingestion/sink_docs/datahub.md) | `pip install 'acryl-datahub[datahub-rest]'` | DataHub sink over REST API | +| [datahub-kafka](../metadata-ingestion/sink_docs/datahub.md) | `pip install 'acryl-datahub[datahub-kafka]'` | DataHub sink over Kafka | + +These plugins can be mixed and matched as desired. For example: + +```shell +pip install 'acryl-datahub[bigquery,datahub-rest]' +``` + +### Check the active plugins + +```shell +datahub check plugins +``` + +[extra requirements]: https://www.python-ldap.org/en/python-ldap-3.3.0/installing.html#build-prerequisites +## Release Notes and CLI versions + +The server release notes can be found in [github releases](https://github.com/datahub-project/datahub/releases). These releases are done approximately every week on a regular cadence unless a blocking issue or regression is discovered. + +CLI release is made through a different repository and release notes can be found in [acryldata releases](https://github.com/acryldata/datahub/releases). At least one release which is tied to the server release is always made alongwith the server release. Multiple other bigfix releases are made in between based on amount of fixes that are merged between the server release mentioned above. + +If server with version `0.8.28` is being used then CLI used to connect to it should be `0.8.28.x`. Tests of new CLI are not ran with older server versions so it is not recommended to update the CLI if the server is not updated. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/components.md b/docs-archive/versioned_docs/version-1.5.0/docs/components.md new file mode 100644 index 00000000..f31e6779 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/components.md @@ -0,0 +1,65 @@ +--- +title: Components +sidebar_label: Components +slug: /components +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/components.md' +--- + +# DataHub Components Overview + +The DataHub platform consists of the components shown in the following diagram. + +

+ +

+ +## Metadata Store + +The Metadata Store is responsible for storing the [Entities & Aspects](/docs/metadata-modeling/metadata-model/) comprising the Metadata Graph. This includes +exposing an API for [ingesting metadata](/docs/metadata-service#ingesting-entities), [fetching Metadata by primary key](/docs/metadata-service#retrieving-entities), [searching entities](/docs/metadata-service#search-an-entity), and [fetching Relationships](/docs/metadata-service#get-relationships-edges) between +entities. It consists of a Spring Java Service hosting a set of [Rest.li](https://linkedin.github.io/rest.li/) API endpoints, along with +MySQL, Elasticsearch, & Kafka for primary storage & indexing. + +Get started with the Metadata Store by following the [Quickstart Guide](/docs/quickstart/). + +## Metadata Models + +Metadata Models are schemas defining the shape of the Entities & Aspects comprising the Metadata Graph, along with the relationships between them. They are defined +using [PDL](https://linkedin.github.io/rest.li/pdl_schema), a modeling language quite similar in form to Protobuf while serializes to JSON. Entities represent a specific class of Metadata +Asset such as a Dataset, a Dashboard, a Data Pipeline, and beyond. Each _instance_ of an Entity is identified by a unique identifier called an `urn`. Aspects represent related bundles of data attached +to an instance of an Entity such as its descriptions, tags, and more. View the current set of Entities supported [here](/docs/metadata-modeling/metadata-model#exploring-datahubs-metadata-model). + +Learn more about DataHub models Metadata [here](/docs/metadata-modeling/metadata-model/). + +## Ingestion Framework + +The Ingestion Framework is a modular, extensible Python library for extracting Metadata from external source systems (e.g. +Snowflake, Looker, MySQL, Kafka), transforming it into DataHub's [Metadata Model](/docs/metadata-modeling/metadata-model/), and writing it into DataHub via +either Kafka or using the Metadata Store Rest APIs directly. DataHub supports an [extensive list of Source connectors](/docs/metadata-ingestion/#installing-plugins) to choose from, along with +a host of capabilities including schema extraction, table & column profiling, usage information extraction, and more. + +Getting started with the Ingestion Framework is as simple: just define a YAML file and execute the `datahub ingest` command. +Learn more by heading over the [Metadata Ingestion](/docs/metadata-ingestion/) guide. + +## GraphQL API + +The [GraphQL](https://graphql.org/) API provides a strongly-typed, entity-oriented API that makes interacting with the Entities comprising the Metadata +Graph simple, including APIs for adding and removing tags, owners, links & more to Metadata Entities! Most notably, this API is consumed by the User Interface (discussed below) for enabling Search & Discovery, Governance, Observability +and more. + +To get started using the GraphQL API, check out the [Getting Started with GraphQL](/docs/api/graphql/getting-started) guide. + +## User Interface + +DataHub comes with a React UI including an ever-evolving set of features to make Discovering, Governing, & Debugging your Data Assets easy & delightful. +For a full overview of the capabilities currently supported, take a look at the [Features](features.md) overview. For a look at what's coming next, +head over to the [Roadmap](/docs/roadmap/). + +## Learn More + +Learn more about the specifics of the [DataHub Architecture](./architecture/architecture.md) in the Architecture Overview. Learn about using & developing the components +of the Platform by visiting the Module READMEs. + +## Feedback / Questions / Concerns + +We want to hear from you! For any inquiries, including Feedback, Questions, or Concerns, reach out on [Slack](https://datahubspace.slack.com/join/shared_invite/zt-nx7i0dj7-I3IJYC551vpnvvjIaNRRGw#/shared-invite/email)! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/datahub_lite.md b/docs-archive/versioned_docs/version-1.5.0/docs/datahub_lite.md new file mode 100644 index 00000000..27828834 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/datahub_lite.md @@ -0,0 +1,621 @@ +--- +title: DataHub Lite (Experimental) +sidebar_label: Lite (Experimental) +slug: /datahub_lite +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/datahub_lite.md' +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHub Lite (Experimental) + +## What is it? + +DataHub Lite is a lightweight embeddable version of DataHub with no external dependencies. It is intended to enable local developer tooling use-cases such as simple access to metadata for scripts and other tools. +DataHub Lite is compatible with the DataHub metadata format and all the ingestion connectors that DataHub supports. +Currently DataHub Lite uses DuckDB under the covers as its default storage layer, but that might change in the future. + +## Features + +- Designed for the CLI +- Available as a Python library or REST API +- Ingest metadata from all DataHub ingestion sources +- Metadata Reads + - navigate metadata using a hierarchy + - get metadata for an entity + - search / query metadata across all entities +- Forward metadata automatically to a central GMS or Kafka instance + +## Architecture + +![architecture](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/lite/lite_architecture.png) + +## What is it not? + +DataHub Lite is NOT meant to be a replacement for the production Java DataHub server ([datahub-gms](./architecture/metadata-serving.md)). It does not offer the full set of API-s that the DataHub GMS server does. +The following features are **NOT** supported: + +- Full-text search with ranking and relevance features +- Graph traversal of relationships (e.g. lineage) +- Metadata change stream over Kafka (only forwarding of writes is supported) +- GraphQL API + +## Prerequisites + +To use `datahub lite` commands, you need to install [`acryl-datahub`](https://pypi.org/project/acryl-datahub/) > 0.9.6 ([install instructions](./cli.md#using-pip)) and the `datahub-lite` plugin. + +```shell +pip install acryl-datahub[datahub-lite] +``` + +## Importing Metadata + +To ingest metadata into DataHub Lite, all you have to do is change the `sink:` in your recipe to be a `datahub-lite` instance. See the detailed sink docs [here](../metadata-ingestion/sink_docs/datahub.md#datahub-lite-experimental). +For example, here is a simple recipe file that ingests mysql metadata into datahub-lite. + +```yaml +# mysql.in.dhub.yaml +source: + type: mysql + config: + host_port: localhost:3306 + username: datahub + password: datahub + +sink: + type: datahub-lite +``` + +By default, `lite` will create a local instance under `~/.datahub/lite/`. + +Ingesting metadata into DataHub Lite is as simple as running ingestion: +`datahub ingest -c mysql.in.dhub.yaml` + +:::note + +DataHub Lite currently doesn't support stateful ingestion, so you'll have to turn off stateful ingestion in your recipe to use it. This will be fixed shortly. + +::: + +### Forwarding to a central DataHub GMS over REST or Kafka + +DataHub Lite can be configured to forward all writes to a central DataHub GMS using either the REST API or the Kafka API. +To configure forwarding, add a **forward_to** section to your DataHub Lite config that conforms to a valid DataHub Sink configuration. Here is an example: + +```yaml +# mysql.in.dhub.yaml with forwarding to datahub-gms over REST +source: + type: mysql + config: + host_port: localhost:3306 + username: datahub + password: datahub + +sink: + type: datahub-lite + config: + forward_to: + type: datahub-rest + config: + server: "http://datahub-gms:8080" +``` + +:::note + +Forwarding is currently best-effort, so there can be losses in metadata if the remote server is down. For a reliable sync mechanism, look at the [exporting metadata](#export-metadata-export) section to generate a standard metadata file. + +::: + +### Importing from a file + +As a convenient short-cut, you can import metadata from any standard DataHub metadata json file (e.g. generated via using a file sink) by issuing a _datahub lite import_ command. + +```shell +> datahub lite import --file metadata_events.json + +``` + +## Exploring Metadata + +The `datahub lite` group of commands provides a set of capabilities for you to explore the metadata you just ingested. + +### List (ls) + +Listing functions like a directory structure that is customized based on the kind of system being explored. DataHub's metadata is automatically organized into databases, tables, views, dashboards, charts, etc. + +:::note + +Using the `ls` command below is much more pleasant when you have tab completion enabled on your shell. Check out the [Setting up Tab Completion](#tab-completion) section at the bottom of the guide. + +::: + +```shell +> datahub lite ls / +databases +bi_tools +tags +# Stepping down one level +> datahub lite ls /databases +mysql +# Stepping down another level +> datahub lite ls /databases/mysql +instances +... +# At the final level +> datahub lite ls /databases/mysql/instances/default/databases/datahub/tables/ +metadata_aspect_v2 + +# Listing a leaf entity functions just like the unix ls command +> datahub lite ls /databases/mysql/instances/default/databases/datahub/tables/metadata_aspect_v2 +metadata_aspect_v2 +``` + +### Read (get) + +Once you have located a path of interest, you can read metadata at that entity, by issuing a **get**. You can additionally filter the metadata retrieved from an entity by the aspect type of the metadata (e.g. to request the schema, filter by the **schemaMetadata** aspect). + +Aside: If you are curious what all types of entities and aspects DataHub supports, check out the metadata model of entities like [Dataset](./generated/metamodel/entities/dataset.md), [Dashboard](./generated/metamodel/entities/dashboard.md) etc. + +The general template for the get responses looks like: + +``` +{ + "urn": , + : { + "value": , # aspect value as a dictionary + "systemMetadata": # present if details are requested + } +} +``` + +Here is what executing a _get_ command looks like: + +
+ +Get metadata for an entity by path + + +```json +> datahub lite get --path /databases/mysql/instances/default/databases/datahub/tables/metadata_aspect_v2 +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)", + "container": { + "value": { + "container": "urn:li:container:21d4204e13d5b984c58acad468ecdbdd" + } + }, + "status": { + "value": { + "removed": false + } + }, + "datasetProperties": { + "value": { + "customProperties": {}, + "name": "metadata_aspect_v2", + "tags": [] + } + }, + "schemaMetadata": { + "value": { + "schemaName": "datahub.metadata_aspect_v2", + "platform": "urn:li:dataPlatform:mysql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "urn", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(collation='utf8mb4_bin', length=500)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "aspect", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(collation='utf8mb4_bin', length=200)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "version", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.NumberType": {} + } + }, + "nativeDataType": "BIGINT(display_width=20)", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "metadata", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "LONGTEXT(collation='utf8mb4_bin')", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "systemmetadata", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "LONGTEXT(collation='utf8mb4_bin')", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdon", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.TimeType": {} + } + }, + "nativeDataType": "DATETIME(fsp=6)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdby", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(collation='utf8mb4_bin', length=255)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "createdfor", + "nullable": true, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(collation='utf8mb4_bin', length=255)", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + "subTypes": { + "value": { + "typeNames": [ + "table" + ] + } + } +} +``` + +
+ +#### Get metadata for an entity filtered by specific aspect + +```json +> datahub lite get --path /databases/mysql/instances/default/databases/datahub/tables/metadata_aspect_v2 --aspect status +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)", + "status": { + "value": { + "removed": false + } + } +} +``` + +:::note + +Using the `get` command by path is much more pleasant when you have tab completion enabled on your shell. Check out the [Setting up Tab Completion](#tab-completion) section at the bottom of the guide. + +::: + +#### Get metadata using the urn of the entity + +```json +> datahub lite get --urn "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)" --aspect status +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)", + "status": { + "value": { + "removed": false + } + } +} +``` + +
+ +Get metadata with additional details (systemMetadata) + + +```json +> datahub lite get --path /databases/mysql/instances/default/databases/datahub/tables/metadata_aspect_v2 --aspect status --verbose +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)", + "status": { + "value": { + "removed": false + }, + "systemMetadata": { + "lastObserved": 1673982834666, + "runId": "mysql-2023_01_17-11_13_12", + "properties": { + "sysVersion": 1 + } + } + } +} +``` + +
+ +#### Point-in-time Queries + +DataHub Lite preserves every version of metadata ingested, just like DataHub GMS. You can also query the metadata as of a specific point in time by adding the _--asof_ parameter to your _get_ command. + +```shell +> datahub lite get "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)" --aspect status --asof 2020-01-01 +null + +> datahub lite get "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)" --aspect status --asof 2023-01-16 +{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD)", + "status": { + "removed": false + } +} +``` + +### Search (search) + +DataHub Lite also allows you to search using queries within the metadata using the `datahub lite search` command. +You can provide a free form search query like: "customer" and DataHub Lite will attempt to find entities that match the name customer either in the id of the entity or within the name fields of aspects in the entities. + +```shell +> datahub lite search pet +{"id": "urn:li:dataset:(urn:li:dataPlatform:looker,long_tail_companions.explore.long_tail_pets,PROD)", "aspect": "urn", "snippet": null} +{"id": "urn:li:dataset:(urn:li:dataPlatform:looker,long_tail_companions.explore.long_tail_pets,PROD)", "aspect": "datasetProperties", "snippet": "{\"customProperties\": {\"looker.explore.label\": \"Long Tail Pets\", \"looker.explore.file\": \"long_tail_companions.model.lkml\"}, \"externalUrl\": \"https://acryl.cloud.looker.com/explore/long_tail_companions/long_tail_pets\", \"name\": \"Long Tail Pets\", \"tags\": []}"} +``` + +You can also query the metadata precisely using DuckDB's [JSON](https://duckdb.org/docs/extensions/json.html) extract functions. +Writing these functions requires that you understand the DataHub metadata model and how the data is laid out in DataHub Lite. + +For example, to find all entities whose _datasetProperties_ aspect includes the _view_definition_ in its _customProperties_ sub-field, we can issue the following command: + +```shell +> datahub lite search --aspect datasetProperties --flavor exact "metadata -> '$.customProperties' ->> '$.view_definition' IS NOT NULL" +``` + +```json +{"id": "urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_MUTEXES,PROD)", "aspect": "datasetProperties", "snippet": "{\"customProperties\": {\"view_definition\": \"CREATE TEMPORARY TABLE `INNODB_MUTEXES` (\\n `NAME` varchar(4000) NOT NULL DEFAULT '',\\n `CREATE_FILE` varchar(4000) NOT NULL DEFAULT '',\\n `CREATE_LINE` int(11) unsigned NOT NULL DEFAULT 0,\\n `OS_WAITS` bigint(21) unsigned NOT NULL DEFAULT 0\\n) ENGINE=MEMORY DEFAULT CHARSET=utf8\", \"is_view\": \"True\"}, \"name\": \"INNODB_MUTEXES\", \"tags\": []}"} +{"id": "urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.user_variables,PROD)", "aspect": "datasetProperties", "snippet": "{\"customProperties\": {\"view_definition\": \"CREATE TEMPORARY TABLE `user_variables` (\\n `VARIABLE_NAME` varchar(64) NOT NULL DEFAULT '',\\n `VARIABLE_VALUE` varchar(2048) DEFAULT NULL,\\n `VARIABLE_TYPE` varchar(64) NOT NULL DEFAULT '',\\n `CHARACTER_SET_NAME` varchar(32) DEFAULT NULL\\n) ENGINE=MEMORY DEFAULT CHARSET=utf8\", \"is_view\": \"True\"}, \"name\": \"user_variables\", \"tags\": []}"} +{"id": "urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_TABLESPACES_ENCRYPTION,PROD)", "aspect": "datasetProperties", "snippet": "{\"customProperties\": {\"view_definition\": \"CREATE TEMPORARY TABLE `INNODB_TABLESPACES_ENCRYPTION` (\\n `SPACE` int(11) unsigned NOT NULL DEFAULT 0,\\n `NAME` varchar(655) DEFAULT NULL,\\n `ENCRYPTION_SCHEME` int(11) unsigned NOT NULL DEFAULT 0,\\n `KEYSERVER_REQUESTS` int(11) unsigned NOT NULL DEFAULT 0,\\n `MIN_KEY_VERSION` int(11) unsigned NOT NULL DEFAULT 0,\\n `CURRENT_KEY_VERSION` int(11) unsigned NOT NULL DEFAULT 0,\\n `KEY_ROTATION_PAGE_NUMBER` bigint(21) unsigned DEFAULT NULL,\\n `KEY_ROTATION_MAX_PAGE_NUMBER` bigint(21) unsigned DEFAULT NULL,\\n `CURRENT_KEY_ID` int(11) unsigned NOT NULL DEFAULT 0,\\n `ROTATING_OR_FLUSHING` int(1) NOT NULL DEFAULT 0\\n) ENGINE=MEMORY DEFAULT CHARSET=utf8\", \"is_view\": \"True\"}, \"name\": \"INNODB_TABLESPACES_ENCRYPTION\", \"tags\": []}"} +``` + +Search will return results that include the _id_ of the entity that matched along with the _aspect_ and the content of the aspect as part of the _snippet_ field. If you just want the _id_ of the entity to be returned, use the _--no-details_ flag. + +```shell +> datahub lite search --aspect datasetProperties --flavor exact "metadata -> '$.customProperties' ->> '$.view_definition' IS NOT NULL" --no-details +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_SYS_FOREIGN,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_CMPMEM_RESET,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_FT_DEFAULT_STOPWORD,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_SYS_TABLES,PROD) +... +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_SYS_COLUMNS,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.INNODB_FT_CONFIG,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.USER_STATISTICS,PROD) +``` + +### List Urns (list-urns) + +List all the ids in the DataHub Lite instance. + +```shell +> datahub lite list-urns +urn:li:container:21d4204e13d5b984c58acad468ecdbdd +urn:li:dataset:(urn:li:dataPlatform:mysql,datahub.metadata_aspect_v2,PROD) + +urn:li:container:aa82e8309ce84acc350640647a54ca3b +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.ALL_PLUGINS,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.APPLICABLE_ROLES,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.CHARACTER_SETS,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.CHECK_CONSTRAINTS,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.COLLATIONS,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.COLLATION_CHARACTER_SET_APPLICABILITY,PROD) +urn:li:dataset:(urn:li:dataPlatform:mysql,information_schema.COLUMNS,PROD) +... + +``` + +### HTTP Server (serve) + +DataHub Lite can be run as a lightweight HTTP server, exposing an OpenAPI spec over FastAPI. + +```shell +> datahub lite serve +INFO: Started server process [33364] +INFO: Waiting for application startup. +INFO: Application startup complete. +INFO: Uvicorn running on http://127.0.0.1:8979 (Press CTRL+C to quit) +``` + +OpenAPI docs are available via your browser at the same port: http://localhost:8979 + +The server exposes similar commands as the **lite** cli commands over HTTP: + +- entities: list of all entity ids and get metadata for an entity +- browse: traverse the entity hierarchy in a path based way +- search: execute search against the metadata + +#### Server Configuration + +Configuration for the server is picked up from the standard location for the **datahub** cli: **~/.datahubenv** and can be created using **datahub lite init**. + +Here is a sample config file with the **lite** section filled out: + +```yaml +gms: + server: http://localhost:8080 + token: '' +lite: + config: + file: /Users//.datahub/lite/datahub.duckdb + type: duckdb + forward_to: + type: datahub-rest + server: "http://datahub-gms:8080 +``` + +## Admin Commands + +### Export Metadata (export) + +The _export_ command allows you to export the contents of DataHub Lite into a metadata events file that you can then send to another DataHub instance (e.g. over REST). + +```shell +> datahub lite export --file datahub_lite_export.json +Successfully exported 1785 events to datahub_lite_export.json +``` + +### Clear (nuke) + +If you want to clear your DataHub lite instance, you can just issue the `nuke` command. + +```shell +> datahub lite nuke +DataHub Lite destroyed at +``` + +### Use a different file (init) + +By default, DataHub Lite will create and use a local duckdb instance located at `~/.datahub/lite/datahub.duckdb`. +If you want to use a different location, you can configure it using the `datahub lite init` command. + +```shell +> datahub lite init --type duckdb --file my_local_datahub.duckdb +Will replace datahub lite config type='duckdb' config={'file': '/Users//.datahub/lite/datahub.duckdb', 'options': {}} with type='duckdb' config={'file': 'my_local_datahub.duckdb', 'options': {}} [y/N]: y +DataHub Lite inited at my_local_datahub.duckdb +``` + +### Reindex + +DataHub Lite maintains a few derived tables to make access possible via both the native id (urn) as well as the logical path of the entity. The `reindex` command recomputes these indexes. + +## Caveat Emptor! + +DataHub Lite is a very new project. Do not use it for production use-cases. The API-s and storage formats are subject to change and we get feedback from early adopters. That said, we are really interested in accepting PR-s and suggestions for improvements to this fledgling project. + +## Advanced Options + +### Tab Completion + +Using the datahub lite commands like `ls` or `get` is much more pleasant when you have tab completion enabled on your shell. Tab completion is supported on the command line through the [Click Shell completion](https://click.palletsprojects.com/en/8.1.x/shell-completion/) module. +To set up shell completion for your shell, follow the instructions below based on your shell variant: + +#### Option 1: Inline eval (easy, less performant) + + + + +Add this to ~/.zshrc: + +```shell +eval "$(_DATAHUB_COMPLETE=zsh_source datahub)" +``` + + + + +Add this to ~/.bashrc: + +```shell +eval "$(_DATAHUB_COMPLETE=bash_source datahub)" +``` + + + + + +#### Option 2: External completion script (recommended, better performance) + +Using eval means that the command is invoked and evaluated every time a shell is started, which can delay shell responsiveness. To speed it up, write the generated script to a file, then source that. + + + + +Save the script somewhere. + +```shell +# the additional sed patches completion to be path oriented and not add spaces between each completed token +_DATAHUB_COMPLETE=zsh_source datahub | sed 's;compadd;compadd -S /;' > ~/.datahub-complete.zsh +``` + +Source the file in ~/.zshrc. + +```shell +. ~/.datahub-complete.zsh +``` + + + + +```shell +_DATAHUB_COMPLETE=bash_source datahub > ~/.datahub-complete.bash +``` + +Source the file in ~/.bashrc. + +```shell +. ~/.datahub-complete.bash +``` + + + + + +Save the script to ~/.config/fish/completions/datahub.fish: + +```shell +_DATAHUB_COMPLETE=fish_source datahub > ~/.config/fish/completions/datahub.fish +``` + + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dataproducts.md b/docs-archive/versioned_docs/version-1.5.0/docs/dataproducts.md new file mode 100644 index 00000000..889f77d2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dataproducts.md @@ -0,0 +1,423 @@ +--- +title: Data Products +slug: /dataproducts +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/dataproducts.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Data Products + + + +**🤝 Version compatibility** + +> DataHub Core: **0.10.3** | DataHub Cloud: **0.2.8** + +## What are Data Products? + +Data Products are an innovative way to organize and manage your Data Assets, such as Tables, Topics, Views, Pipelines, Charts, Dashboards, etc., within DataHub. These Data Products belong to a specific Domain and can be easily accessed by various teams or stakeholders within your organization. + +## Why Data Products? + +A key concept in data mesh architecture, Data Products are independent units of data managed by a specific domain team. They are responsible for defining, publishing, and maintaining their data assets while ensuring high-quality data that meets the needs of its consumers. + +## Benefits of Data Products + +Data Products help in curating a coherent set of logical entities, simplifying data discovery and governance. By grouping related Data Assets into a Data Product, it allows stakeholders to discover and understand available data easily, supporting data governance efforts by managing and controlling access to Data Products. + +## How Can You Use Data Products? + +Data Products can be easily published to the DataHub catalog, allowing other teams to discover and consume them. By doing this, data teams can streamline the process of sharing data, making data-driven decisions faster and more efficient. + +## Data Products Setup, Prerequisites, and Permissions + +What you need to create and add data products: + +- **Manage Data Product** metadata privilege for Domains to create/delete Data Products at the entity level. If a user has this privilege for a given Domain, they will be able to create and delete Data Products underneath it. +- **Edit Data Product** metadata privilege to add or remove the Data Product for a given entity. + +You can create this privileges by creating a new [Metadata Policy](./authorization/policies.md). + +## Using Data Products + +Data Products can be created using the UI or via a YAML file that is managed using software engineering (GitOps) practices. + +### Creating a Data Product (UI) + +To create a Data Product, first navigate to the Domain that will contain this Data Product. + +

+ +

+ +Then navigate to the Data Products tab on the Domain's home page, and click '+ New Data Product'. +This will open a new modal where you can configure the settings for your data product. Inside the form, you can choose a name for your Data Product. Most often, this will align with the logical purpose of the Data Product, for example +'Customer Orders' or 'Revenue Attribution'. You can also add documentation for your product to help other users easily discover it. Don't worry, this can be changed later. + +

+ +

+ +Once you've chosen a name and a description, click 'Create' to create the new Data Product. Once you've created the Data Product, you can click on it to continue on to the next step, adding assets to it. + +### Assigning an Asset to a Data Product (UI) + +You can assign an asset to a Data Product either using the Data Product page as the starting point or the Asset's page as the starting point. +On a Data Product page, click the 'Add Assets' button on the top right corner to add assets to the Data Product. + +

+ +

+ +On an Asset's profile page, use the right sidebar to locate the Data Product section. Click 'Set Data Product', and then search for the Data Product you'd like to add this asset to. When you're done, click 'Add'. + +

+ +

+ +To remove an asset from a Data Product, click the 'x' icon on the Data Product label. + +> Notice: Adding or removing an asset from a Data Product requires the `Edit Data Product` Metadata Privilege, which can be granted +> by a [Policy](authorization/policies.md). + +### Creating a Data Product (YAML + git) + +DataHub ships with a YAML-based Data Product spec for defining and managing Data Products as code. + +Here is an example of a Data Product named "Pet of the Week" which belongs to the **Marketing** domain and contains three data assets. The **Spec** tab describes the JSON Schema spec for a DataHub data product file. + + + + + +```yaml +# Inlined from /metadata-ingestion/examples/data_product/dataproduct.yaml +id: pet_of_the_week +domain: Marketing +display_name: Pet of the Week Campaign +description: |- + This campaign includes Pet of the Week data. + +# List of assets that belong to this Data Product +assets: + - urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.pet_details,PROD) + - urn:li:dashboard:(looker,dashboards.19) + - urn:li:dataFlow:(airflow,snowflake_load,prod) + +output_ports: + - urn:li:dataset:(urn:li:dataPlatform:snowflake,long_tail_companions.analytics.pet_details,PROD) + +owners: + - id: urn:li:corpuser:jdoe + type: BUSINESS_OWNER + - id: urn:li:corpuser:fbar + type: urn:li:ownershipType:architect # Maps to a custom ownership type + +# Tags associated with this Data Product +tags: + - urn:li:tag:adoption + +# Glossary Terms associated with this Data Product +terms: + - urn:li:glossaryTerm:ClientsAndAccounts.AccountBalance + +institutional_memory: + elements: + - title: URL for campaign + description: |- + Go here to see the campaign. + url: https://example.com/pet_of_the_week + +# Custom Properties +properties: + lifecycle: production + sla: 7am every day + +``` + + +:::note + +When bare domain names like `Marketing` is used, `datahub` will first check if a domain like `urn:li:domain:Marketing` is provisioned, failing that; it will check for a provisioned domain that has the same name. If we are unable to resolve bare domain names to provisioned domains, then yaml-based ingestion will refuse to proceeed until the domain is provisioned on DataHub. + +This applies to other fields as well, such as owners, ownership types, tags, and terms. + +::: + +You can also provide fully-qualified domain names (e.g. `urn:li:domain:dcadded3-2b70-4679-8b28-02ac9abc92eb`) to ensure that no ingestion-time domain resolution is needed. + + + + +```json +{ + "$defs": { + "InstitutionMemory": { + "additionalProperties": false, + "properties": { + "elements": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/InstitutionMemoryElement" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Elements" + } + }, + "title": "InstitutionMemory", + "type": "object" + }, + "InstitutionMemoryElement": { + "additionalProperties": false, + "properties": { + "url": { + "title": "Url", + "type": "string" + }, + "description": { + "title": "Description", + "type": "string" + } + }, + "required": [ + "url", + "description" + ], + "title": "InstitutionMemoryElement", + "type": "object" + }, + "Ownership": { + "additionalProperties": false, + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + } + }, + "required": [ + "id", + "type" + ], + "title": "Ownership", + "type": "object" + } + }, + "additionalProperties": false, + "description": "This is a DataProduct class which represents a DataProduct\n\nArgs:\n id (str): The id of the Data Product\n domain (str): The domain that the Data Product belongs to. Either as a name or a fully-qualified urn.\n owners (Optional[List[str, Ownership]]): A list of owners and their types.\n institutional_memory (Optional[InstitutionMemory]): A list of institutional memory elements\n display_name (Optional[str]): The name of the Data Product to display in the UI\n description (Optional[str]): A documentation string for the Data Product\n tags (Optional[List[str]]): An array of tags (either bare ids or urns) for the Data Product\n terms (Optional[List[str]]): An array of terms (either bare ids or urns) for the Data Product\n assets (List[str]): An array of entity urns that are part of the Data Product", + "properties": { + "id": { + "title": "Id", + "type": "string" + }, + "domain": { + "title": "Domain", + "type": "string" + }, + "assets": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Assets" + }, + "display_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Display Name" + }, + "owners": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/Ownership" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Owners" + }, + "institutional_memory": { + "anyOf": [ + { + "$ref": "#/$defs/InstitutionMemory" + }, + { + "type": "null" + } + ], + "default": null + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Description" + }, + "tags": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + }, + "terms": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Terms" + }, + "properties": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Properties" + }, + "external_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "External Url" + }, + "output_ports": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Output Ports" + } + }, + "required": [ + "id", + "domain" + ], + "title": "DataProduct", + "type": "object" +} + +``` + + + + +To sync this yaml file to DataHub, use the `datahub` cli via the `dataproduct` group of commands. + +```shell +datahub dataproduct upsert -f user_dataproduct.yaml +``` + +### Keeping the YAML file sync-ed with changes in UI + +The `datahub` cli allows you to keep this YAML file synced with changes happening in the UI. All you have to do is run the `datahub dataproduct diff` command. + +Here is an example invocation that checks if there is any diff and updates the file in place: + +```shell +datahub dataproduct diff -f user_dataproduct.yaml --update +``` + +This allows you to manage your data product definition in git while still allowing for edits in the UI. Business Users and Developers can both collaborate on the definition of a data product with ease using this workflow. + +### Advanced cli commands for managing Data Products + +There are many more advanced cli commands for managing Data Products as code. Take a look at the [Data Products section](./cli.md#dataproduct-data-product-entity) on the CLI reference guide for more details. + +### What updates are planned for the Data Products feature? + +The following features are next on the roadmap for Data Products + +- Support for marking data assets in a Data Product as private versus shareable for other teams to consume +- Support for declaring data lineage manually to upstream and downstream data products +- Support for declaring logical schema for Data Products +- Support for associating data contracts with Data Products +- Support for semantic versioning of the Data Product entity + +### Related Features + +- [Domains](./domains.md) +- [Glossary Terms](./glossary/business-glossary.md) +- [Tags](./tags.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/BASE_PATH_CONFIGURATION.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/BASE_PATH_CONFIGURATION.md new file mode 100644 index 00000000..cf2c12bc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/BASE_PATH_CONFIGURATION.md @@ -0,0 +1,154 @@ +--- +title: DataHub Base Path Configuration +sidebar_label: Base Path Configuration +slug: /deploy/base_path_configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/deploy/BASE_PATH_CONFIGURATION.md +--- +# DataHub Base Path Configuration + +This document describes how to configure DataHub to run from a custom base path (e.g., `/my-app/` instead of `/`). + +## Overview + +DataHub now supports serving from a custom base path, allowing you to: + +- Serve DataHub from a subdirectory (e.g., `https://company.com/datahub/`) +- Deploy multiple DataHub instances on the same domain +- Integrate DataHub behind reverse proxies with path-based routing + +## Configuration + +### Environment Variables + +Set the following environment variable to configure the base path: + +```bash +export DATAHUB_BASE_PATH="/my-app" +``` + +**Important Notes:** + +- The base path should start with `/` but not end with `/` +- Examples: `/datahub`, `/my-app`, `/tools/datahub` +- Leave empty or unset for root path deployment + +### Component Configuration + +The base path configuration affects three main components: + +#### 1. Frontend (datahub-web-react) + +The frontend determines the base path using multiple detection methods (in priority order): + +1. **Environment Variable** (highest priority): + + ```bash + # Set in .env file or environment + REACT_APP_BASE_PATH=/my-app + ``` + +2. **Server Configuration** (fetched from `/config` endpoint): + + - The React app fetches base path from the backend's configuration + - Most reliable for production deployments + +3. **URL Analysis** (fallback): + - Analyzes current URL to detect base path + - Used when server config is unavailable + +**Development Configuration:** + +```bash +# Method 1: Environment variable (recommended for development) +echo "REACT_APP_BASE_PATH=/my-app" >> .env +yarn start # Development server at http://localhost:3000/my-app + +# Method 2: Using Vite preview (for testing production builds) +yarn build +npx vite preview --port 3001 # Uses base path from REACT_APP_BASE_PATH +# Accessible at: http://localhost:3001/my-app + +# Method 3: Override base path via command line +npx vite preview --port 3001 --base /custom-path +``` + +**Production Behavior:** + +- Automatically fetches base path from server configuration +- Falls back to URL detection if server config unavailable +- Provides console warnings to help with debugging + +#### 2. DataHub Frontend Service (Play Framework) + +The Play Framework application supports base path through: + +- `datahub.basePath` configuration in `application.conf` +- `play.http.context` for Play Framework native support +- Environment variable: `DATAHUB_BASE_PATH` + +#### 3. Metadata Service (Spring Boot) + +The metadata service (GMS) supports base path through: + +- `server.servlet.context-path` in `application.yaml` +- Environment variable: `DATAHUB_BASE_PATH` + +## Deployment Examples + +### Docker Compose + +```yaml +version: "3.8" +services: + datahub-frontend: + environment: + - DATAHUB_BASE_PATH=/datahub + - PLAY_HTTP_CONTEXT=/datahub + + datahub-gms: + environment: + - DATAHUB_BASE_PATH=/datahub +``` + +### Kubernetes + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datahub-frontend +spec: + template: + spec: + containers: + - name: datahub-frontend + env: + - name: DATAHUB_BASE_PATH + value: "/datahub" + - name: DATAHUB_GMS_BASE_PATH + value: "/datahub" + - name: PLAY_HTTP_CONTEXT + value: "/datahub" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datahub-gms +spec: + template: + spec: + containers: + - name: datahub-frontend + env: + - name: DATAHUB_BASE_PATH + value: "/datahub" + - name: DATAHUB_GMS_BASE_PATH + value: "/datahub" +``` + +## Limitations + +- Base path must be consistent across all DataHub components +- Changing base path requires restat of GMS and front end deployments. +- Testing of multiple DataHub environments served by the same URL, e.g. http://example.com/datahub1, http://example.com/datahub2, hasn't been extensively validated. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/aws.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/aws.md new file mode 100644 index 00000000..3b843b83 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/aws.md @@ -0,0 +1,500 @@ +--- +title: Deploying to AWS +sidebar_label: Deploying to AWS +slug: /deploy/aws +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/deploy/aws.md' +--- + +# AWS setup guide + +The following is a set of instructions to quickstart DataHub on AWS Elastic Kubernetes Service (EKS). Note, the guide +assumes that you do not have a kubernetes cluster set up. If you are deploying DataHub to an existing cluster, please +skip the corresponding sections. + +## Prerequisites + +This guide requires the following tools: + +- [kubectl](https://kubernetes.io/docs/tasks/tools/) to manage kubernetes resources +- [helm](https://helm.sh/docs/intro/install/) to deploy the resources based on helm charts. Note, we only support Helm 3. +- [eksctl](https://eksctl.io/installation/) to create and manage clusters on EKS +- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) to manage AWS resources + +To use the above tools, you need to set up AWS credentials by following +this [guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). + +## Start up a kubernetes cluster on AWS EKS + +Let’s follow this [guide](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html) to create a new +cluster using eksctl. Run the following command with cluster-name set to the cluster name of choice, and region set to +the AWS region you are operating on. + +``` +eksctl create cluster \ + --name <> \ + --region <> \ + --with-oidc \ + --nodes=3 +``` + +The command will provision an EKS cluster powered by 3 EC2 m3.large nodes and provision a VPC based networking layer. + +If you are planning to run the storage layer (MySQL, Elasticsearch, Kafka) as pods in the cluster, you need at least 3 +nodes. If you decide to use managed storage services, you can reduce the number of nodes or use m3.medium nodes to save +cost. Refer to this [guide](https://eksctl.io/usage/creating-and-managing-clusters/) to further customize the cluster +before provisioning. + +Note, OIDC setup is required for following this guide when setting up the load balancer. + +Run `kubectl get nodes` to confirm that the cluster has been setup correctly. You should get results like below + +``` +NAME STATUS ROLES AGE VERSION +ip-192-168-49-49.us-west-2.compute.internal Ready 3h v1.18.9-eks-d1db3c +ip-192-168-64-56.us-west-2.compute.internal Ready 3h v1.18.9-eks-d1db3c +ip-192-168-8-126.us-west-2.compute.internal Ready 3h v1.18.9-eks-d1db3c +``` + +### Install EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes + +Once your cluster is running, make sure to install the EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes. [add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html). By default Core DNS and VPC CNI plugins are installed. You need to manually install the EBS CSI driver. It show look this in your console when you are done. + +![Screenshot 2024-11-15 at 4 42 09 PM](https://github.com/user-attachments/assets/5a9a2af0-e804-4896-85bb-dc5834208719) + +### Add the AmazonEBSCSIDriverPolicy role to the EKS node group + +Next is to add the AmazonEBSCSIDriverPolicy role to the EKS node group. You will from the EKS Node group by going to the Compute tab in your EKS cluster and clicking on the IAM entry for the EKS node group. Add the AmazonEBSCSIDriverPolicy policy. + +![Screenshot 2024-11-15 at 4 42 29 PM](https://github.com/user-attachments/assets/8971c8d6-8543-408b-9a07-814aacb2532d) +![Screenshot 2024-11-15 at 4 42 46 PM](https://github.com/user-attachments/assets/397f9131-5f13-4d9f-a664-9921d9bbf44e) + +## Setup DataHub using Helm + +Once the kubernetes cluster has been set up, you can deploy DataHub and it’s prerequisites using helm. Please follow the +steps in this [guide](kubernetes.md) + +## Expose endpoints using a load balancer + +Now that all the pods are up and running, you need to expose the datahub-frontend end point by setting +up [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/). To do this, you need to first set up an +ingress controller. There are +many [ingress controllers](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) to choose +from, but here, we will follow +this [guide](https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html) to set up the AWS +Application Load Balancer(ALB) Controller. + +First, if you did not use eksctl to setup the kubernetes cluster, make sure to go through the prerequisites listed +[here](https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html). + +Download the IAM policy document for allowing the controller to make calls to AWS APIs on your behalf. + +``` +curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json +``` + +Create an IAM policy based on the policy document by running the following. + +``` +aws iam create-policy \ + --policy-name AWSLoadBalancerControllerIAMPolicy \ + --policy-document file://iam_policy.json +``` + +Use eksctl to create a service account that allows us to attach the above policy to kubernetes pods. + +``` +eksctl create iamserviceaccount \ + --cluster=<> \ + --namespace=kube-system \ + --name=aws-load-balancer-controller \ + --attach-policy-arn=arn:aws:iam::<>:policy/AWSLoadBalancerControllerIAMPolicy \ + --override-existing-serviceaccounts \ + --approve +``` + +Install the TargetGroupBinding custom resource definition by running the following. + +``` +kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master" +``` + +Add the helm chart repository containing the latest version of the ALB controller. + +``` +helm repo add eks https://aws.github.io/eks-charts +helm repo update +``` + +Install the controller into the kubernetes cluster by running the following. + +``` +helm upgrade -i aws-load-balancer-controller eks/aws-load-balancer-controller \ + --set clusterName=<> \ + --set serviceAccount.create=false \ + --set serviceAccount.name=aws-load-balancer-controller \ + -n kube-system +``` + +Verify the install completed by running `kubectl get deployment -n kube-system aws-load-balancer-controller`. It should +return a result like the following. + +``` +NAME READY UP-TO-DATE AVAILABLE AGE +aws-load-balancer-controller 2/2 2 2 142m +``` + +Now that the controller has been set up, we can enable ingress by updating the values.yaml (or any other values.yaml +file used to deploy datahub). Change datahub-frontend values to the following. + +``` +datahub-frontend: + enabled: true + image: + repository: acryldata/datahub-frontend-react + tag: "head" + ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: alb + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/target-type: instance + alb.ingress.kubernetes.io/certificate-arn: <> + alb.ingress.kubernetes.io/inbound-cidrs: 0.0.0.0/0 + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/ssl-redirect: '443' + hosts: + - host: <> + paths: + - /* +``` + +Do not use the 'latest' or 'debug' tags for any of the images, as those are not supported and are present only due to legacy reasons. Please use 'head' or version-specific tags, like v0.8.40. For production, we recommend using version-specific tags, not 'head'. + +You need to request a certificate in the AWS Certificate Manager by following this +[guide](https://docs.aws.amazon.com/acm/latest/userguide/gs-acm-request-public.html), and replace certificate-arn with +the ARN of the new certificate. You also need to replace host-name with the hostname of choice like +demo.datahub.com. + +To have the metadata [authentication service](../authentication/introducing-metadata-service-authentication.md#configuring-metadata-service-authentication) enabled and use [API tokens](../authentication/personal-access-tokens.md#creating-personal-access-tokens) from the UI you will need to set the configuration in the values.yaml for the `gms` and the `frontend` deployments. This could be done by enabling the `metadata_service_authentication`: + +``` +datahub: + metadata_service_authentication: + enabled: true +``` + +After updating the yaml file, run the following to apply the updates. + +``` +helm upgrade --install datahub datahub/datahub --values values.yaml +``` + +Once the upgrade completes, run `kubectl get ingress` to verify the ingress setup. You should see a result like the +following. + +``` +NAME CLASS HOSTS ADDRESS PORTS AGE +datahub-datahub-frontend demo.datahub.com k8s-default-datahubd-80b034d83e-904097062.us-west-2.elb.amazonaws.com 80 3h5m +``` + +Note down the elb address in the address column. Add the DNS CNAME record to the host domain pointing the host-name ( +from above) to the elb address. DNS updates generally take a few minutes to an hour. Once that is done, you should be +able to access datahub-frontend through the host-name. + +## Use AWS managed services for the storage layer + +Managing the storage services like MySQL, Elasticsearch, and Kafka as kubernetes pods requires a great deal of +maintenance workload. To reduce the workload, you can use managed services like AWS [RDS](https://aws.amazon.com/rds), +[Elasticsearch Service](https://aws.amazon.com/elasticsearch-service/), and [Managed Kafka](https://aws.amazon.com/msk/) +as the storage layer for DataHub. Support for using AWS Neptune as graph DB is coming soon. + +### RDS + +Provision a MySQL database in AWS RDS that shares the VPC with the kubernetes cluster or has VPC peering set up between +the VPC of the kubernetes cluster. Once the database is provisioned, you should be able to see the following page. Take +a note of the endpoint marked by the red box. + +

+ +

+ +First, add the DB password to kubernetes by running the following. + +``` +kubectl delete secret mysql-secrets +kubectl create secret generic mysql-secrets --from-literal=mysql-root-password=<> +``` + +Update the sql settings under global in the values.yaml as follows. + +``` + sql: + datasource: + host: "<>:3306" + hostForMysqlClient: "<>" + port: "3306" + url: "jdbc:mysql://<>:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8" + driver: "com.mysql.jdbc.Driver" + username: "root" + password: + secretRef: mysql-secrets + secretKey: mysql-root-password +``` + +Run `helm upgrade --install datahub datahub/datahub --values values.yaml` to apply the changes. + +### Elasticsearch Service + +Provision an elasticsearch domain running elasticsearch version 7.10 or above that shares the VPC with the kubernetes +cluster or has VPC peering set up between the VPC of the kubernetes cluster. Once the domain is provisioned, you should +be able to see the following page. Take a note of the endpoint marked by the red box. + +

+ +

+ +Update the elasticsearch settings under global in the values.yaml as follows. + +``` + elasticsearch: + host: <> + port: "443" + useSSL: "true" +``` + +You can also allow communication via HTTP (without SSL) by using the settings below. + +``` + elasticsearch: + host: <> + port: "80" +``` + +If you have fine-grained access control enabled with basic authentication, first run the following to create a k8s +secret with the password. + +``` +kubectl delete secret elasticsearch-secrets +kubectl create secret generic elasticsearch-secrets --from-literal=elasticsearch-password=<> +``` + +Then use the settings below. + +``` + elasticsearch: + host: <> + port: "443" + useSSL: "true" + auth: + username: <> + password: + secretRef: elasticsearch-secrets + secretKey: elasticsearch-password +``` + +If you have access control enabled with IAM auth, enable AWS auth signing in DataHub + +``` + OPENSEARCH_USE_AWS_IAM_AUTH=true +``` + +Then use the settings below. + +``` + elasticsearch: + host: <> + port: "443" + useSSL: "true" + region: <> +``` + +For AWS Elasticsearch/OpenSearch, set `USE_AWS_ELASTICSEARCH: "true"` on the system-update job (e.g. via `datahubSystemUpdate.extraEnvs` or `datahub.upgrade.env`). The system-update job performs index setup; standalone setup jobs are disabled by default. + +Run `helm upgrade --install datahub datahub/datahub --values values.yaml` to apply the changes. + +**Note:** +If you have a custom setup of elastic search cluster and are deploying through docker, you can modify the configurations +in datahub to point to the specific ES instance - + +1. If you are using `docker quickstart` you can modify the hostname and port of the ES instance in docker compose + quickstart files located [here](https://github.com/datahub-project/datahub/blob/master/docker/quickstart/). + 1. Once you have modified the quickstart recipes you can run the quickstart command using a specific docker compose + file. Sample command for that is + - `datahub docker quickstart --quickstart-compose-file docker/quickstart/docker-compose-without-neo4j.quickstart.yml` +2. If you are not using quickstart recipes, you can modify environment variable in GMS to point to the ES instance. The + env files for datahub-gms are located [here](https://github.com/datahub-project/datahub/blob/master/docker/datahub-gms/env/). + +Further, you can find a list of properties supported to work with a custom ES +instance [here](https://github.com/datahub-project/datahub/blob/master/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/ElasticsearchSSLContextFactory.java) +and [here](https://github.com/datahub-project/datahub/blob/master/metadata-service/configuration/src/main/java/com/linkedin/metadata/config/search/ElasticSearchConfiguration.java) +. + +A mapping between the property name used in the above two files and the name used in docker/env file can be +found [here](https://github.com/datahub-project/datahub/blob/master/metadata-service/configuration/src/main/resources/application.yaml). + +### Managed Streaming for Apache Kafka (MSK) + +Provision an MSK cluster that shares the VPC with the kubernetes cluster or has VPC peering set up between the VPC of +the kubernetes cluster. Once the domain is provisioned, click on the “View client information” button in the ‘Cluster +Summary” section. You should see a page like below. Take a note of the endpoints marked by the red boxes. + +

+ +

+ +Update the kafka settings under global in the values.yaml as follows. + +``` +kafka: + bootstrap: + server: "<>" + zookeeper: + server: "<>" + schemaregistry: + url: "http://prerequisites-cp-schema-registry:8081" + partitions: 3 + replicationFactor: 3 +``` + +Note, the number of partitions and replicationFactor should match the number of bootstrap servers. This is by default 3 +for AWS MSK. + +Run `helm upgrade --install datahub datahub/datahub --values values.yaml` to apply the changes. + +### AWS Glue Schema Registry + +> **WARNING**: AWS Glue Schema Registry DOES NOT have a python SDK. As such, python based libraries like ingestion or datahub-actions (UI ingestion) is not supported when using AWS Glue Schema Registry + +You can use AWS Glue schema registry instead of the kafka schema registry. To do so, first provision an AWS Glue schema +registry in the "Schema Registry" tab in the AWS Glue console page. + +Once the registry is provisioned, you can change helm chart as follows. + +``` +kafka: + bootstrap: + ... + zookeeper: + ... + schemaregistry: + type: AWS_GLUE + glue: + region: <> + registry: <> +``` + +Note, it will use the name of the topic as the schema name in the registry. + +Before you update the pods, you need to give the k8s worker nodes the correct permissions to access the schema registry. + +The minimum permissions required looks like this + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "glue:GetRegistry", + "glue:ListRegistries", + "glue:CreateSchema", + "glue:UpdateSchema", + "glue:GetSchema", + "glue:ListSchemas", + "glue:RegisterSchemaVersion", + "glue:GetSchemaByDefinition", + "glue:GetSchemaVersion", + "glue:GetSchemaVersionsDiff", + "glue:ListSchemaVersions", + "glue:CheckSchemaVersionValidity", + "glue:PutSchemaVersionMetadata", + "glue:QuerySchemaVersionMetadata" + ], + "Resource": [ + "arn:aws:glue:*:795586375822:schema/*", + "arn:aws:glue:us-west-2:795586375822:registry/demo-shared" + ] + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "glue:GetSchemaVersion" + ], + "Resource": [ + "*" + ] + } + ] +} +``` + +The latter part is required to have "\*" as the resource because of an issue in the AWS Glue schema registry library. +Refer to [this issue](https://github.com/awslabs/aws-glue-schema-registry/issues/68) for any updates. + +Glue currently doesn't support AWS Signature V4. As such, we cannot use service accounts to give permissions to access +the schema registry. The workaround is to give the above permission to the EKS worker node's IAM role. Refer +to [this issue](https://github.com/awslabs/aws-glue-schema-registry/issues/69) for any updates. + +Run `helm upgrade --install datahub datahub/datahub --values values.yaml` to apply the changes. + +Note, you will be seeing log "Schema Version Id is null. Trying to register the schema" on every request. This log is +misleading, so should be ignored. Schemas are cached, so it does not register a new version on every request (aka no +performance issues). This has been fixed by [this PR](https://github.com/awslabs/aws-glue-schema-registry/pull/64) but +the code has not been released yet. We will update version once a new release is out. + +### IAM policies for UI-based ingestion + +This section details how to attach policies to the acryl-datahub-actions pod that powers UI-based ingestion. For some of +the ingestion recipes, you sepecify login creds in the recipe itself, making it easy to set up auth to grab metadata +from the data source. However, for AWS resources, the recommendation is to use IAM roles and policies to gate requests +to access metadata on these resources. + +To do this, let's follow +this [guide](https://docs.aws.amazon.com/eks/latest/userguide/create-service-account-iam-policy-and-role.html) to +associate a kubernetes service account with an IAM role. Then we can attach this IAM role to the acryl-datahub-actions +pod to let the pod assume the specified role. + +First, you must create an IAM policy with all the permissions needed to run ingestion. This is specific to each +connector and the set of metadata you are trying to pull. i.e. profiling requires more permissions, since it needs +access to the data, not just the metadata. Let's say assume the ARN of that policy +is `arn:aws:iam::<>:policy/policy1`. + +Then, create a service account with the policy attached is to use [eksctl](https://eksctl.io/). You can run the +following command to do so. + +``` +eksctl create iamserviceaccount \ + --name <> \ + --namespace <> \ + --cluster <> \ + --attach-policy-arn <> \ + --approve \ + --override-existing-serviceaccounts +``` + +For example, running the following will create a service account "acryl-datahub-actions" in the datahub namespace of +datahub EKS cluster with `arn:aws:iam::<>:policy/policy1` attached. + +``` +eksctl create iamserviceaccount \ + --name acryl-datahub-actions \ + --namespace datahub \ + --cluster datahub \ + --attach-policy-arn arn:aws:iam::<>:policy/policy1 \ + --approve \ + --override-existing-serviceaccounts +``` + +Lastly, in the helm values.yaml, you can add the following to the acryl-datahub-actions to attach the service account to +the acryl-datahub-actions pod. + +```yaml +acryl-datahub-actions: + enabled: true + serviceAccount: + name: <> + ... +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/azure.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/azure.md new file mode 100644 index 00000000..bdd73c1d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/azure.md @@ -0,0 +1,231 @@ +--- +title: Deploying to Azure +sidebar_label: Deploying to Azure +slug: /deploy/azure +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/deploy/azure.md' +--- + +# Azure setup guide + +The following is a set of instructions to quickstart DataHub on Azure Kubernetes Service (AKS). Note, the guide +assumes that you do not have a Kubernetes cluster set up. + +## Prerequisites + +This guide requires the following tools: + +- [kubectl](https://kubernetes.io/docs/tasks/tools/) to manage Kubernetes resources +- [helm](https://helm.sh/docs/intro/install/) to deploy the resources based on helm charts. Note, we only support Helm 3. +- [AZ CLI](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) to manage Azure resources + +To use the above tools, you need to set up Azure credentials by following +this [guide](https://learn.microsoft.com/en-us/cli/azure/authenticate-azure-cli). + +## Start up a Kubernetes cluster on AKS + +You can follow this [guide](https://learn.microsoft.com/en-us/azure/aks/learn/quick-kubernetes-deploy-cli) to create a new +cluster using az cli. + +Note: you can skip the application deployment step since we are deploying DataHub instead. If you are deploying DataHub to an existing cluster, please +skip the corresponding sections. + +- Verify you have the Microsoft.OperationsManagement and Microsoft.OperationalInsights providers registered on your subscription. These Azure resource providers are required to support Container insights. Check the registration status using the following commands: + +``` +az provider show -n Microsoft.OperationsManagement -o table +az provider show -n Microsoft.OperationalInsights -o table +``` + +If they're not registered, register them using the following commands: + +``` +az provider register --namespace Microsoft.OperationsManagement +az provider register --namespace Microsoft.OperationalInsights +``` + +- Create a resource group. Change name, location to your choosing. + +``` +az group create --name myResourceGroup --location eastus +``` + +The following output indicates that the command execution was successful: + +``` +{ + "id": "/subscriptions//resourceGroups/myResourceGroup", + "location": "eastus", + "managedBy": null, + "name": "myResourceGroup", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null +} +``` + +- Create an AKS Cluster. For this project, it is best to increase node count to at least 3. Change cluster name, node count, and addons to your choosing. + +``` +az aks create -g myResourceGroup -n myAKSCluster --enable-managed-identity --node-count 3 --enable-addons monitoring --generate-ssh-keys +``` + +After a few minutes, the command completes and returns JSON-formatted information about the cluster. + +- Connect to the cluster + +Configure kubectl to connect to your Kubernetes cluster using the az aks get-credentials command. + +``` +az aks get-credentials --resource-group myResourceGroup --name myAKSCluster +``` + +Verify the connection to your cluster using the `kubectl get` command. This command returns a list of the cluster nodes. + +``` +kubectl get nodes +``` + +You should get results like below. Make sure node status is Ready. + +``` +NAME STATUS ROLES AGE VERSION +aks-nodepool1-37660971-vmss000000 Ready agent 24h v1.25.6 +aks-nodepool1-37660971-vmss000001 Ready agent 24h v1.25.6 +aks-nodepool1-37660971-vmss000002 Ready agent 24h v1.25.6 +``` + +## Setup DataHub using Helm + +Once the Kubernetes cluster has been set up, you can deploy DataHub and its prerequisites using helm. Please follow the +steps in this [guide](kubernetes.md). + +Notes: +Since we are using PostgreSQL as the storage layer, change postgresql enabled to true and mysql to false in the values.yaml file of prerequisites. +Additionally, create a postgresql secret. Make sure to include 3 passwords for the postgresql secret: postgres-password, replication-password, and password. + +## Expose endpoints using a load balancer + +Now that all the pods are up and running, you need to expose the datahub-frontend end point by setting +up [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/). To do this, you need to first set up an +ingress controller. + +There are many [ingress controllers](https://kubernetes.io/docs/concepts/services-networking/ingress-controllers/) to choose +from, but here, we will follow this [guide](https://learn.microsoft.com/en-us/azure/application-gateway/tutorial-ingress-controller-add-on-existing) to set up the Azure +Application Gateway Ingress Controller. + +- Deploy a New Application Gateway. + +First, you need to create a WAF policy + +``` +az network application-gateway waf-policy create -g myResourceGroup -n myWAFPolicy +``` + +- Before the application gateway can be deployed, you'll also need to create a public IP resource, a new virtual network with address space 10.0.0.0/16, and a subnet with address space 10.0.0.0/24. + Then, you can deploy your application gateway in the subnet using the publicIP. + +Caution: When you use an AKS cluster and application gateway in separate virtual networks, the address spaces of the two virtual networks must not overlap. The default address space that an AKS cluster deploys in is 10.224.0.0/12. + +``` +az network public-ip create -n myPublicIp -g myResourceGroup --allocation-method Static --sku Standard +az network vnet create -n myVnet -g myResourceGroup --address-prefix 10.0.0.0/16 --subnet-name mySubnet --subnet-prefix 10.0.0.0/24 +az network application-gateway create -n myApplicationGateway -l eastus -g myResourceGroup --sku WAF_v2 --public-ip-address myPublicIp --vnet-name myVnet --subnet mySubnet --priority 100 --waf-policy /subscriptions/{subscription_id}/resourceGroups/myResourceGroup/providers/Microsoft.Network/ApplicationGatewayWebApplicationFirewallPolicies/myWAFPolicy +``` + +Change myPublicIp, myResourceGroup, myVnet, mySubnet, and myApplicationGateway to names of your choosing. + +- Enable the AGIC Add-On in Existing AKS Cluster Through Azure CLI + +``` +appgwId=$(az network application-gateway show -n myApplicationGateway -g myResourceGroup -o tsv --query "id") +az aks enable-addons -n myCluster -g myResourceGroup -a ingress-appgw --appgw-id $appgwId +``` + +- Peer the Two Virtual Networks Together + +Since you deployed the AKS cluster in its own virtual network and the Application gateway in another virtual network, you'll need to peer the two virtual networks together in order for traffic to flow from the Application gateway to the pods in the cluster. + +``` +nodeResourceGroup=$(az aks show -n myCluster -g myResourceGroup -o tsv --query "nodeResourceGroup") +aksVnetName=$(az network vnet list -g $nodeResourceGroup -o tsv --query "[0].name") + +aksVnetId=$(az network vnet show -n $aksVnetName -g $nodeResourceGroup -o tsv --query "id") +az network vnet peering create -n AppGWtoAKSVnetPeering -g myResourceGroup --vnet-name myVnet --remote-vnet $aksVnetId --allow-vnet-access + +appGWVnetId=$(az network vnet show -n myVnet -g myResourceGroup -o tsv --query "id") +az network vnet peering create -n AKStoAppGWVnetPeering -g $nodeResourceGroup --vnet-name $aksVnetName --remote-vnet $appGWVnetId --allow-vnet-access +``` + +- Deploy the Ingress on the Frontend Pod + +In order to use the ingress controller to expose frontend pod, we need to update the datahub-frontend section of the values.yaml file that was used to deploy DataHub. Here is a sample configuration: + +``` +datahub-frontend: + enabled: true + image: + repository: acryldata/datahub-frontend-react + # tag: "v0.10.0 # defaults to .global.datahub.version + + # Set up ingress to expose react front-end + ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: azure/application-gateway + appgw.ingress.kubernetes.io/backend-protocol: "http" + + hosts: + - paths: + - /* + defaultUserCredentials: {} +``` + +You can then apply the updates: + +``` +helm upgrade --install datahub datahub/datahub --values values.yaml +``` + +You can now verify that the ingress was created correctly + +``` +kubectl get ingress +``` + +You should see a result like this: + +![frontend-image](https://github.com/Saketh-Mahesh/azure-docs-images/blob/main/frontend-status.png?raw=true) + +## Use PostgresSQL for the storage layer + +Configure a PostgreSQL database in the same virtual network as the Kubernetes cluster or implement virtual network peering to connect both networks. Once the database is provisioned, you should be able to see the following page under the Connect tab on the left side. + +Note: PostgreSQL Database MUST be deployed in same location as AKS/resource group (eastus, centralus, etc.) +Take a note of the connection details: + +![postgres-info](https://github.com/Saketh-Mahesh/azure-docs-images/blob/main/postgres-info.png?raw=true) + +- Update the postgresql settings under global in the values.yaml as follows. + +``` +global: + sql: + datasource: + host: "${POSTGRES_HOST}.postgres.database.azure.com:5432" + hostForpostgresqlClient: "${POSTGRES_HOST}.postgres.database.azure.com" + port: "5432" + url: "jdbc:postgresql://${POSTGRES_HOST}.postgres.database.azure.com:5432/datahub?user=${POSTGRES_ADMIN_LOGIN}&password=${POSTGRES_ADMIN_PASSWORD}&sslmode=require" + driver: "org.postgresql.Driver" + username: "${POSTGRES_ADMIN_LOGIN}" + password: + value: "${POSTGRES_ADMIN_PASSWORD}" +``` + +Run this helm command to update datahub configuration + +``` +helm upgrade --install datahub datahub/datahub --values values.yaml +``` + +And there you go! You have now installed DataHub on an Azure Kubernetes Cluster with an ingress controller set up to expose the frontend. Additionally you have utilized PostgreSQL as the storage layer of DataHub. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/confluent-cloud.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/confluent-cloud.md new file mode 100644 index 00000000..0006eeed --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/confluent-cloud.md @@ -0,0 +1,239 @@ +--- +title: Integrating with Confluent Cloud +slug: /deploy/confluent-cloud +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/deploy/confluent-cloud.md +--- +# Integrating with Confluent Cloud + +DataHub provides the ability to easily leverage Confluent Cloud as your Kafka provider. To do so, you'll need to configure DataHub to talk to a broker and schema registry hosted by Confluent. + +Doing this is a matter of configuring the Kafka Producer and Consumers used by DataHub correctly. There are 2 places where Kafka configuration should be provided: the metadata service (GMS) and the frontend server (datahub-frontend). Follow the steps below to configure these components for your deployment. + +## **Step 1: Create topics in Confluent Control Center** + +First, you'll need to create following new topics in the [Confluent Control Center](https://docs.confluent.io/platform/current/control-center/index.html). By default they have the following names: + +1. **MetadataChangeProposal_v1** +2. **FailedMetadataChangeProposal_v1** +3. **MetadataChangeLog_Versioned_v1** +4. **MetadataChangeLog_Timeseries_v1** +5. **DataHubUsageEvent_v1**: User behavior tracking event for UI +6. (Deprecated) **MetadataChangeEvent_v4**: Metadata change proposal messages +7. (Deprecated) **MetadataAuditEvent_v4**: Metadata change log messages +8. (Deprecated) **FailedMetadataChangeEvent_v4**: Failed to process #1 event +9. **MetadataGraphEvent_v4**: +10. **PlatformEvent_v1** +11. **DataHubUpgradeHistory_v1**: Notifies the end of DataHub Upgrade job so dependants can act accordingly (_eg_, startup). + Note this topic requires special configuration: **Infinite retention**. Also, 1 partition is enough for the occasional traffic. + +The first five are the most important, and are explained in more depth in [MCP/MCL](../advanced/mcp-mcl.md). The final topics are +those which are deprecated but still used under certain circumstances. It is likely that in the future they will be completely +decommissioned. + +To create the topics, navigate to your **Cluster** and click "Create Topic". Feel free to tweak the default topic configurations to +match your preferences. + +

+ +

+ +## Step 2: Configure DataHub Container to use Confluent Cloud Topics + +### Docker Compose + +If you are deploying DataHub via docker compose, enabling connection to Confluent is a matter of a) creating topics in the Confluent Control Center and b) changing the default container environment variables. + +First, configure GMS to connect to Confluent Cloud by changing `docker/gms/env/docker.env`: + +``` +KAFKA_BOOTSTRAP_SERVER=pkc-g4ml2.eu-west-2.aws.confluent.cloud:9092 +KAFKA_SCHEMAREGISTRY_URL=https://plrm-qwlpp.us-east-2.aws.confluent.cloud + +# Confluent Cloud Configs +SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL=SASL_SSL +SPRING_KAFKA_PROPERTIES_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username='XFA45EL1QFUQP4PA' password='ltyf96EvR1YYutsjLB3ZYfrk+yfCXD8sQHCE3EMp57A2jNs4RR7J1bU9k6lM6rU'; +SPRING_KAFKA_PROPERTIES_SASL_MECHANISM=PLAIN +SPRING_KAFKA_PROPERTIES_CLIENT_DNS_LOOKUP=use_all_dns_ips +SPRING_KAFKA_PROPERTIES_BASIC_AUTH_CREDENTIALS_SOURCE=USER_INFO +SPRING_KAFKA_PROPERTIES_BASIC_AUTH_USER_INFO=P2ETAN5QR2LCWL14:RTjqw7AfETDl0RZo/7R0123LhPYs2TGjFKmvMWUFnlJ3uKubFbB1Sfs7aOjjNi1m23 +``` + +Next, configure datahub-frontend to connect to Confluent Cloud by changing `docker/datahub-frontend/env/docker.env`: + +``` +KAFKA_BOOTSTRAP_SERVER=pkc-g4ml2.eu-west-2.aws.confluent.cloud:9092 + +# Confluent Cloud Configs +KAFKA_PROPERTIES_SECURITY_PROTOCOL=SASL_SSL +KAFKA_PROPERTIES_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username='XFA45EL1QFUQP4PA' password='ltyf96EvR1YYutsjLB3ZYfrk+yfCXD8sQHCE3EMp57A2jNs4RR7J1bU9k6lM6rU'; +KAFKA_PROPERTIES_SASL_MECHANISM=PLAIN +KAFKA_PROPERTIES_CLIENT_DNS_LOOKUP=use_all_dns_ips +KAFKA_PROPERTIES_BASIC_AUTH_CREDENTIALS_SOURCE=USER_INFO +KAFKA_PROPERTIES_BASIC_AUTH_USER_INFO=P2ETAN5QR2LCWL14:RTjqw7AfETDl0RZo/7R0123LhPYs2TGjFKmvMWUFnlJ3uKubFbB1Sfs7aOjjNi1m23 +``` + +Note that this step is only required if `DATAHUB_ANALYTICS_ENABLED` environment variable is not explicitly set to false for the datahub-frontend +container. + +If you're deploying with Docker Compose, you do not need to deploy the Zookeeper, Kafka Broker, or Schema Registry containers that ship by default. + +#### DataHub Actions + +Configuring Confluent Cloud for DataHub Actions requires some additional edits to your `executor.yaml`. Under the Kafka +source connection config you will need to add the Python style client connection information: + +```yaml +connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} + consumer_config: + security.protocol: ${KAFKA_PROPERTIES_SECURITY_PROTOCOL:-PLAINTEXT} + sasl.mechanism: ${KAFKA_PROPERTIES_SASL_MECHANISM:-PLAIN} + sasl.username: ${KAFKA_PROPERTIES_SASL_USERNAME} + sasl.password: ${KAFKA_PROPERTIES_SASL_PASSWORD} + schema_registry_config: + basic.auth.user.info: ${KAFKA_PROPERTIES_BASIC_AUTH_USER_INFO} +``` + +Specifically `sasl.username` and `sasl.password` are the differences from the base `executor.yaml` example file. + +Additionally, you will need to set up environment variables for `KAFKA_PROPERTIES_SASL_USERNAME` and `KAFKA_PROPERTIES_SASL_PASSWORD` +which will use the same username and API Key you generated for the JAAS config. + +See [Overwriting a System Action Config](https://github.com/datahub-project/datahub/blob/master/docker/datahub-actions/README.md#overwriting-a-system-action-config) for detailed reflection procedures. + +Next, configure datahub-actions to connect to Confluent Cloud by changing `docker/datahub-actions/env/docker.env`: + +``` +KAFKA_BOOTSTRAP_SERVER=pkc-g4ml2.eu-west-2.aws.confluent.cloud:9092 +SCHEMA_REGISTRY_URL=https://plrm-qwlpp.us-east-2.aws.confluent.cloud + +# Confluent Cloud Configs +KAFKA_PROPERTIES_SECURITY_PROTOCOL=SASL_SSL +KAFKA_PROPERTIES_SASL_USERNAME=XFA45EL1QFUQP4PA +KAFKA_PROPERTIES_SASL_PASSWORD=ltyf96EvR1YYutsjLB3ZYfrk+yfCXD8sQHCE3EMp57A2jNs4RR7J1bU9k6lM6rU +KAFKA_PROPERTIES_SASL_MECHANISM=PLAIN +KAFKA_PROPERTIES_CLIENT_DNS_LOOKUP=use_all_dns_ips +KAFKA_PROPERTIES_BASIC_AUTH_CREDENTIALS_SOURCE=USER_INFO +KAFKA_PROPERTIES_BASIC_AUTH_USER_INFO=P2ETAN5QR2LCWL14:RTjqw7AfETDl0RZo/7R0123LhPYs2TGjFKmvMWUFnlJ3uKubFbB1Sfs7aOjjNi1m23 +``` + +### Helm + +If you're deploying on K8s using Helm, you can simply change the **datahub-helm** `values.yml` to point to Confluent Cloud and disable some default containers: + +First, disable the `cp-schema-registry` service: + +``` +cp-schema-registry: + enabled: false +``` + +Next, disable the automatic creation of topics by the system update job: + +``` +global: + kafka: + precreateTopics: false +``` + +Then, update the `kafka` configurations to point to your Confluent Cloud broker and schema registry instance, along with the topics you've created in Step 1: + +``` +global: + kafka: + bootstrap: + server: pkc-g4ml2.eu-west-2.aws.confluent.cloud:9092 + schemaregistry: + url: https://plrm-qwlpp.us-east-2.aws.confluent.cloud +``` + +Next, you'll want to create 2 new Kubernetes secrets, one for the JaaS configuration which contains the username and password for Confluent, +and another for the user info used for connecting to the schema registry. You'll find the values for each within the Confluent Control Center. Specifically, +select "Clients" -> "Configure new Java Client". You should see a page like the following: + +

+ +

+ +You'll want to generate both a Kafka Cluster API Key & a Schema Registry key. Once you do so,you should see the config +automatically populate with your new secrets: + +

+ +

+ +You'll need to copy the values of `sasl.jaas.config` and `basic.auth.user.info` +for the next step. + +The next step is to create K8s secrets containing the config values you've just generated. Specifically, you can run the following commands: + +```shell +kubectl create secret generic confluent-secrets --from-literal=sasl_jaas_config="" +kubectl create secret generic confluent-secrets --from-literal=basic_auth_user_info="" +``` + +With your config values substituted as appropriate. For example, in our case we'd run: + +```shell +kubectl create secret generic confluent-secrets --from-literal=sasl_jaas_config="org.apache.kafka.common.security.plain.PlainLoginModule required username='XFA45EL1QFUQP4PA' password='ltyf96EvR1YYutsjLB3ZYfrk+yfCXD8sQHCE3EMp57A2jNs4RR7J1bU9k6lM6rU';" +kubectl create secret generic confluent-secrets --from-literal=basic_auth_user_info="P2ETAN5QR2LCWL14:RTjqw7AfETDl0RZo/7R0123LhPYs2TGjFKmvMWUFnlJ3uKubFbB1Sfs7aOjjNi1m23" +``` + +Finally, we'll configure our containers to pick up the Confluent Kafka Configs by changing two config blocks in our `values.yaml` file. You +should see these blocks commented at the bottom of the template. You'll want to uncomment them and set them to the following values: + +``` +credentialsAndCertsSecrets: + name: confluent-secrets + secureEnv: + sasl.jaas.config: sasl_jaas_config + basic.auth.user.info: basic_auth_user_info + + +springKafkaConfigurationOverrides: + security.protocol: SASL_SSL + sasl.mechanism: PLAIN + client.dns.lookup: use_all_dns_ips + basic.auth.credentials.source: USER_INFO +``` + +Then simply apply the updated `values.yaml` to your K8s cluster via `kubectl apply`. + +#### DataHub Actions + +Configuring Confluent Cloud for DataHub Actions requires some additional edits to your `executor.yaml`. Under the Kafka +source connection config you will need to add the Python style client connection information: + +```yaml +connection: + bootstrap: ${KAFKA_BOOTSTRAP_SERVER:-localhost:9092} + schema_registry_url: ${SCHEMA_REGISTRY_URL:-http://localhost:8081} + consumer_config: + security.protocol: ${KAFKA_PROPERTIES_SECURITY_PROTOCOL:-PLAINTEXT} + sasl.mechanism: ${KAFKA_PROPERTIES_SASL_MECHANISM:-PLAIN} + sasl.username: ${KAFKA_PROPERTIES_SASL_USERNAME} + sasl.password: ${KAFKA_PROPERTIES_SASL_PASSWORD} + schema_registry_config: + basic.auth.user.info: ${KAFKA_PROPERTIES_BASIC_AUTH_USER_INFO} +``` + +Specifically `sasl.username` and `sasl.password` are the differences from the base `executor.yaml` example file. + +Additionally, you will need to set up secrets for `KAFKA_PROPERTIES_SASL_USERNAME` and `KAFKA_PROPERTIES_SASL_PASSWORD` +which will use the same username and API Key you generated for the JAAS config. + +See [Overwriting a System Action Config](https://github.com/datahub-project/datahub/blob/master/docker/datahub-actions/README.md#overwriting-a-system-action-config) for detailed reflection procedures. + +```yaml +credentialsAndCertsSecrets: + name: confluent-secrets + secureEnv: + sasl.jaas.config: sasl_jaas_config + basic.auth.user.info: basic_auth_user_info + sasl.username: sasl_username + sasl.password: sasl_password +``` + +The Actions pod will automatically pick these up in the correctly named environment variables when they are named this exact way. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/environment-vars.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/environment-vars.md new file mode 100644 index 00000000..105b42b3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/environment-vars.md @@ -0,0 +1,1211 @@ +--- +title: Deployment Environment Variables +sidebar_label: Deployment Environment Variables +slug: /deploy/environment-vars +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/deploy/environment-vars.md +--- + +# Environment Variables + +The following is a summary of a few important environment variables which expose various levers which control how +DataHub works. + +--- + +# DataHub Java Components + +This includes GMS, System Update, MAE/MCE Consumers. + +## Authentication & Authorization + +Reference Links: + +- **Authentication Overview**: [Authentication Overview](../authentication/README.md) +- **Authentication Concepts**: [Authentication Concepts](../authentication/concepts.md) +- **Metadata Service Authentication**: [Introducing Metadata Service Authentication](../authentication/introducing-metadata-service-authentication.md) +- **OIDC Configuration**: [Configure OIDC Authentication](../authentication/guides/sso/configure-oidc-react.md) +- **Adding Users**: [Adding Users Guide](../authentication/guides/add-users.md) +- **Plugin Configuration**: [Plugin Documentation](../plugins.md) + +### Authentication Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------------- | ---------- | --------------------------------------------------------------------------- | --------------------------------------------------------------- | +| `METADATA_SERVICE_AUTH_ENABLED` | `true` | Enable if you want all requests to the Metadata Service to be authenticated | GMS, MAE Consumer, MCE Consumer, PE Consumer, Frontend | +| `DATAHUB_SYSTEM_CLIENT_SECRET` | | System client secret used by AuthServiceController | GMS, MAE Consumer, MCE Consumer, PE Consumer, Actions, Frontend | +| `METADATA_SERVICE_AUTHENTICATOR_EXCEPTIONS_ENABLED` | `false` | Normally failures are only warnings, enable this to throw them | GMS | +| `DATAHUB_TOKEN_SERVICE_SIGNING_KEY` | `Empty` | Key used to validate incoming tokens and sign new tokens | GMS | +| `DATAHUB_TOKEN_SERVICE_SALT` | `Empty` | Salt used for token validation and signing | GMS | +| `DATAHUB_TOKEN_SERVICE_SIGNING_ALGORITHM` | `HS256` | Signing algorithm for DataHub tokens | GMS | +| `SESSION_TOKEN_DURATION_MS` | `86400000` | The max duration of a UI session in milliseconds (defaults to 1 day) | GMS | +| `GUEST_AUTHENTICATION_USER` | `guest` | Guest user for unauthenticated access | GMS | +| `GUEST_AUTHENTICATION_ENABLED` | `false` | Enable guest authentication | GMS | + +### Authorization Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------- | ------- | ---------------------------------------------------------------- | ---------- | +| `AUTH_POLICIES_ENABLED` | `true` | Enable the default DataHub policies-based authorizer | GMS | +| `POLICY_CACHE_REFRESH_INTERVAL_SECONDS` | `120` | Cache refresh interval for policies in seconds | GMS | +| `POLICY_CACHE_FETCH_SIZE` | `1000` | Cache policy fetch size | GMS | +| `REST_API_AUTHORIZATION_ENABLED` | `true` | Enable authorization of reads, writes, and deletes on REST APIs | GMS | +| `VIEW_AUTHORIZATION_ENABLED` | `false` | Controls whether entity pages can limit access based on policies | GMS | +| `VIEW_AUTHORIZATION_RECOMMENDATIONS_PEER_GROUP_ENABLED` | `true` | Enable peer group recommendations for view authorization | GMS | + +## Ingestion Configuration + +Reference Links: + +- **CLI Configuration**: [CLI Documentation](../cli.md) +- **DataHub Actions**: [Actions Documentation](../actions/README.md) + +| Environment Variable | Default | Description | Components | +| ------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------ | ----------------- | +| `UI_INGESTION_ENABLED` | `true` | Enable UI-based ingestion | GMS, MAE Consumer | +| `INGESTION_BATCH_REFRESH_COUNT` | `100` | Number of entities to refresh in a single batch when refreshing entities after ingestion | GMS | +| `INGESTION_SOURCE_REFRESH_INTERVAL_SECONDS` | `43200` | Interval at which the ingestion source scheduler will check for new or updated ingestion sources | GMS | + +## Telemetry & Analytics + +| Environment Variable | Default | Description | Components | +| ----------------------------- | ------- | ------------------------------------ | ---------- | +| `INGESTION_REPORTING_ENABLED` | `false` | Enable ingestion reporting | GMS | +| `ENABLE_THIRD_PARTY_LOGGING` | `false` | Whether mixpanel tracking is enabled | GMS | + +## DataHub Core Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------------------- | ----------- | ----------------------------------------------------------------- | ---------- | +| `DATAHUB_SERVER_TYPE` | `prod` | DataHub server type | GMS | +| `DATAHUB_GMS_ASYNC_REQUEST_TIMEOUT_MS` | `55000` | Async request timeout for GMS | GMS | +| `DATAHUB_GMS_HOST` | `localhost` | GMS host | Frontend | +| `DATAHUB_GMS_PORT` | `8080` | GMS port | Frontend | +| `DATAHUB_GMS_USE_SSL` | `false` | Use SSL for GMS connections | Frontend | +| `DATAHUB_GMS_URI` | `null` | URI instead of separate host/port/ssl parameters (takes priority) | Frontend | +| `DATAHUB_GMS_SSL_PROTOCOL` | `null` | SSL protocol for GMS | Frontend | + +### Plugin Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------------------------------------- | -------------------------------- | ------------------------------------------------------ | ---------- | +| `PLUGIN_SECURITY_MODE` | `RESTRICTED` | Plugin security mode (RESTRICTED or LENIENT) | GMS | +| `ENTITY_REGISTRY_PLUGIN_PATH` | `/etc/datahub/plugins/models` | Path for entity registry plugins | GMS | +| `ENTITY_REGISTRY_PLUGIN_LOAD_DELAY_SECONDS` | `60` | Rate at which plugin runnable executes | GMS | +| `IGNORE_FAILURE_WHEN_LOADING_ENTITY_REGISTRY_PLUGIN` | `true` | Whether to ignore failure when loading entity registry | GMS | +| `RETENTION_PLUGIN_PATH` | `/etc/datahub/plugins/retention` | Path for retention plugins | GMS | +| `AUTH_PLUGIN_PATH` | `/etc/datahub/plugins/auth` | Path for auth plugins | GMS | + +### Metrics Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------- | --------------------------------- | ---------------------------------------------- | ----------------- | +| `DATAHUB_METRICS_HOOK_LATENCY_PERCENTILES` | `0.5,0.95,0.99,0.999` | Hook latency percentiles | GMS, MAE Consumer | +| `DATAHUB_METRICS_HOOK_LATENCY_SERVICE_LEVEL_OBJECTIVES` | `300,1800,3000,10800,21600,43200` | Hook latency SLOs in seconds | GMS, MAE Consumer | +| `DATAHUB_METRICS_HOOK_LATENCY_MAX_EXPECTED_VALUE` | `86000` | Maximum expected hook latency value in seconds | GMS, MAE Consumer | + +## Entity Service Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------ | ------- | ----------------------------- | ----------------- | +| `ENTITY_SERVICE_IMPL` | `ebean` | Entity service implementation | GMS, MCE Consumer | +| `ENTITY_SERVICE_ENABLE_RETENTION` | `true` | Enable entity retention | GMS, MCE Consumer | +| `ENTITY_SERVICE_APPLY_RETENTION_BOOTSTRAP` | `false` | Apply retention on bootstrap | GMS, MCE Consumer | + +### Aspect Size Validation + +Protects against aspects exceeding deserialization limits. **Debugging flags - enable only when troubleshooting service crashes or memory pressure from oversized aspects.** + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------- | ---------- | +| `DATAHUB_VALIDATION_ASPECT_SIZE_PRE_PATCH_ENABLED` | `false` | Enable pre-patch validation - checks existing aspects from DB before patch application | GMS | +| `DATAHUB_VALIDATION_ASPECT_SIZE_PRE_PATCH_WARN_SIZE_BYTES` | `null` | Optional warning threshold in bytes. Logs warning when exceeded but allows write. Should be lower than maxSizeBytes. | GMS | +| `DATAHUB_VALIDATION_ASPECT_SIZE_PRE_PATCH_MAX_SIZE_BYTES` | 16000000 | Max size in bytes for pre-patch aspects (16MB, matches `INGESTION_MAX_SERIALIZED_STRING_LENGTH`). Skips write when exceeded. | GMS | +| `DATAHUB_VALIDATION_ASPECT_SIZE_PRE_PATCH_OVERSIZED_REMEDIATION` | `IGNORE` | Remediation for oversized pre-patch aspects: `IGNORE` (skip write), `DELETE` (skip write and delete aspect) | GMS | +| `DATAHUB_VALIDATION_ASPECT_SIZE_POST_PATCH_ENABLED` | `false` | Enable post-patch validation - checks aspects after patch application, before DB write | GMS | +| `DATAHUB_VALIDATION_ASPECT_SIZE_POST_PATCH_WARN_SIZE_BYTES` | `null` | Optional warning threshold in bytes. Logs warning when exceeded but allows write. Should be lower than maxSizeBytes. | GMS | +| `DATAHUB_VALIDATION_ASPECT_SIZE_POST_PATCH_MAX_SIZE_BYTES` | 16000000 | Max size in bytes for post-patch aspects (16MB, matches `INGESTION_MAX_SERIALIZED_STRING_LENGTH`). Skips write when exceeded. | GMS | +| `DATAHUB_VALIDATION_ASPECT_SIZE_POST_PATCH_OVERSIZED_REMEDIATION` | `IGNORE` | Remediation for oversized post-patch aspects: `IGNORE` (skip write), `DELETE` (skip write and delete aspect) | GMS | + +**Validation points:** + +- **Pre-patch:** Validates existing aspect from database before applying patches. Zero overhead (uses JSON already fetched from DB). +- **Post-patch:** Validates aspect after patch application, before DB write. Zero overhead (uses JSON already created for DB write). + +**Remediation strategies:** + +- `IGNORE`: Logs warning, skips write, routes MCP to FailedMetadataChangeProposal topic. Pre-patch: existing oversized aspect remains in database. +- `DELETE`: Logs warning, skips write, routes MCP to FailedMetadataChangeProposal topic, and deletes the aspect. + +**When to enable:** Use temporarily when investigating GMS crashes, debugging memory pressure, or cleaning up pre-existing oversized data. Prefer fixing the root cause at ingestion time. + +See [MCP/MCL Events - Aspect Size Validation](../advanced/mcp-mcl.md#aspect-size-validation) for configuration examples and troubleshooting. + +## Graph Service Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------------------------- | --------------- | --------------------------------------------------------------------------- | ----------------- | +| `GRAPH_SERVICE_IMPL` | `elasticsearch` | Graph service implementation | GMS, MAE Consumer | +| `GRAPH_SERVICE_LIMIT_RESULTS_MAX` | `10000` | Maximum allowed result count for queries | GMS | +| `GRAPH_SERVICE_LIMIT_RESULTS_API_DEFAULT` | `5000` | Default API result limit | GMS | +| `GRAPH_SERVICE_LIMIT_RESULTS_STRICT` | `false` | Throw exception if strict is true, otherwise override with default and warn | GMS | + +## Search Service Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------------- | ------------------- | --------------------------------------------------------------------------- | ---------- | +| `SEARCH_SERVICE_BATCH_SIZE` | `100` | Search service batch size | GMS | +| `SEARCH_SERVICE_ENABLE_CACHE` | `false` | Enable search service cache | GMS | +| `SEARCH_SERVICE_ENABLE_CACHE_EVICTION` | `false` | Enable search service cache eviction | GMS | +| `SEARCH_SERVICE_CACHE_IMPLEMENTATION` | `caffeine` | Search service cache implementation | GMS | +| `SEARCH_SERVICE_HAZELCAST_SERVICE_NAME` | `hazelcast-service` | Hazelcast service name for search cache | GMS | +| `SEARCH_SERVICE_FILTER_CONTAINER_EXPANSION_ENABLED` | `true` | Enable container expansion in search filters | GMS | +| `SEARCH_SERVICE_FILTER_CONTAINER_EXPANSION_PAGE_SIZE` | `100` | Page size for container expansion | GMS | +| `SEARCH_SERVICE_FILTER_CONTAINER_EXPANSION_LIMIT` | `100` | Limit for container expansion | GMS | +| `SEARCH_SERVICE_FILTER_DOMAIN_EXPANSION_ENABLED` | `true` | Enable domain expansion in search filters | GMS | +| `SEARCH_SERVICE_FILTER_DOMAIN_EXPANSION_PAGE_SIZE` | `100` | Page size for domain expansion | GMS | +| `SEARCH_SERVICE_FILTER_DOMAIN_EXPANSION_LIMIT` | `100` | Limit for domain expansion | GMS | +| `SEARCH_SERVICE_LIMIT_RESULTS_MAX` | `10000` | Maximum allowed result count for queries | GMS | +| `SEARCH_SERVICE_LIMIT_RESULTS_API_DEFAULT` | `5000` | Default API result limit | GMS | +| `SEARCH_SERVICE_LIMIT_RESULTS_STRICT` | `false` | Throw exception if strict is true, otherwise override with default and warn | GMS | + +## Timeseries Aspect Service + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------------- | ------- | --------------------------------------------------------------------------- | ---------- | +| `TIMESERIES_ASPECT_SERVICE_QUERY_CONCURRENCY` | `10` | Parallel threads for timeseries queries | GMS | +| `TIMESERIES_ASPECT_SERVICE_QUERY_QUEUE_SIZE` | `500` | Queue size for timeseries queries | GMS | +| `TIMESERIES_ASPECT_SERVICE_QUERY_THREAD_KEEP_ALIVE` | `60` | Thread keep alive time for timeseries queries | GMS | +| `TIMESERIES_ASPECT_SERVICE_LIMIT_RESULTS_MAX` | `10000` | Maximum allowed result count for queries | GMS | +| `TIMESERIES_ASPECT_SERVICE_LIMIT_RESULTS_API_DEFAULT` | `5000` | Default API result limit | GMS | +| `TIMESERIES_ASPECT_SERVICE_LIMIT_RESULTS_STRICT` | `false` | Throw exception if strict is true, otherwise override with default and warn | GMS | + +## System Metadata Service + +| Environment Variable | Default | Description | Components | +| --------------------------------------------------- | ------- | --------------------------------------------------------------------------- | ---------- | +| `SYSTEM_METADATA_SERVICE_LIMIT_RESULTS_MAX` | `10000` | Maximum allowed result count for queries | GMS | +| `SYSTEM_METADATA_SERVICE_LIMIT_RESULTS_API_DEFAULT` | `5000` | Default API result limit | GMS | +| `SYSTEM_METADATA_SERVICE_LIMIT_RESULTS_STRICT` | `false` | Throw exception if strict is true, otherwise override with default and warn | GMS | + +## Platform Analytics + +| Environment Variable | Default | Description | Components | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | --------------------------- | +| `DATAHUB_ANALYTICS_ENABLED` | `true` | Enable platform analytics | GMS, MAE Consumer, Frontend | +| `DATAHUB_ANALYTICS_TRACING_ENABLED` | `true` | Enable backend usage tracing | GMS | +| `ANALYTICS_DATAHUB_USAGE_EVENT_TYPES` | `CreateAccessTokenEvent,CreatePolicyEvent,UpdatePolicyEvent,CreateIngestionSourceEvent,UpdateIngestionSourceEvent,RevokeAccessTokenEvent,CreateUserEvent,UpdateUserEvent,DeletePolicyEvent` | Comma separated list of usage event types to listen to | GMS | +| `ANALYTICS_GENERIC_ASPECT_TYPES` | `` | Filter list for generic aspect events | GMS | +| `ANALYTICS_USER_FILTERS` | `` | Filter out specific users' events from being published | GMS | + +## Visual Configuration + +### Queries Tab + +| Environment Variable | Default | Description | Components | +| ----------------------------------- | ------- | -------------------------------------- | ---------- | +| `REACT_APP_QUERIES_TAB_RESULT_SIZE` | `5` | Queries tab result size (experimental) | Frontend | + +### Theme Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------- | ------- | ------------------------------------------------- | ---------- | +| `REACT_APP_CUSTOM_THEME_ID` | `` | Custom theme ID for rendering specific theme file | Frontend | + +### Assets Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------- | ----------------------------------- | ------------------------------- | ---------- | +| `REACT_APP_LOGO_URL` | `/assets/platforms/datahublogo.png` | Logo URL for the application | Frontend | +| `REACT_APP_FAVICON_URL` | `/assets/icons/favicon.ico` | Favicon URL for the application | Frontend | +| `REACT_APP_TITLE` | `` | Application title | Frontend | + +### UI Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------- | ------- | ---------------------------------------------------------------------------------- | ---------- | +| `REACT_APP_HIDE_GLOSSARY` | `false` | Hide glossary in the UI | Frontend | +| `REACT_APP_SHOW_FULL_TITLE_IN_LINEAGE` | `false` | Show full title in lineage | Frontend | +| `DOMAIN_DEFAULT_TAB` | `` | Default tab for domains (set to DOCUMENTATION_TAB to show documentation tab first) | Frontend | +| `APPLICATION_SHOW_SIDEBAR_SECTION_WHEN_EMPTY` | `false` | Show sidebar section when empty (deprecated) | Frontend | +| `SEARCH_RESULT_NAME_HIGHLIGHT_ENABLED` | `true` | Enable visual highlighting on search result names/descriptions | Frontend | + +## Storage Layer Configuration + +### EBean Configuration (MySQL/PostgreSQL) + +| Environment Variable | Default | Description | Components | +| --------------------------------- | ------------------------------------- | ----------------------------------------------- | -------------------------------- | +| `EBEAN_DATASOURCE_USERNAME` | `datahub` | Database username | GMS, MCE Consumer, System Update | +| `EBEAN_DATASOURCE_PASSWORD` | `datahub` | Database password | GMS, MCE Consumer, System Update | +| `EBEAN_DATASOURCE_URL` | `jdbc:mysql://localhost:3306/datahub` | JDBC URL | GMS, MCE Consumer, System Update | +| `EBEAN_DATASOURCE_DRIVER` | `com.mysql.jdbc.Driver` | JDBC Driver | GMS, MCE Consumer, System Update | +| `EBEAN_MIN_CONNECTIONS` | `2` | Minimum database connections | GMS, MCE Consumer, System Update | +| `EBEAN_MAX_CONNECTIONS` | `50` | Maximum database connections | GMS, MCE Consumer, System Update | +| `EBEAN_MAX_INACTIVE_TIME_IN_SECS` | `120` | Maximum inactive time in seconds | GMS, MCE Consumer, System Update | +| `EBEAN_MAX_AGE_MINUTES` | `120` | Maximum age in minutes | GMS, MCE Consumer, System Update | +| `EBEAN_LEAK_TIME_MINUTES` | `15` | Leak time in minutes | GMS, MCE Consumer, System Update | +| `EBEAN_WAIT_TIMEOUT_MILLIS` | `1000` | Wait timeout in milliseconds | GMS, MCE Consumer, System Update | +| `EBEAN_AUTOCREATE` | `false` | Auto-create DDL | GMS, MCE Consumer, System Update | +| `EBEAN_POSTGRES_USE_AWS_IAM_AUTH` | `false` | Use AWS IAM authentication for PostgreSQL | GMS, MCE Consumer, System Update | +| `EBEAN_USE_IAM_AUTH` | `false` | Enable cross-cloud IAM authentication (AWS/GCP) | GMS, MCE Consumer, System Update | +| `EBEAN_CLOUD_PROVIDER` | `auto` | Cloud provider (auto/aws/gcp/traditional) | GMS, MCE Consumer, System Update | +| `EBEAN_BATCH_GET_METHOD` | `IN` | Batch get method (IN or UNION) | GMS, MCE Consumer, System Update | +| `EBEAN_URL` | _same as EBEAN_DATASOURCE_URL_ | Alternative property for database URL | System Update | +| `EBEAN_MAX_TRANSACTION_RETRY` | `null` | Maximum transaction retries for Ebean | System Update | + +#### Cross-Cloud IAM Authentication + +DataHub supports cross-cloud IAM authentication for both AWS and GCP cloud providers. This enables secure, passwordless database connections using cloud identity services. + +**AWS IAM Authentication:** + +- **PostgreSQL**: Uses native `wrapperPlugins: "iam"` configuration +- **MySQL**: Automatically swaps to MariaDB driver with `credentialType=AWS-IAM` +- **Detection**: Based on `AWS_REGION`, `AWS_ACCESS_KEY_ID`, or RDS URLs + +**GCP IAM Authentication:** + +- **MySQL/PostgreSQL**: Uses Cloud SQL Connector with `enableIamAuth=true` +- **Detection**: Based on `GOOGLE_APPLICATION_CREDENTIALS`, `GCP_PROJECT`, or Cloud SQL URLs + +**Configuration Examples:** + +```bash +# AWS RDS with IAM authentication +export EBEAN_USE_IAM_AUTH=true +export EBEAN_CLOUD_PROVIDER=aws +export EBEAN_DATASOURCE_URL=jdbc:mysql://rds-instance.amazonaws.com:3306/datahub +export AWS_REGION=us-west-2 + +# GCP Cloud SQL with IAM authentication +export EBEAN_USE_IAM_AUTH=true +export EBEAN_CLOUD_PROVIDER=gcp +export EBEAN_DATASOURCE_URL=jdbc:mysql://cloudsql-instance:3306/datahub +export INSTANCE_CONNECTION_NAME="project:region:instance" +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" + +# Auto-detection (recommended) +export EBEAN_USE_IAM_AUTH=true +export EBEAN_CLOUD_PROVIDER=auto +# Cloud provider automatically detected from environment variables +``` + +**Required Cloud-Specific Environment Variables:** + +| Cloud Provider | Required Variables | Description | +| -------------- | -------------------------------- | ------------------------------------------------------- | +| **AWS** | `AWS_REGION` | AWS region for RDS instances | +| **AWS** | `AWS_ACCESS_KEY_ID` | AWS access key (or use instance profile) | +| **AWS** | `AWS_SECRET_ACCESS_KEY` | AWS secret key (or use instance profile) | +| **AWS** | `AWS_SESSION_TOKEN` | AWS session token (optional, for temporary credentials) | +| **GCP** | `INSTANCE_CONNECTION_NAME` | Cloud SQL instance connection name | +| **GCP** | `GOOGLE_APPLICATION_CREDENTIALS` | Path to service account JSON file | +| **GCP** | `GCP_PROJECT` | GCP project ID (optional, for auto-detection) | + +### SQL Setup Configuration + +The SQL Setup system provides automated database initialization and user management capabilities. These environment variables control the behavior of the `SqlSetup` upgrade step. + +| Environment Variable | Default | Description | Components | +| ---------------------------- | ------------- | ---------------------------------------------------------------------------- | ------------- | +| `DATAHUB_SQL_SETUP_ENABLED` | `false` | Enable SQL setup functionality (alternative to passing SqlSetup upgrade arg) | System Update | +| `CREATE_TABLES` | `true` | Whether to create database tables | System Update | +| `CREATE_DB` | `true` | Whether to create the database (PostgreSQL only) | System Update | +| `CREATE_USER` | `false` | Whether to create a new database user | System Update | +| `CREATE_USER_USERNAME` | _none_ | Username for the new database user to create (required if CREATE_USER=true) | System Update | +| `CREATE_USER_PASSWORD` | _none_ | Password for the new database user to create (required for traditional auth) | System Update | +| `CDC_MCL_PROCESSING_ENABLED` | `false` | Whether to create a CDC (Change Data Capture) user | System Update | +| `CDC_USER` | `datahub_cdc` | Username for the CDC user | System Update | +| `CDC_PASSWORD` | `datahub_cdc` | Password for the CDC user | System Update | + +**Note:** When `CREATE_USER=true`, you must explicitly set `CREATE_USER_USERNAME` environment variable. The system will not fall back to Ebean connection credentials for security reasons. + +### Kubernetes scale-down (system update) + +When the system-update job runs in a Kubernetes cluster, it can optionally prepare for blocking upgrades (e.g. reindex) by scaling down selected deployments and updating environment variables on others. Scale-down is conditional: it runs only when a blocking upgrade (such as BuildIndices when reindex is needed) requires it. + +- **Scale-to-zero:** Deployments matched by label selectors (e.g. MAE/MCE consumers) are scaled to zero; their KEDA ScaledObjects are removed. Deployments that do not exist are skipped. +- **Deployment env updates:** A JSON array configures which deployments (by label selector) receive which env vars when scaling down (e.g. GMS: disable embedded consumers). Previous env values are stored and restored on failure or when retries are exceeded. +- **Parallel execution:** Rollout and scale-down operations run in parallel across deployments to reduce total wait time. +- **Retries and restore:** State (replica counts and env per deployment) is stored in a ConfigMap. If the step runs again (e.g. job restart), the attempt count increments. When attempts exceed `maxRetries`, the step restores all saved state, deletes the ConfigMap, and fails so the next run is not blocked. + +These variables are typically set by the Helm chart for the system-update job; they are only used when the job runs in-cluster. + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------- | ------------- | +| `DATAHUB_UPGRADE_K8_SCALE_DOWN_ENABLED` | `true` | Master switch for K8 scale-down (must be true for scale-down to run) | System Update | +| `DATAHUB_UPGRADE_K8_SCALE_DOWN_JAVA_ENABLED` | `false` | Use the Java-based scale-down step (opt-in; set true to enable) | System Update | +| `DATAHUB_UPGRADE_K8_SCALE_DOWN_MAX_RETRIES` | `3` | Max scale-down attempts across job restarts before restoring state, deleting the ConfigMap, failing | System Update | +| `DATAHUB_UPGRADE_K8_SCALE_DOWN_LABEL_SELECTORS` | (Helm) | Comma-separated label selectors for deployments to scale to zero (e.g. MAE, MCE) | System Update | +| `DATAHUB_UPGRADE_K8_DEPLOYMENT_ENV_UPDATES` | (Helm) | JSON array of `{"labelSelector":"...","env":{...}}` for deployments that get env vars when scaling down | System Update | +| `DATAHUB_UPGRADE_K8_ROLLOUT_MAX_WAIT_SECONDS` | `1800` | Max seconds to wait per deployment rollout (scale or env change); default 30 min | System Update | +| `DATAHUB_UPGRADE_K8_ROLLOUT_POLL_SECONDS` | `5` | Seconds between polls when waiting for rollout | System Update | +| `NAMESPACE` | (pod) | Kubernetes namespace (set via downward API in Helm) | System Update | +| `HELM_RELEASE_NAME` | (Helm) | Helm release name (used for state ConfigMap name) | System Update | + +### Kubernetes Operations API (OpenAPI) + +When GMS runs inside a Kubernetes cluster, the OpenAPI service can expose a Kubernetes operations controller that allows listing and modifying cluster resources (deployments, pods, config maps, cron jobs) in the current namespace. This is disabled when not in a K8 environment or when the flag below is false. + +| Environment Variable | Default | Description | Components | +| ----------------------------------- | ------- | ----------------------------------------------------------------------------------------------------- | ---------- | +| `KUBERNETES_OPERATIONS_API_ENABLED` | `true` | Enable the OpenAPI Kubernetes operations controller. Set to `false` to disable the K8 operations API. | GMS | + +**IAM Authentication:** IAM authentication is automatically detected when `CREATE_USER=true` and `CREATE_USER_PASSWORD` is not set or is empty. The system will create users with IAM authentication for supported cloud databases: + +- **AWS RDS MySQL**: Creates user with AWSAuthenticationPlugin +- **AWS RDS PostgreSQL**: Creates user and grants `rds_iam` role +- **GCP Cloud SQL**: IAM authentication is managed through Cloud SQL IAM database users + +When using traditional username/password authentication, both `CREATE_USER_USERNAME` and `CREATE_USER_PASSWORD` must be set. + +### Cassandra Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------- | ------------- | --------------------- | -------------------------------- | +| `CASSANDRA_DATASOURCE_USERNAME` | `cassandra` | Cassandra username | GMS, MCE Consumer, System Update | +| `CASSANDRA_DATASOURCE_PASSWORD` | `cassandra` | Cassandra password | GMS, MCE Consumer, System Update | +| `CASSANDRA_HOSTS` | `cassandra` | Cassandra hosts | GMS, MCE Consumer, System Update | +| `CASSANDRA_PORT` | `9042` | Cassandra port | GMS, MCE Consumer, System Update | +| `CASSANDRA_DATACENTER` | `datacenter1` | Cassandra datacenter | GMS, MCE Consumer, System Update | +| `CASSANDRA_KEYSPACE` | `datahub` | Cassandra keyspace | GMS, MCE Consumer, System Update | +| `CASSANDRA_USE_SSL` | `false` | Use SSL for Cassandra | GMS, MCE Consumer, System Update | + +### Elasticsearch Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------ | --------------- | ------------------------------------------------------------ | ---------------------------------------------- | +| `ELASTICSEARCH_HOST` | `localhost` | Elasticsearch host | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_PORT` | `9200` | Elasticsearch port | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_THREAD_COUNT` | `2` | Elasticsearch thread count | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_CONNECTION_REQUEST_TIMEOUT` | `5000` | Connection request timeout (in milliseconds) | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SOCKET_TIMEOUT` | `30000` | Socket timeout for established connections (in milliseconds) | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_USERNAME` | `null` | Elasticsearch username | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_PASSWORD` | `null` | Elasticsearch password | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_PATH_PREFIX` | `null` | Elasticsearch path prefix | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_USE_SSL` | `false` | Use SSL for Elasticsearch | GMS, MAE Consumer, MCE Consumer, System Update | +| `OPENSEARCH_USE_AWS_IAM_AUTH` | `false` | Use AWS IAM authentication for OpenSearch | GMS, MAE Consumer, MCE Consumer, System Update | +| `AWS_REGION` | `null` | AWS region | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_IMPLEMENTATION` | `elasticsearch` | Implementation (elasticsearch or opensearch) | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTIC_ID_HASH_ALGO` | `MD5` | ID hash algorithm | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_DATA_NODE_COUNT` | `1` | Number of Elasticsearch data nodes | GMS, MAE Consumer, MCE Consumer, System Update | + +#### SSL Context Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------- | ------- | -------------------------------- | ---------------------------------------------- | +| `ELASTICSEARCH_SSL_PROTOCOL` | `null` | SSL protocol | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_SECURE_RANDOM_IMPL` | `null` | SSL secure random implementation | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_TRUSTSTORE_FILE` | `null` | SSL truststore file | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_TRUSTSTORE_TYPE` | `null` | SSL truststore type | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_TRUSTSTORE_PASSWORD` | `null` | SSL truststore password | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_KEYSTORE_FILE` | `null` | SSL keystore file | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_KEYSTORE_TYPE` | `null` | SSL keystore type | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_KEYSTORE_PASSWORD` | `null` | SSL keystore password | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_SSL_KEY_PASSWORD` | `null` | SSL key password | GMS, MAE Consumer, MCE Consumer, System Update | + +#### Bulk Operations Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------ | --------- | ----------------------------- | ----------------- | +| `ES_BULK_DELETE_BATCH_SIZE` | `5000` | Bulk delete batch size | GMS, MAE Consumer | +| `ES_BULK_DELETE_SLICES` | `auto` | Bulk delete slices | GMS, MAE Consumer | +| `ES_BULK_DELETE_POLL_INTERVAL` | `30` | Bulk delete poll interval | GMS, MAE Consumer | +| `ES_BULK_DELETE_POLL_UNIT` | `SECONDS` | Bulk delete poll unit | GMS, MAE Consumer | +| `ES_BULK_DELETE_TIMEOUT` | `30` | Bulk delete timeout | GMS, MAE Consumer | +| `ES_BULK_DELETE_TIMEOUT_UNIT` | `MINUTES` | Bulk delete timeout unit | GMS, MAE Consumer | +| `ES_BULK_DELETE_NUM_RETRIES` | `3` | Bulk delete number of retries | GMS, MAE Consumer | +| `ES_BULK_ASYNC` | `true` | Enable async bulk operations | GMS, MAE Consumer | +| `ES_BULK_REQUESTS_LIMIT` | `1000` | Bulk requests limit | GMS, MAE Consumer | +| `ES_BULK_FLUSH_PERIOD` | `1` | Bulk flush period | GMS, MAE Consumer | +| `ES_BULK_NUM_RETRIES` | `3` | Bulk number of retries | GMS, MAE Consumer | +| `ES_BULK_RETRY_INTERVAL` | `1` | Bulk retry interval | GMS, MAE Consumer | +| `ES_BULK_REFRESH_POLICY` | `NONE` | Bulk refresh policy | GMS, MAE Consumer | +| `ES_BULK_ENABLE_BATCH_DELETE` | `false` | Enable batch delete | GMS, MAE Consumer | + +#### Index Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------------------------------------------- | ------- | --------------------------------------- | ---------------------------------------------- | +| `INDEX_PREFIX` | `` | Index prefix | GMS, MAE Consumer, MCE Consumer, System Update | +| `ELASTICSEARCH_INDEX_DOC_IDS_SCHEMA_FIELD_HASH_ID_ENABLED` | `false` | Enable hash ID for schema field doc IDs | GMS, MAE Consumer, MCE Consumer, System Update | + +#### Build Indices Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------------------------- | -------------------------------- | -------------------------------------------------------------------- | ------------- | +| `ELASTICSEARCH_BUILD_INDICES_ALLOW_DOC_COUNT_MISMATCH` | `false` | Allow document count mismatch when clone indices is enabled | System Update | +| `ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES` | `true` | Clone indices | System Update | +| `ELASTICSEARCH_BUILD_INDICES_RETENTION_UNIT` | `DAYS` | Retention unit for indices | System Update | +| `ELASTICSEARCH_BUILD_INDICES_RETENTION_VALUE` | `60` | Retention value for indices | System Update | +| `ELASTICSEARCH_BUILD_INDICES_REINDEX_OPTIMIZATION_ENABLED` | `true` | Enable reindex optimization | System Update | +| `ELASTICSEARCH_BUILD_INDICES_REINDEX_BATCH_SIZE` | `5000` | Documents per scroll batch during reindex | System Update | +| `ELASTICSEARCH_BUILD_INDICES_REINDEX_MAX_SLICES` | `256` | Maximum parallel reindex slices (capped from target shards) | System Update | +| `ELASTICSEARCH_BUILD_INDICES_REINDEX_NO_PROGRESS_RETRY_MINUTES` | `5` | Minutes without document-count progress before re-triggering reindex | System Update | +| `ELASTICSEARCH_NUM_SHARDS_PER_INDEX` | `${elasticsearch.dataNodeCount}` | Number of shards per index, defaults to dataNodeCount | System Update | +| `ELASTICSEARCH_NUM_REPLICAS_PER_INDEX` | `1` | Number of replicas per index | System Update | +| `ELASTICSEARCH_INDEX_BUILDER_NUM_RETRIES` | `3` | Index builder number of retries | System Update | +| `ELASTICSEARCH_INDEX_BUILDER_REFRESH_INTERVAL_SECONDS` | `3` | Index builder refresh interval | System Update | +| `SEARCH_DOCUMENT_MAX_ARRAY_LENGTH` | `1000` | Maximum array length in search documents | System Update | +| `SEARCH_DOCUMENT_MAX_OBJECT_KEYS` | `1000` | Maximum object keys in search documents | System Update | +| `SEARCH_DOCUMENT_MAX_VALUE_LENGTH` | `4096` | Maximum value length in search documents | System Update | +| `ELASTICSEARCH_MAIN_TOKENIZER` | `null` | Main tokenizer | System Update | +| `ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX` | `false` | Enable mappings reindex | System Update | +| `ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX` | `false` | Enable settings reindex | System Update | +| `ELASTICSEARCH_INDEX_BUILDER_MAX_REINDEX_HOURS` | `0` | Maximum reindex hours (0 = no timeout) | System Update | +| `ELASTICSEARCH_INDEX_BUILDER_SETTINGS_OVERRIDES` | `null` | Index builder settings overrides | System Update | +| `ELASTICSEARCH_MIN_SEARCH_FILTER_LENGTH` | `3` | Minimum search filter length | System Update | +| `ELASTICSEARCH_INDEX_BUILDER_ENTITY_SETTINGS_OVERRIDES` | `null` | Entity settings overrides | System Update | + +#### Search Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------- | -------------------- | ---------------------------------------------- | ---------- | +| `ELASTICSEARCH_QUERY_MAX_TERM_BUCKET_SIZE` | `60` | Maximum term bucket size | GMS | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_EXCLUSIVE` | `false` | Only return exact matches when using quotes | GMS | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_WITH_PREFIX` | `true` | Include prefix match in exact match results | GMS | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_FACTOR` | `16.0` | Multiply by this number on true exact match | GMS | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_PREFIX_FACTOR` | `1.1` | Multiply by this number when prefix match | GMS | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_CASE_FACTOR` | `0.0` | Stacked boost multiplier when case mismatch | GMS | +| `ELASTICSEARCH_QUERY_EXACT_MATCH_ENABLE_STRUCTURED` | `true` | Enable exact match on structured search | GMS | +| `ELASTICSEARCH_QUERY_TWO_GRAM_FACTOR` | `1.2` | Boost multiplier when match on 2-gram tokens | GMS | +| `ELASTICSEARCH_QUERY_THREE_GRAM_FACTOR` | `1.5` | Boost multiplier when match on 3-gram tokens | GMS | +| `ELASTICSEARCH_QUERY_FOUR_GRAM_FACTOR` | `1.8` | Boost multiplier when match on 4-gram tokens | GMS | +| `ELASTICSEARCH_QUERY_PARTIAL_URN_FACTOR` | `0.5` | Multiplier on Urn token match | GMS | +| `ELASTICSEARCH_QUERY_PARTIAL_FACTOR` | `0.4` | Multiplier on possible non-Urn token match | GMS | +| `ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED` | `true` | Enable search query and ranking customization | GMS | +| `ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE` | `search_config.yaml` | Location of search customization configuration | GMS | +| `ELASTICSEARCH_QUERY_SEARCH_FIELD_CONFIG_DEFAULT` | `legacy` | Default field configuration for search | GMS | +| `ELASTICSEARCH_QUERY_AUTOCOMPLETE_FIELD_CONFIG_DEFAULT` | `legacy` | Default field configuration for autocomplete | GMS | + +#### Graph Search Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------------------- | -------------------------------- | ----------------------------------------------------------------------------------------------------- | ---------- | +| `ELASTICSEARCH_SEARCH_GRAPH_TIMEOUT_SECONDS` | `50` | Graph DAO timeout seconds | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_BATCH_SIZE` | `1000` | Graph DAO batch size | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_MULTI_PATH_SEARCH` | `false` | Allow path retraversal for all paths | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_BOOST_VIA_NODES` | `true` | Boost graph edges with via nodes | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_STATUS_ENABLED` | `false` | Enable soft delete tracking of URNs on edges | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_LINEAGE_MAX_HOPS` | `20` | Maximum hops to traverse lineage graph | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_IMPACT_MAX_HOPS` | `1000` | Maximum hops to traverse for impact analysis (impact.maxHops) | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_IMPACT_MAX_RELATIONS` | `40000` | Maximum number of relationships for impact analysis (impact.maxRelations) | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_IMPACT_SLICES` | `${elasticsearch.dataNodeCount}` | Number of slices for parallel search operations (impact.slices), defaults to dataNodeCount, minimum 2 | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_IMPACT_KEEP_ALIVE` | `5m` | Point-in-Time keepAlive duration for impact analysis queries (impact.keepAlive) | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_IMPACT_PARTIAL_RESULTS` | `false` | If true, return partial results when maxRelations is reached; if false (default), throw an error | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_IMPACT_MAX_THREADS` | `32` | Maximum parallel lineage graph queries | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_QUERY_OPTIMIZATION` | `true` | Reduce query nesting if possible | GMS | +| `ELASTICSEARCH_SEARCH_GRAPH_POINT_IN_TIME_CREATION_ENABLED` | `true` | Enable creation of point in time snapshots for graph queries | GMS | + +### Neo4j Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------------- | ------------------ | -------------------------------------- | -------------------------------- | +| `NEO4J_USERNAME` | `neo4j` | Neo4j username | GMS, MAE Consumer, System Update | +| `NEO4J_PASSWORD` | `datahub` | Neo4j password | GMS, MAE Consumer, System Update | +| `NEO4J_URI` | `bolt://localhost` | Neo4j URI | GMS, MAE Consumer, System Update | +| `NEO4J_DATABASE` | `graph.db` | Neo4j database | GMS, MAE Consumer, System Update | +| `NEO4J_MAX_CONNECTION_POOL_SIZE` | `100` | Maximum connection pool size | GMS, MAE Consumer, System Update | +| `NEO4J_MAX_CONNECTION_ACQUISITION_TIMEOUT_IN_SECONDS` | `60` | Maximum connection acquisition timeout | GMS, MAE Consumer, System Update | +| `NEO4j_MAX_CONNECTION_LIFETIME_IN_SECONDS` | `3600` | Maximum connection lifetime | GMS, MAE Consumer, System Update | +| `NEO4J_MAX_TRANSACTION_RETRY_TIME_IN_SECONDS` | `30` | Maximum transaction retry time | GMS, MAE Consumer, System Update | +| `NEO4J_CONNECTION_LIVENESS_CHECK_TIMEOUT_IN_SECONDS` | `-1` | Connection liveness check timeout | GMS, MAE Consumer, System Update | + +## Kafka Configuration + +Reference Links: + +- **Kafka Configuration**: [Kafka Configuration Guide](../how/kafka-config.md) +- **Confluent Cloud**: [Confluent Cloud Integration](confluent-cloud.md) +- **DataHub Actions**: [Actions Documentation](../actions/README.md) + +### Topic Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------- | ---------------------- | ------------------------------ | -------------------------------------------------- | +| `DATAHUB_USAGE_EVENT_NAME` | `DataHubUsageEvent_v1` | DataHub usage event topic name | GMS, MAE Consumer, MCE Consumer, Actions, Frontend | + +### Bootstrap Servers + +| Environment Variable | Default | Description | Components | +| ------------------------ | ----------------------- | ----------------------- | --------------------------------------------------------------- | +| `KAFKA_BOOTSTRAP_SERVER` | `http://localhost:9092` | Kafka bootstrap servers | GMS, MAE Consumer, MCE Consumer, PE Consumer, Actions, Frontend | + +### Producer Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------- | --------- | ------------------------------ | -------------------------------- | +| `KAFKA_PRODUCER_RETRY_COUNT` | `3` | Producer retry count | GMS, MCE Consumer, System Update | +| `KAFKA_PRODUCER_DELIVERY_TIMEOUT` | `30000` | Producer delivery timeout | GMS, MCE Consumer, System Update | +| `KAFKA_PRODUCER_REQUEST_TIMEOUT` | `3000` | Producer request timeout | GMS, MCE Consumer, System Update | +| `KAFKA_PRODUCER_BACKOFF_TIMEOUT` | `500` | Producer backoff timeout | GMS, MCE Consumer, System Update | +| `KAFKA_PRODUCER_COMPRESSION_TYPE` | `snappy` | Producer compression algorithm | GMS, MCE Consumer, System Update | +| `KAFKA_PRODUCER_MAX_REQUEST_SIZE` | `5242880` | Maximum bytes sent by producer | GMS, MCE Consumer, System Update | + +### Consumer Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------- | --------------------------------- | ------------------------------------------ | --------------------------------------------------------- | +| `KAFKA_LISTENER_CONCURRENCY` | `1` | Number of Kafka consumer threads | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_CONSUMER_MAX_PARTITION_FETCH_BYTES` | `5242880` | Maximum data per partition | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_CONSUMER_STOP_ON_DESERIALIZATION_ERROR` | `true` | Stop on deserialization error | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_CONSUMER_HEALTH_CHECK_ENABLED` | `true` | Enable health check for consumers | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_CONSUMER_MCP_AUTO_OFFSET_RESET` | `earliest` | MCP consumer auto offset reset | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_CONSUMER_MCL_AUTO_OFFSET_RESET` | `earliest` | MCL consumer auto offset reset | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_CONSUMER_MCL_FINE_GRAINED_LOGGING_ENABLED` | `false` | Enable fine-grained logging for MCL | GMS, MAE Consumer | +| `KAFKA_CONSUMER_MCL_ASPECTS_TO_DROP` | `` | Aspects to drop for MCL | GMS, MAE Consumer | +| `KAFKA_CONSUMER_PE_AUTO_OFFSET_RESET` | `latest` | PE consumer auto offset reset | GMS, PE Consumer | +| `KAFKA_CONSUMER_PERCENTILES` | `0.5,0.95,0.99,0.999` | Consumer percentiles | GMS, MAE Consumer, MCE Consumer, PE Consumer, PE Consumer | +| `KAFKA_CONSUMER_SERVICE_LEVEL_OBJECTIVES` | `300,1800,3000,10800,21600,43200` | Consumer SLOs in seconds | GMS, MAE Consumer, MCE Consumer, PE Consumer, PE Consumer | +| `KAFKA_CONSUMER_MAX_EXPECTED_VALUE` | `86000` | Maximum expected consumer value in seconds | GMS, MAE Consumer, MCE Consumer, PE Consumer, PE Consumer | + +### Consumer Pool Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------- | ------- | ----------------------------------------------------------- | ---------- | +| `KAFKA_CONSUMER_POOL_INITIAL_SIZE` | `1` | Consumer pool initial size | GMS | +| `KAFKA_CONSUMER_POOL_MAX_SIZE` | `5` | Consumer pool maximum size | GMS | +| `KAFKA_CONSUMER_POOL_VALIDATION_TIMEOUT_SECONDS` | `5` | Timeout in seconds for validating consumer health | GMS | +| `KAFKA_CONSUMER_POOL_VALIDATION_CACHE_INTERVAL_MINUTES` | `5` | Interval in minutes for caching consumer validation results | GMS | + +### Schema Registry Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------ | ----------------------- | --------------------------------------------------- | ----------------------------------------------------- | +| `SCHEMA_REGISTRY_TYPE` | `KAFKA` | Schema registry type (INTERNAL, KAFKA, or AWS_GLUE) | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_SCHEMAREGISTRY_URL` | `http://localhost:8081` | Schema registry URL | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `SCHEMA_REGISTRY_URL` | `http://localhost:8081` | Schema registry URL (Actions) | Actions | +| `AWS_GLUE_SCHEMA_REGISTRY_REGION` | `us-east-1` | AWS Glue schema registry region | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `AWS_GLUE_SCHEMA_REGISTRY_NAME` | `null` | AWS Glue schema registry name | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `KAFKA_PROPERTIES_SECURITY_PROTOCOL` | `PLAINTEXT` | Kafka security protocol | GMS, MAE Consumer, MCE Consumer, PE Consumer, Actions | + +## Spring Configuration + +### Kafka Security + +| Environment Variable | Default | Description | Components | +| -------------------------------- | ----------- | ----------------------- | -------------------------------------------- | +| `spring.kafka.security.protocol` | `PLAINTEXT` | Kafka security protocol | GMS, MAE Consumer, MCE Consumer, PE Consumer | + +## Management & Monitoring + +### JMX Configuration + +| Environment Variable | Default | Description | Components | +| -------------------- | ------- | ----------- | -------------------------------------------- | +| `spring.jmx.enabled` | `true` | Enable JMX | GMS, MAE Consumer, MCE Consumer, PE Consumer | + +### Endpoints Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------- | ------------------------------------- | --------------------- | ---------- | +| `management.endpoints.web.exposure.include` | `prometheus,info,healthcheck,metrics` | Exposed web endpoints | GMS | +| `management.endpoints.jmx.enabled` | `true` | Enable JMX endpoints | GMS | + +### Metrics Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------------------------------- | ------- | -------------------------------- | -------------------------------------------- | +| `management.metrics.cache.enabled` | `false` | Enable cache metrics | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `management.metrics.export.jmx.enabled` | `true` | Enable JMX metrics export | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `management.metrics.export.prometheus.enabled` | `true` | Enable Prometheus metrics export | GMS, MAE Consumer, MCE Consumer, PE Consumer | + +### Server Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------- | ------- | ------------- | ---------- | +| `server.server-header` | `false` | Server header | GMS | + +## Feature Flags + +Reference Links: + +- **Access Management**: [Access Management Feature](../features/feature-guides/access-roles.md) +- **Structured Properties**: [Structured Properties Overview](../features/feature-guides/properties/overview.md) +- **Lineage Features**: [Data Lineage](../features/feature-guides/lineage.md), [UI Lineage Management](../features/feature-guides/ui-lineage.md) +- **Compliance Forms**: [Compliance Forms Overview](../features/feature-guides/compliance-forms/overview.md) +- **Dataset Usage**: [Dataset Usage & Query History](../features/dataset-usage-and-query-history.md) +- **MCP Server**: [DataHub MCP Server](../features/feature-guides/mcp.md) + +| Environment Variable | Default | Description | Components | +| --------------------------------------- | ------- | ------------------------------------------------------------------ | ---------- | +| `SHOW_SIMPLIFIED_HOMEPAGE_BY_DEFAULT` | `false` | Show simplified homepage with just datasets, charts and dashboards | GMS | +| `LINEAGE_SEARCH_CACHE_ENABLED` | `true` | Enable in-memory cache for searchAcrossLineage query | GMS | +| `GRAPH_SERVICE_DIFF_MODE_ENABLED` | `true` | Enable diff mode for graph writes | GMS | +| `POINT_IN_TIME_CREATION_ENABLED` | `false` | Enable creation of point in time snapshots for scroll API | GMS | +| `ALWAYS_EMIT_CHANGE_LOG` | `false` | Always emit MCL even when no changes detected | GMS | +| `SEARCH_SERVICE_DIFF_MODE_ENABLED` | `true` | Enable diff mode for search document writes | GMS | +| `READ_ONLY_MODE_ENABLED` | `false` | Enable read only mode for instance | GMS | +| `SHOW_ACCESS_MANAGEMENT` | `false` | Show AccessManagement tab in UI | GMS | +| `SHOW_SEARCH_FILTERS_V2` | `true` | Show search filters V2 experience | GMS | +| `SHOW_BROWSE_V2` | `true` | Show browse v2 sidebar experience | GMS | +| `PLATFORM_BROWSE_V2` | `true` | Enable platform browse experience | GMS | +| `LINEAGE_GRAPH_V2` | `true` | Enable new lineage visualization | GMS | +| `PRE_PROCESS_HOOKS_UI_ENABLED` | `true` | Circumvent Kafka for UI changes | GMS | +| `PRE_PROCESS_HOOKS_UI_ENABLED` | `false` | Reprocess UI sourced events asynchronously | GMS | +| `SHOW_ACRYL_INFO` | `false` | Show CTAs around moving to DataHub Cloud | GMS | +| `ER_MODEL_RELATIONSHIP_FEATURE_ENABLED` | `false` | Enable Join Tables Feature | GMS | +| `NESTED_DOMAINS_ENABLED` | `true` | Enable nested Domains feature | GMS | +| `SCHEMA_FIELD_ENTITY_FETCH_ENABLED` | `true` | Enable fetching schema field entities | GMS | +| `BUSINESS_ATTRIBUTE_ENTITY_ENABLED` | `false` | Enable business attribute entity | GMS | +| `DATA_CONTRACTS_ENABLED` | `true` | Enable Data Contracts feature | GMS | +| `ALTERNATE_MCP_VALIDATION` | `false` | Enable alternate MCP validation flow | GMS | +| `THEME_V2_ENABLED` | `true` | Allow theme v2 to be turned on | GMS | +| `THEME_V2_DEFAULT` | `true` | Set default theme for users | GMS | +| `THEME_V2_TOGGLEABLE` | `false` | Allow theme v2 to be toggled (Acryl only) | GMS | +| `SCHEMA_FIELD_CLL_ENABLED` | `false` | Enable schema field-level lineage links | GMS | +| `SCHEMA_FIELD_LINEAGE_IGNORE_STATUS` | `true` | Ignore schema field status in lineage | GMS | +| `SHOW_SEPARATE_SIBLINGS` | `false` | Separate siblings with no combined view | GMS | +| `EDITABLE_DATASET_NAME_ENABLED` | `false` | Enable editing dataset name in UI | GMS | +| `SHOW_MANAGE_STRUCTURED_PROPERTIES` | `true` | Show manage structured properties button | GMS | +| `HIDE_DBT_SOURCE_IN_LINEAGE` | `false` | Hide dbt sources in lineage | GMS | +| `SHOW_NAV_BAR_REDESIGN` | `true` | Show newly designed nav bar | GMS | +| `SHOW_AUTO_COMPLETE_RESULTS` | `true` | Show auto complete results in search bar | GMS | +| `ENTITY_VERSIONING_ENABLED` | `false` | Enable entity versioning APIs | GMS | +| `SHOW_HAS_SIBLINGS_FILTER` | `false` | Show "has siblings" filter in search | GMS | +| `SHOW_SEARCH_BAR_AUTOCOMPLETE_REDESIGN` | `false` | Show redesigned search bar autocomplete | GMS | +| `SHOW_MANAGE_TAGS` | `true` | Allow users to manage tags in UI | GMS | +| `SHOW_INTRODUCE_PAGE` | `true` | Show introduce page in V2 UI | GMS | +| `SHOW_INGESTION_PAGE_REDESIGN` | `true` | Show re-designed Ingestion page | GMS | +| `SHOW_LINEAGE_EXPAND_MORE` | `true` | Show expand more button in lineage graph | GMS | +| `SHOW_HOME_PAGE_REDESIGN` | `true` | Show re-designed home page | GMS | +| `LINEAGE_GRAPH_V3` | `true` | Enable redesign of lineage v2 graph | GMS | +| `SHOW_PRODUCT_UPDATES` | `true` | Show in-product update popover | GMS | +| `LOGICAL_MODELS_ENABLED` | `false` | Enable logical models feature | GMS | +| `SHOW_HOMEPAGE_USER_ROLE` | `false` | Display homepage user role underneath name | GMS | +| `VIEWS_ENABLED` | `true` | Enable views feature | GMS | + +## System Updates + +Reference Links: + +- **Updating DataHub**: [Updating DataHub Guide](../how/updating-datahub.md) + +### Bootstrap Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------------- | ------------------------------ | --------------------------------------------- | ---------- | +| `BOOTSTRAP_POLICIES_FILE` | `classpath:boot/policies.json` | Bootstrap policies file | GMS | +| `BOOTSTRAP_SERVLETS_WAITTIMEOUT` | `60` | Total waiting time for servlets to initialize | GMS | + +### System Update Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------- | --------------------- | ------------------------------------ | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_INITIAL_BACK_OFF_MILLIS` | `5000` | Initial back off for system updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_MAX_BACK_OFFS` | `50` | Maximum back offs for system updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_BACK_OFF_FACTOR` | `2` | Multiplicative factor for back off | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_WAIT_FOR_SYSTEM_UPDATE` | `true` | Wait for system update to complete | System Update | +| `SYSTEM_UPDATE_BOOTSTRAP_MCP_CONFIG` | `bootstrap_mcps.yaml` | Bootstrap MCP configuration | System Update | + +### Data Job Node CLL Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------ | ------- | --------------------------------------- | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_ENABLED` | `false` | Enable data job node CLL | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_BATCH_SIZE` | `1000` | Data job node CLL batch size | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_DELAY_MS` | `30000` | Data job node CLL delay in milliseconds | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DATA_JOB_NODE_CLL_LIMIT` | `0` | Data job node CLL limit | System Update | + +### Domain Description Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------- | ------- | ---------------------------------------- | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_ENABLED` | `true` | Enable domain description updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_BATCH_SIZE` | `1000` | Domain description batch size | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_DELAY_MS` | `30000` | Domain description delay in milliseconds | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DOMAIN_DESCRIPTION_CLL_LIMIT` | `0` | Domain description CLL limit | System Update | + +### Dashboard Info Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------------- | ------- | ------------------------------------ | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_ENABLED` | `true` | Enable dashboard info updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_BATCH_SIZE` | `1000` | Dashboard info batch size | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_DELAY_MS` | `30000` | Dashboard info delay in milliseconds | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_DASHBOARD_INFO_CLL_LIMIT` | `0` | Dashboard info CLL limit | System Update | + +### Browse Paths V2 Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------------------------------------- | ------- | --------------------------------- | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_ENABLED` | `true` | Enable browse paths V2 updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_BROWSE_PATHS_V2_BATCH_SIZE` | `5000` | Browse paths V2 batch size | System Update | +| `REPROCESS_DEFAULT_BROWSE_PATHS_V2` | `false` | Reprocess default browse paths V2 | System Update | + +### Ingestion Indices Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------ | ------- | --------------------------------------- | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_ENABLED` | `true` | Enable ingestion indices updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_BATCH_SIZE` | `5000` | Ingestion indices batch size | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_DELAY_MS` | `1000` | Ingestion indices delay in milliseconds | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_INGESTION_INDICES_CLL_LIMIT` | `0` | Ingestion indices CLL limit | System Update | + +### Policy Fields Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------------------------------- | ------- | ------------------------------- | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_POLICY_FIELDS_ENABLED` | `true` | Enable policy fields updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_POLICY_FIELDS_BATCH_SIZE` | `5000` | Policy fields batch size | System Update | +| `REPROCESS_DEFAULT_POLICY_FIELDS` | `false` | Reprocess default policy fields | System Update | + +### Ownership Types Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------------------------------------- | ------- | ------------------------------ | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_OWNERSHIP_TYPES_ENABLED` | `true` | Enable ownership types updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_OWNERSHIP_TYPES_BATCH_SIZE` | `1000` | Ownership types batch size | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_OWNERSHIP_TYPES_REPROCESS` | `false` | Reprocess ownership types | System Update | + +### Schema Fields Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------------------- | ------- | --------------------------------------------- | ------------- | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_ENABLED` | `false` | Enable schema fields from schema metadata | System Update | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_BATCH_SIZE` | `500` | Schema fields from schema metadata batch size | System Update | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_DELAY_MS` | `1000` | Schema fields from schema metadata delay | System Update | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA_LIMIT` | `0` | Schema fields from schema metadata limit | System Update | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_ENABLED` | `false` | Enable schema fields doc IDs | System Update | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_BATCH_SIZE` | `500` | Schema fields doc IDs batch size | System Update | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_DELAY_MS` | `5000` | Schema fields doc IDs delay | System Update | +| `SYSTEM_UPDATE_SCHEMA_FIELDS_DOC_IDS_LIMIT` | `0` | Schema fields doc IDs limit | System Update | + +### Process Instance Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------------------- | ------- | ------------------------------------------- | ------------- | +| `SYSTEM_UPDATE_PROCESS_INSTANCE_HAS_RUN_EVENTS_ENABLED` | `true` | Enable process instance has run events | System Update | +| `SYSTEM_UPDATE_PROCESS_INSTANCE_HAS_RUN_EVENTS_BATCH_SIZE` | `100` | Process instance has run events batch size | System Update | +| `SYSTEM_UPDATE_PROCESS_INSTANCE_HAS_RUN_EVENTS_DELAY_MS` | `1000` | Process instance has run events delay | System Update | +| `SYSTEM_UPDATE_PROCESS_INSTANCE_HAS_RUN_EVENTS_TOTAL_DAYS` | `90` | Process instance has run events total days | System Update | +| `SYSTEM_UPDATE_PROCESS_INSTANCE_HAS_RUN_EVENTS_WINDOW_DAYS` | `1` | Process instance has run events window days | System Update | +| `SYSTEM_UPDATE_PROCESS_INSTANCE_HAS_RUN_EVENTS_REPROCESS` | `false` | Reprocess process instance has run events | System Update | + +### Edge Status Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------ | ------- | --------------------------------- | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_EDGE_STATUS_ENABLED` | `false` | Enable edge status updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_EDGE_STATUS_BATCH_SIZE` | `1000` | Edge status batch size | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_EDGE_STATUS_DELAY_MS` | `5000` | Edge status delay in milliseconds | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_EDGE_STATUS_LIMIT` | `0` | Edge status limit | System Update | + +### Property Definitions Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------------------- | ------- | ------------------------------------------ | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_ENABLED` | `true` | Enable property definitions updates | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_BATCH_SIZE` | `500` | Property definitions batch size | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_DELAY_MS` | `1000` | Property definitions delay in milliseconds | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_PROPERTY_DEFINITIONS_CLL_LIMIT` | `0` | Property definitions CLL limit | System Update | + +### Remove Query Edges Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------------------------------------- | ------- | -------------------------- | ------------- | +| `BOOTSTRAP_SYSTEM_UPDATE_REMOVE_QUERY_EDGES_ENABLED` | `true` | Enable remove query edges | System Update | +| `BOOTSTRAP_SYSTEM_UPDATE_REMOVE_QUERY_EDGES_RETRIES` | `20` | Remove query edges retries | System Update | + +## Additional Environment Variables + +The following environment variables are used in the codebase but may not be explicitly defined in the application.yaml file: + +### Ingestion and Processing + +| Environment Variable | Default | Description | Components | +| ----------------------------------- | ------- | ---------------------------------------------------------- | ---------- | +| `ASYNC_INGEST_DEFAULT` | `false` | Asynchronously process ingestProposals by writing to Kafka | GMS | +| `STRICT_URN_VALIDATION_ENABLED` | `false` | Enable stricter URN validation logic | GMS | +| `DATAHUB_DATASET_URN_TO_LOWER` | `null` | Convert dataset URN names to lowercase | GMS | +| `BUSINESS_ATTRIBUTE_ENTITY_ENABLED` | `false` | Enable business attribute entity feature | GMS | + +### REST and Servlet Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------ | ------- | ---------------------------------- | ----------------- | +| `RESTLI_SERVLET_THREADS` | `null` | Number of threads for REST servlet | GMS, MCE Consumer | +| `RESTLI_TIMEOUT_SECONDS` | `60` | REST timeout in seconds | GMS, MCE Consumer | + +### System and Version Information + +| Environment Variable | Default | Description | Components | +| ---------------------- | ------- | ------------------------- | ------------- | +| `DATAHUB_GMS_PROTOCOL` | `http` | GMS protocol (http/https) | GMS | +| `DATAHUB_REVISION` | `0` | DataHub revision version | System Update | + +### Upgrade and Migration + +| Environment Variable | Default | Description | Components | +| -------------------------------------------------- | ------- | -------------------------------------------------- | ------------- | +| `SKIP_REINDEX_EDGE_STATUS` | `false` | Skip reindexing edge status | System Update | +| `SKIP_REINDEX_DATA_JOB_INPUT_OUTPUT` | `false` | Skip reindexing data job input/output | System Update | +| `SKIP_GENERATE_SCHEMA_FIELDS_FROM_SCHEMA_METADATA` | `false` | Skip generating schema fields from schema metadata | System Update | +| `SKIP_MIGRATE_SCHEMA_FIELDS_DOC_ID` | `false` | Skip migrating schema fields doc IDs | System Update | +| `SKIP_CREATE_USAGE_EVENT_INDICES_STEP` | `false` | Skip creating usage event indices/data streams | System Update | +| `BACKFILL_BROWSE_PATHS_V2` | `false` | Enable backfilling browse paths V2 | System Update | +| `READER_POOL_SIZE` | `null` | Reader pool size for restore operations | System Update | +| `WRITER_POOL_SIZE` | `null` | Writer pool size for restore operations | System Update | + +### OpenTelemetry Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------- | ------- | ------------------------------ | -------------------------------------------- | +| `OTEL_METRICS_EXPORTER` | `none` | OpenTelemetry metrics exporter | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `OTEL_TRACES_EXPORTER` | `none` | OpenTelemetry traces exporter | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `OTEL_LOGS_EXPORTER` | `none` | OpenTelemetry logs exporter | GMS, MAE Consumer, MCE Consumer, PE Consumer | +| `OTEL_PROPAGATORS` | `null` | OpenTelemetry propagators | GMS, MAE Consumer, MCE Consumer, PE Consumer | + +### Secret Service Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------- | ---------------- | -------------------------------------- | ---------- | +| `SECRET_SERVICE_ENCRYPTION_KEY` | `ENCRYPTION_KEY` | Secret service encryption key | GMS | +| `SECRET_SERVICE_V1_ALGORITHM_ENABLED` | `true` | Enable v1 algorithm for secret service | GMS | + +### Health Check Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------- | ------- | --------------------------- | ---------- | +| `HEALTH_CHECK_CACHE_DURATION_SECONDS` | `5` | Health check cache duration | GMS | + +### Metadata Tests Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------ | ------- | --------------------- | ---------- | +| `METADATA_TESTS_ENABLED` | `false` | Enable metadata tests | GMS | + +### Hooks Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------------ | ------------- | ---------------------------------------------------- | ----------------- | +| `ENABLE_SIBLING_HOOK` | `true` | Enable automatic sibling associations | GMS, MAE Consumer | +| `SIBLINGS_HOOK_CONSUMER_GROUP_SUFFIX` | `` | Siblings hook consumer group suffix | GMS, MAE Consumer | +| `ENABLE_UPDATE_INDICES_HOOK` | `true` | Enable update indices hook | GMS, MAE Consumer | +| `UPDATE_INDICES_CONSUMER_GROUP_SUFFIX` | `` | Update indices consumer group suffix | GMS, MAE Consumer | +| `ENABLE_INGESTION_SCHEDULER_HOOK` | `true` | Enable ingestion scheduling | GMS, MAE Consumer | +| `INGESTION_SCHEDULER_HOOK_CONSUMER_GROUP_SUFFIX` | `` | Ingestion scheduler hook consumer group suffix | GMS, MAE Consumer | +| `ENABLE_INCIDENTS_HOOK` | `true` | Enable incidents hook | GMS, MAE Consumer | +| `MAX_INCIDENT_HISTORY` | `100` | Maximum incident history | GMS, MAE Consumer | +| `INCIDENTS_HOOK_CONSUMER_GROUP_SUFFIX` | `` | Incidents hook consumer group suffix | GMS, MAE Consumer | +| `ENABLE_STRUCTURED_PROPERTIES_HOOK` | `true` | Enable structured properties mappings | GMS, MAE Consumer | +| `ENABLE_STRUCTURED_PROPERTIES_WRITE` | `true` | Enable writing structured property values | GMS, MAE Consumer | +| `ENABLE_STRUCTURED_PROPERTIES_SYSTEM_UPDATE` | `false` | Enable structured property mappings in system update | GMS, MAE Consumer | +| `ENABLE_ENTITY_CHANGE_EVENTS_HOOK` | `true` | Enable entity change events hook | GMS, MAE Consumer | +| `ECE_CONSUMER_GROUP_SUFFIX` | `` | Entity change events consumer group suffix | GMS, MAE Consumer | +| `ECE_ENTITY_EXCLUSIONS` | `schemaField` | Entities to exclude from ECE hook | GMS, MAE Consumer | +| `FORMS_HOOK_ENABLED` | `true` | Enable forms hook | GMS, MAE Consumer | +| `FORMS_HOOK_CONSUMER_GROUP_SUFFIX` | `` | Forms hook consumer group suffix | GMS, MAE Consumer | + +### Search and API Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------- | --------------------------- | ------------------------------ | ---------- | +| `SEARCH_BAR_API_VARIANT` | `AUTOCOMPLETE_FOR_MULTIPLE` | Search bar API variant | Frontend | +| `FIRST_IN_PERSONAL_SIDEBAR` | `YOUR_ASSETS` | First item in personal sidebar | Frontend | + +### Client Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------------- | ------- | --------------------------------------------------- | ------------------------------ | +| `ENTITY_CLIENT_RETRY_INTERVAL` | `2` | Entity client retry interval | GMS | +| `ENTITY_CLIENT_NUM_RETRIES` | `3` | Entity client number of retries | GMS | +| `ENTITY_CLIENT_JAVA_GET_BATCH_SIZE` | `375` | Entity client Java get batch size | GMS | +| `ENTITY_CLIENT_JAVA_INGEST_BATCH_SIZE` | `375` | Entity client Java ingest batch size | GMS | +| `ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE` | `100` | Entity client RESTli get batch size | GMS, MAE Consumer, PE Consumer | +| `ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY` | `2` | Entity client RESTli get batch concurrency | GMS, MAE Consumer, PE Consumer | +| `ENTITY_CLIENT_RESTLI_GET_BATCH_QUEUE_SIZE` | `500` | Entity client RESTli get batch queue size | GMS, MAE Consumer, PE Consumer | +| `ENTITY_CLIENT_RESTLI_GET_BATCH_THREAD_KEEP_ALIVE` | `60` | Entity client RESTli get batch thread keep alive | GMS, MAE Consumer, PE Consumer | +| `ENTITY_CLIENT_RESTLI_INGEST_BATCH_SIZE` | `50` | Entity client RESTli ingest batch size | GMS, MAE Consumer, PE Consumer | +| `ENTITY_CLIENT_RESTLI_INGEST_BATCH_CONCURRENCY` | `2` | Entity client RESTli ingest batch concurrency | GMS, MAE Consumer, PE Consumer | +| `ENTITY_CLIENT_RESTLI_INGEST_BATCH_QUEUE_SIZE` | `500` | Entity client RESTli ingest batch queue size | GMS, MAE Consumer, PE Consumer | +| `ENTITY_CLIENT_RESTLI_INGEST_BATCH_THREAD_KEEP_ALIVE` | `60` | Entity client RESTli ingest batch thread keep alive | GMS, MAE Consumer, PE Consumer | +| `USAGE_CLIENT_RETRY_INTERVAL` | `2` | Usage client retry interval | GMS, MAE Consumer, PE Consumer | +| `USAGE_CLIENT_NUM_RETRIES` | `0` | Usage client number of retries | GMS, MAE Consumer, PE Consumer | +| `USAGE_CLIENT_TIMEOUT_MS` | `3000` | Usage client timeout in milliseconds | GMS, MAE Consumer, PE Consumer | + +### Cache Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------------- | ----------- | -------------------------------------------------------- | ------------------------------ | +| `CACHE_TTL_SECONDS` | `600` | Default cache time to live | GMS | +| `CACHE_MAX_SIZE` | `10000` | Maximum number of items to cache | GMS | +| `CACHE_ENTITY_COUNTS_TTL_SECONDS` | `600` | Homepage entity count time to live | GMS | +| `CACHE_SEARCH_LINEAGE_TTL_SECONDS` | `86400` | Search lineage cache time to live | GMS | +| `CACHE_SEARCH_LINEAGE_LIGHTNING_THRESHOLD` | `300` | Lineage graphs exceeding this limit will use local cache | GMS | +| `CACHE_CLIENT_USAGE_CLIENT_ENABLED` | `true` | Enable usage client cache | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_USAGE_CLIENT_STATS_ENABLED` | `true` | Enable usage client cache stats | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_USAGE_CLIENT_STATS_INTERVAL_SECONDS` | `120` | Usage client cache stats interval | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_USAGE_CLIENT_TTL_SECONDS` | `86400` | Usage client cache TTL | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_USAGE_CLIENT_MAX_BYTES` | `52428800` | Usage client cache max bytes (50MB) | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_ENTITY_CLIENT_ENABLED` | `true` | Enable entity client cache | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_ENTITY_CLIENT_STATS_ENABLED` | `true` | Enable entity client cache stats | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_ENTITY_CLIENT_STATS_INTERVAL_SECONDS` | `120` | Entity client cache stats interval | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_ENTITY_CLIENT_TTL_SECONDS` | `0` | Entity client cache TTL (0 = no cache) | GMS, MAE Consumer, PE Consumer | +| `CACHE_CLIENT_ENTITY_CLIENT_MAX_BYTES` | `104857600` | Entity client cache max bytes (100MB) | GMS, MAE Consumer, PE Consumer | + +### GraphQL Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------------------------------- | ---------------------------------------------------------- | ------------------------------------------------ | ---------- | +| `GRAPHQL_CONCURRENCY_SEPARATE_THREAD_POOL` | `false` | Enable separate thread pool for GraphQL | GMS | +| `GRAPHQL_CONCURRENCY_STACK_SIZE` | `256000` | GraphQL thread pool stack size | GMS | +| `GRAPHQL_CONCURRENCY_CORE_POOL_SIZE` | `-1` | GraphQL core pool size (default 5 \* cores) | GMS | +| `GRAPHQL_CONCURRENCY_MAX_POOL_SIZE` | `-1` | GraphQL max pool size (default 100 \* cores) | GMS | +| `GRAPHQL_CONCURRENCY_KEEP_ALIVE` | `60` | GraphQL thread keep alive time | GMS | +| `GRAPHQL_QUERY_COMPLEXITY_LIMIT` | `2000` | GraphQL query complexity limit | GMS | +| `GRAPHQL_QUERY_DEPTH_LIMIT` | `50` | GraphQL query depth limit | GMS | +| `GRAPHQL_QUERY_INTROSPECTION_ENABLED` | `true` | Enable GraphQL introspection | GMS | +| `GRAPHQL_METRICS_ENABLED` | `true` | Enable GraphQL metrics collection | GMS | +| `GRAPHQL_PERCENTILES` | `0.5,0.75,0.95,0.98,0.99,0.999` | GraphQL percentiles | GMS | +| `GRAPHQL_METRICS_FIELD_LEVEL_ENABLED` | `false` | Enable field-level GraphQL metrics | GMS | +| `GRAPHQL_METRICS_FIELD_LEVEL_OPERATIONS` | `getSearchResultsForMultiple,searchAcrossLineageStructure` | GraphQL field-level operations | GMS | +| `GRAPHQL_METRICS_FIELD_LEVEL_PATH_ENABLED` | `false` | Include field path in GraphQL metrics | GMS | +| `GRAPHQL_METRICS_FIELD_LEVEL_PATHS` | `` | GraphQL field-level paths | GMS | +| `GRAPHQL_METRICS_TRIVIAL_DATA_FETCHERS_ENABLED` | `false` | Include trivial data fetchers in GraphQL metrics | GMS | + +### Chrome Extension Configuration + +| Environment Variable | Default | Description | Components | +| ---------------------------------- | ------- | ------------------------------- | ---------- | +| `CHROME_EXTENSION_ENABLED` | `true` | Enable Chrome extension | Frontend | +| `CHROME_EXTENSION_LINEAGE_ENABLED` | `true` | Enable Chrome extension lineage | Frontend | + +### Business Attribute Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------------------- | ------- | ---------------------------------------------------------------- | ---------- | +| `BUSINESS_ATTRIBUTE_RELATED_ENTITIES_COUNT` | `20000` | Business attribute related entities count | GMS | +| `BUSINESS_ATTRIBUTE_RELATED_ENTITIES_BATCH_SIZE` | `1000` | Business attribute related entities batch size | GMS | +| `BUSINESS_ATTRIBUTE_PROPAGATION_CONCURRENCY_THREAD_COUNT` | `-1` | Business attribute propagation thread count (default 2 \* cores) | GMS | +| `BUSINESS_ATTRIBUTE_PROPAGATION_CONCURRENCY_KEEP_ALIVE` | `60` | Business attribute propagation keep alive time | GMS | + +### Metadata Change Proposal Configuration + +| Environment Variable | Default | Description | Components | +| --------------------------------------------- | ---------- | ---------------------------------------------- | ----------------- | +| `MCP_CONSUMER_BATCH_ENABLED` | `false` | Enable MCP consumer batch processing | GMS, MCE Consumer | +| `MCP_CONSUMER_BATCH_SIZE` | `15744000` | MCP consumer batch size | GMS, MCE Consumer | +| `MCP_VALIDATION_IGNORE_UNKNOWN` | `true` | Ignore unknown fields in MCP validation | GMS, MCE Consumer | +| `MCP_VALIDATION_PRIVILEGE_CONSTRAINTS` | `true` | Enable privilege constraints in MCP validation | GMS, MCE Consumer | +| `MCP_VALIDATION_EXTENSIONS_ENABLED` | `false` | Enable extensions in MCP validation | GMS, MCE Consumer | +| `MCP_SIDE_EFFECTS_SCHEMA_FIELD_ENABLED` | `false` | Enable schema field side effects | GMS, MCE Consumer | +| `MCP_SIDE_EFFECTS_DATA_PRODUCT_UNSET_ENABLED` | `true` | Enable data product unset side effects | GMS, MCE Consumer | +| `MCP_THROTTLE_UPDATE_INTERVAL_MS` | `60000` | MCP throttle update interval | GMS, MCE Consumer | +| `MCP_MCE_CONSUMER_THROTTLE_ENABLED` | `false` | Enable MCE consumer throttling | GMS, MCE Consumer | +| `MCP_API_REQUESTS_THROTTLE_ENABLED` | `false` | Enable API requests throttling | GMS, MCE Consumer | +| `MCP_VERSIONED_THROTTLE_ENABLED` | `false` | Enable versioned MCL topic throttling | GMS, MCE Consumer | +| `MCP_VERSIONED_THRESHOLD` | `4000` | Versioned throttle threshold | GMS, MCE Consumer | +| `MCP_VERSIONED_MAX_ATTEMPTS` | `1000` | Versioned max attempts | GMS, MCE Consumer | +| `MCP_VERSIONED_INITIAL_INTERVAL_MS` | `100` | Versioned initial interval | GMS, MCE Consumer | +| `MCP_VERSIONED_MULTIPLIER` | `10` | Versioned multiplier | GMS, MCE Consumer | +| `MCP_VERSIONED_MAX_INTERVAL_MS` | `30000` | Versioned max interval | GMS, MCE Consumer | +| `MCP_TIMESERIES_THROTTLE_ENABLED` | `false` | Enable timeseries MCL topic throttling | GMS, MCE Consumer | +| `MCP_TIMESERIES_THRESHOLD` | `4000` | Timeseries throttle threshold | GMS, MCE Consumer | +| `MCP_TIMESERIES_MAX_ATTEMPTS` | `1000` | Timeseries max attempts | GMS, MCE Consumer | +| `MCP_TIMESERIES_INITIAL_INTERVAL_MS` | `100` | Timeseries initial interval | GMS, MCE Consumer | +| `MCP_TIMESERIES_MULTIPLIER` | `10` | Timeseries multiplier | GMS, MCE Consumer | +| `MCP_TIMESERIES_MAX_INTERVAL_MS` | `30000` | Timeseries max interval | GMS, MCE Consumer | + +### Events API Configuration + +| Environment Variable | Default | Description | Components | +| -------------------- | ------- | ----------------- | ---------- | +| `EVENTS_API_ENABLED` | `true` | Enable events API | GMS | + +### Iceberg Catalog Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------- | ------------------- | ----------------------------------------- | ---------- | +| `ENABLE_PUBLIC_READ` | `false` | Enable public read for Iceberg catalog | GMS | +| `PUBLICLY_READABLE_TAG` | `PUBLICLY_READABLE` | Publicly readable tag for Iceberg catalog | GMS | + +## Change Data Capture (CDC) Configuration + +Reference Links: + +- **MCP/MCL Documentation**: [MCP & MCL Events](../advanced/mcp-mcl.md) +- **CDC Configuration Guide**: [Configure CDC Mode](../how/configure-cdc.md) + +DataHub supports CDC mode for MetadataChangeLog generation, which guarantees ordered MCL events matching the order of database writes. CDC mode is optional and disabled by default. + +### CDC Processing (Common) + +| Environment Variable | Default | Description | Components | +| ----------------------------------- | -------------------------------- | -------------------------------------------------------------------- | -------------------------------- | +| `CDC_MCL_PROCESSING_ENABLED` | `false` | Enable CDC mode for MCL generation | GMS, MCE Consumer, System Update | +| `CDC_CONFIGURE_SOURCE` | `false` | Auto-configure Debezium connector (recommended false for production) | System Update | +| `CDC_DB_TYPE` | `mysql` | Database type for CDC (mysql or postgres) | System Update | +| `DATAHUB_CDC_CONNECTOR_NAME` | `datahub-cdc-connector` | Name of the Debezium connector | System Update | +| `CDC_KAFKA_CONNECT_URL` | `http://kafka-connect:8083` | Kafka Connect REST API URL | System Update | +| `CDC_KAFKA_CONNECT_REQUEST_TIMEOUT` | `10000` | Request timeout for Kafka Connect API calls in milliseconds | System Update | +| `CDC_USER` | `datahub_cdc` | Database username for CDC connector | System Update | +| `CDC_PASSWORD` | `datahub_cdc` | Database password for CDC connector | System Update | +| `CDC_TOPIC_NAME` | `datahub.metadata_aspect_v2` | Kafka topic name for CDC events | GMS, MCE Consumer, System Update | +| `CDC_URN_KEY_SPEC` | `datahub.metadata_aspect_v2:urn` | Partitioning key specification (table:column format) | System Update | + +### CDC MySQL Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------- | -------------------------------------------- | ---------------------------------------- | ------------- | +| `DEBEZIUM_CONNECTOR_CLASS` | `io.debezium.connector.mysql.MySqlConnector` | Debezium connector class for MySQL | System Update | +| `DEBEZIUM_PLUGIN_NAME` | `decoderbufs` | Logical decoding plugin for MySQL | System Update | +| `CDC_SERVER_ID` | `184001` | Unique server ID for MySQL CDC connector | System Update | + +### CDC PostgreSQL Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------- | ---------------------------------------------------- | --------------------------------------- | ------------- | +| `DEBEZIUM_CONNECTOR_CLASS` | `io.debezium.connector.postgresql.PostgresConnector` | Debezium connector class for PostgreSQL | System Update | +| `DEBEZIUM_PLUGIN_NAME` | `pgoutput` | PostgreSQL logical decoding plugin | System Update | +| `CDC_INCLUDE_TABLE` | `public.metadata_aspect_v2` | Tables to include in CDC capture | System Update | +| `CDC_INCLUDE_SCHEMA` | `public` | Schemas to include in CDC capture | System Update | + +## Component Configuration + +| Variable | Default | Description | Components | +| ---------------------- | ------- | ----------------------------------------------------------------------------------------- | ----------------- | +| `MCP_CONSUMER_ENABLED` | `true` | When running in standalone mode, disabled on `GMS` and enable on separate `MCE Consumer`. | GMS, MCE Consumer | +| `MCL_CONSUMER_ENABLED` | `true` | When running in standalone mode, disabled on `GMS` and enable on separate `MAE Consumer`. | GMS, MAE Consumer | +| `PE_CONSUMER_ENABLED` | `true` | When running in standalone mode, disabled on `GMS` and enable on separate `MAE Consumer`. | GMS, PE Consumer | + +--- + +# DataHub Frontend + +## Play Framework Configuration + +### Secret Key Configuration + +| Environment Variable | Default | Description | Components | +| -------------------- | ------- | ------------------------------------------------- | ---------- | +| `DATAHUB_SECRET` | `null` | Secret key used to secure cryptographic functions | Frontend | + +### HTTP Parser Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------ | ------- | ------------------------------------------ | ---------- | +| `DATAHUB_PLAY_MEM_BUFFER_SIZE` | `10MB` | Maximum memory buffer size for HTTP parser | Frontend | + +### Server Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------------------- | ------- | --------------------------------- | ---------- | +| `DATAHUB_AKKA_MAX_HEADER_COUNT` | `64` | Maximum number of headers allowed | Frontend | +| `DATAHUB_AKKA_MAX_HEADER_VALUE_LENGTH` | `32k` | Maximum header value length | Frontend | + +### Session Configuration + +| Environment Variable | Default | Description | Components | +| ----------------------- | ------- | ----------------------------------------------- | ---------- | +| `AUTH_COOKIE_SAME_SITE` | `LAX` | SameSite attribute for authentication cookies | Frontend | +| `AUTH_COOKIE_SECURE` | `false` | Whether authentication cookies should be secure | Frontend | + +## Authentication Configuration + +### OIDC Configuration + +Reference Links: + +- **OIDC Setup Guide**: [Configure OIDC Authentication](../authentication/guides/sso/configure-oidc-react.md) +- **OIDC Prerequisites**: [Initialize OIDC](../authentication/guides/sso/initialize-oidc.md) + +#### Required OIDC Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------- | ------- | ---------------------------------------------------- | ---------- | +| `AUTH_OIDC_ENABLED` | `false` | Enable OIDC authentication | Frontend | +| `AUTH_OIDC_CLIENT_ID` | `null` | Unique client ID issued by the identity provider | Frontend | +| `AUTH_OIDC_CLIENT_SECRET` | `null` | Unique client secret issued by the identity provider | Frontend | +| `AUTH_OIDC_DISCOVERY_URI` | `null` | The IdP OIDC discovery URL | Frontend | +| `AUTH_OIDC_BASE_URL` | `null` | The base URL associated with your DataHub deployment | Frontend | + +#### Optional OIDC Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------------------- | --------------------- | ------------------------------------------------------------------------ | ---------- | +| `AUTH_OIDC_USER_NAME_CLAIM` | `preferred_username` | The attribute/claim used to derive the DataHub username | Frontend | +| `AUTH_OIDC_USER_NAME_CLAIM_REGEX` | `(.*)` | The regex used to parse the DataHub username from the user name claim | Frontend | +| `AUTH_OIDC_SCOPE` | `oidc email profile` | String representing the requested scope from the IdP | Frontend | +| `AUTH_OIDC_CLIENT_AUTHENTICATION_METHOD` | `client_secret_basic` | Authentication method to pass credentials to token endpoint | Frontend | +| `AUTH_OIDC_JIT_PROVISIONING_ENABLED` | `true` | Whether DataHub users should be provisioned on login if they don't exist | Frontend | +| `AUTH_OIDC_PRE_PROVISIONING_REQUIRED` | `false` | Whether the user should already exist in DataHub on login | Frontend | +| `AUTH_OIDC_EXTRACT_GROUPS_ENABLED` | `true` | Whether groups should be extracted from a claim in the OIDC profile | Frontend | +| `AUTH_OIDC_GROUPS_CLAIM` | `groups` | The OIDC claim to extract groups information from | Frontend | +| `AUTH_OIDC_RESPONSE_TYPE` | `null` | OIDC response type | Frontend | +| `AUTH_OIDC_RESPONSE_MODE` | `null` | OIDC response mode | Frontend | +| `AUTH_OIDC_USE_NONCE` | `null` | Whether to use nonce in OIDC flow | Frontend | +| `AUTH_OIDC_CUSTOM_PARAM_RESOURCE` | `null` | Custom resource parameter for OIDC | Frontend | +| `AUTH_OIDC_READ_TIMEOUT` | `null` | OIDC read timeout | Frontend | +| `AUTH_OIDC_CONNECT_TIMEOUT` | `null` | OIDC connect timeout | Frontend | +| `AUTH_OIDC_EXTRACT_JWT_ACCESS_TOKEN_CLAIMS` | `false` | Whether to extract claims from JWT access token | Frontend | +| `AUTH_OIDC_PREFERRED_JWS_ALGORITHM` | `null` | Which JWS algorithm to use | Frontend | +| `AUTH_OIDC_ACR_VALUES` | `null` | OIDC ACR values | Frontend | +| `AUTH_OIDC_GRANT_TYPE` | `null` | OIDC grant type | Frontend | + +### Authentication Methods Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------ | ------- | -------------------------------------------------------- | ---------- | +| `AUTH_JAAS_ENABLED` | `true` | Enable JAAS authentication | Frontend | +| `AUTH_NATIVE_ENABLED` | `true` | Enable native authentication | Frontend | +| `GUEST_AUTHENTICATION_ENABLED` | `false` | Enable guest authentication | Frontend | +| `GUEST_AUTHENTICATION_USER` | `guest` | The name of the guest user ID | Frontend | +| `GUEST_AUTHENTICATION_PATH` | `null` | The path to bypass login page and get logged in as guest | Frontend | +| `ENFORCE_VALID_EMAIL` | `true` | Enforce the usage of a valid email for user sign up | Frontend | + +### Authentication Logging + +| Environment Variable | Default | Description | Components | +| ---------------------- | ------- | ------------------------------------- | ---------- | +| `AUTH_VERBOSE_LOGGING` | `false` | Enable verbose authentication logging | Frontend | + +### Session Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------ | ------- | -------------------------------------- | ---------- | +| `AUTH_SESSION_TTL_HOURS` | `24` | Login session expiration time in hours | Frontend | +| `MAX_SESSION_TOKEN_AGE` | `24h` | Maximum age of session token | Frontend | + +## Metadata Service Configuration + +### Connection Configuration + +| Environment Variable | Default | Description | Components | +| --------------------- | ----------- | -------------------------------------------------- | ---------- | +| `DATAHUB_GMS_HOST` | `localhost` | Metadata service host | Frontend | +| `DATAHUB_GMS_PORT` | `8080` | Metadata service port | Frontend | +| `DATAHUB_GMS_USE_SSL` | `false` | Whether to use SSL for metadata service connection | Frontend | + +### Authentication Configuration + +| Environment Variable | Default | Description | Components | +| ------------------------------- | ---------------------- | ----------------------------------------- | ---------- | +| `METADATA_SERVICE_AUTH_ENABLED` | `false` | Enable metadata service authentication | Frontend | +| `DATAHUB_SYSTEM_CLIENT_SECRET` | `JohnSnowKnowsNothing` | System client secret for metadata service | Frontend | + +## Entity Client Configuration + +| Environment Variable | Default | Description | Components | +| -------------------------------------------- | ------- | ------------------------------------------ | ---------- | +| `ENTITY_CLIENT_RETRY_INTERVAL` | `2` | Entity client retry interval | Frontend | +| `ENTITY_CLIENT_NUM_RETRIES` | `3` | Entity client number of retries | Frontend | +| `ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE` | `50` | Entity client RESTli get batch size | Frontend | +| `ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY` | `2` | Entity client RESTli get batch concurrency | Frontend | + +--- + +## Notes + +- Environment variables follow the pattern of converting YAML property paths to uppercase with underscores +- Default values are shown in the table above +- For Kafka configuration, refer to the official Spring Kafka documentation for additional properties +- Feature flags control experimental or optional functionality +- System update configurations control various background maintenance tasks +- Cache configurations help optimize performance for different use cases +- GraphQL configurations control query complexity and performance monitoring +- OpenTelemetry variables control observability and tracing behavior +- Play Framework properties are converted to environment variables by: + - Converting dots (`.`) to underscores (`_`) + - Converting to uppercase diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/gcp.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/gcp.md new file mode 100644 index 00000000..6d2ea721 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/gcp.md @@ -0,0 +1,115 @@ +--- +title: Deploying to GCP +sidebar_label: Deploying to GCP +slug: /deploy/gcp +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/deploy/gcp.md' +--- + +# GCP setup guide + +The following is a set of instructions to quickstart DataHub on GCP Google Kubernetes Engine (GKE). Note, the guide +assumes that you do not have a kubernetes cluster set up. If you are deploying DataHub to an existing cluster, please +skip the corresponding sections. + +## Prerequisites + +This guide requires the following tools: + +- [kubectl](https://kubernetes.io/docs/tasks/tools/) to manage kubernetes resources +- [helm](https://helm.sh/docs/intro/install/) to deploy the resources based on helm charts. Note, we only support Helm 3. +- [gcloud](https://cloud.google.com/sdk/docs/install) to manage GCP resources + +Follow the +following [guide](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster#before_you_begin) to +correctly set up Google Cloud SDK. + +After setting up, run `gcloud services enable container.googleapis.com` to make sure GKE service is enabled. + +## Start up a kubernetes cluster on GKE + +Let’s follow this [guide](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-zonal-cluster) to create a +new cluster using gcloud. Run the following command with cluster-name set to the cluster name of choice, and zone set to +the GCP zone you are operating on. + +``` +gcloud container clusters create <> \ + --zone <> \ + -m e2-standard-2 +``` + +The command will provision a GKE cluster powered by 3 e2-standard-2 (2 CPU, 8GB RAM) nodes. + +If you are planning to run the storage layer (MySQL, Elasticsearch, Kafka) as pods in the cluster, you need at least 3 +nodes with the above specs. If you decide to use managed storage services, you can reduce the number of nodes or use +m3.medium nodes to save cost. Refer to +this [guide](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-regional-cluster) for creating a regional +cluster for better robustness. + +Run `kubectl get nodes` to confirm that the cluster has been setup correctly. You should get results like below + +``` +NAME STATUS ROLES AGE VERSION +gke-datahub-default-pool-e5be7c4f-8s97 Ready 34h v1.19.10-gke.1600 +gke-datahub-default-pool-e5be7c4f-d68l Ready 34h v1.19.10-gke.1600 +gke-datahub-default-pool-e5be7c4f-rksj Ready 34h v1.19.10-gke.1600 +``` + +## Setup DataHub using Helm + +Once the kubernetes cluster has been set up, you can deploy DataHub and it’s prerequisites using helm. Please follow the +steps in this [guide](kubernetes.md) + +## Expose endpoints using GKE ingress controller + +Now that all the pods are up and running, you need to expose the datahub-frontend end point by setting +up [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/). Easiest way to set up ingress is to use +the GKE page on [GCP website](https://console.cloud.google.com/kubernetes/discovery). + +Once all deploy is successful, you should see a page like below in the "Services & Ingress" tab on the left. + +

+ +

+ +Tick the checkbox for datahub-datahub-frontend and click "CREATE INGRESS" button. You should land on the following page. + +

+ +

+ +Type in an arbitrary name for the ingress and click on the second step "Host and path rules". You should land on the +following page. + +

+ +

+ +Select "datahub-datahub-frontend" in the dropdown menu for backends, and then click on "ADD HOST AND PATH RULE" button. +In the second row that got created, add in the host name of choice (here gcp.datahubproject.io) and select +"datahub-datahub-frontend" in the backends dropdown. + +This step adds the rule allowing requests from the host name of choice to get routed to datahub-frontend service. Click +on step 3 "Frontend configuration". You should land on the following page. + +

+ +

+ +Choose HTTPS in the dropdown menu for protocol. To enable SSL, you need to add a certificate. If you do not have one, +you can click "CREATE A NEW CERTIFICATE" and input the host name of choice. GCP will create a certificate for you. + +Now press "CREATE" button on the left to create ingress! After around 5 minutes, you should see the following. + +

+ +

+ +In your domain provider, add an A record for the host name set above using the IP address on the ingress page (noted +with the red box). Once DNS updates, you should be able to access DataHub through the host name!! + +Note, ignore the warning icon next to ingress. It takes about ten minutes for ingress to check that the backend service +is ready and show a check mark as follows. However, ingress is fully functional once you see the above page. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/kubernetes.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/kubernetes.md new file mode 100644 index 00000000..73bb6c3d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/kubernetes.md @@ -0,0 +1,164 @@ +--- +title: Deploying with Kubernetes +sidebar_label: Deploying with Kubernetes +slug: /deploy/kubernetes +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/deploy/kubernetes.md +--- + +# Deploying DataHub with Kubernetes + +## Introduction + +Helm charts for deploying DataHub on a kubernetes cluster is located in +this [repository](https://github.com/acryldata/datahub-helm). We provide charts for +deploying [DataHub](https://github.com/acryldata/datahub-helm/tree/master/charts/datahub) and +its [dependencies](https://github.com/acryldata/datahub-helm/tree/master/charts/prerequisites) +(Elasticsearch, optionally Neo4j, MySQL, and Kafka) on a Kubernetes cluster. + +This doc is a guide to deploy an instance of DataHub on a kubernetes cluster using the above charts from scratch. + +## Setup + +1. Set up a kubernetes cluster + - In a cloud platform of choice like [Amazon EKS](https://aws.amazon.com/eks), + [Google Kubernetes Engine](https://cloud.google.com/kubernetes-engine), + and [Azure Kubernetes Service](https://azure.microsoft.com/en-us/services/kubernetes-service/) OR + - In local environment using [Minikube](https://minikube.sigs.k8s.io/docs/). Note, more than 7GB of RAM is required + to run DataHub and its dependencies +2. Install the following tools: + - [kubectl](https://kubernetes.io/docs/tasks/tools/) to manage kubernetes resources + - [helm](https://helm.sh/docs/intro/install/) to deploy the resources based on helm charts. Note, we only support + Helm 3. + +## Components + +DataHub consists of 4 main components: [GMS](/docs/metadata-service), +[MAE Consumer](/docs/metadata-jobs/mae-consumer-job) (optional), +[MCE Consumer](/docs/metadata-jobs/mce-consumer-job) (optional), and +[Frontend](/docs/datahub-frontend). Kubernetes deployment for each of the components are +defined as subcharts under the main +[DataHub](https://github.com/acryldata/datahub-helm/tree/master/charts/datahub) +helm chart. + +The main components are powered by 4 external dependencies: + +- Kafka +- Local DB (MySQL, Postgres, MariaDB) +- Search Index (Elasticsearch) +- Graph Index (Supports either Neo4j or Elasticsearch) + +The dependencies must be deployed before deploying DataHub. We created a separate +[chart](https://github.com/acryldata/datahub-helm/tree/master/charts/prerequisites) +for deploying the dependencies with example configuration. They could also be deployed separately on-prem or leveraged +as managed services. To remove your dependency on Neo4j, set enabled to false in +the [values.yaml](https://github.com/acryldata/datahub-helm/blob/master/charts/prerequisites/values.yaml#L54) for +prerequisites. Then, override the `graph_service_impl` field in +the [values.yaml](https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/values.yaml#L63) of datahub +instead of `neo4j`. + +## Quickstart + +Assuming kubectl context points to the correct kubernetes cluster, first create kubernetes secrets that contain MySQL +and Neo4j passwords. + +```(shell) +kubectl create secret generic mysql-secrets --from-literal=mysql-root-password=datahub +kubectl create secret generic neo4j-secrets --from-literal=neo4j-password=datahub +``` + +The above commands sets the passwords to "datahub" as an example. Change to any password of choice. + +Add datahub helm repo by running the following + +```(shell) +helm repo add datahub https://helm.datahubproject.io/ +``` + +Then, deploy the dependencies by running the following + +```(shell) +helm install prerequisites datahub/datahub-prerequisites +``` + +Note, the above uses the default configuration +defined [here](https://github.com/acryldata/datahub-helm/blob/master/charts/prerequisites/values.yaml). You can change +any of the configuration and deploy by running the following command. + +```(shell) +helm install prerequisites datahub/datahub-prerequisites --values <> +``` + +Run `kubectl get pods` to check whether all the pods for the dependencies are running. You should get a result similar +to below. + +``` +NAME READY STATUS RESTARTS AGE +elasticsearch-master-0 1/1 Running 0 62m +elasticsearch-master-1 1/1 Running 0 62m +elasticsearch-master-2 1/1 Running 0 62m +prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 63m +prerequisites-kafka-0 1/1 Running 2 62m +prerequisites-mysql-0 1/1 Running 1 62m +prerequisites-neo4j-community-0 1/1 Running 0 52m +prerequisites-zookeeper-0 1/1 Running 0 62m +``` + +deploy DataHub by running the following + +```(shell) +helm install datahub datahub/datahub +``` + +Values in [values.yaml](https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/values.yaml) +have been preset to point to the dependencies deployed using +the [prerequisites](https://github.com/acryldata/datahub-helm/tree/master/charts/prerequisites) +chart with release name "prerequisites". If you deployed the helm chart using a different release name, update the +quickstart-values.yaml file accordingly before installing. + +:::note +The default values in [values.yaml](https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/values.yaml) auto generate secret values for the signing key and salt used for metadata_service_authentication. (To learn more about DataHub's backend authentication, check out [Introducing Metadata Service Authentication](../authentication/introducing-metadata-service-authentication.md). If you want to provide your own values refer to the `datahub.metadata_service_authentication` block in the [values.yaml](https://github.com/acryldata/datahub-helm/blob/master/charts/datahub/values.yaml) +::: + +Run `kubectl get pods` to check whether all the datahub pods are running. You should get a result similar to below. The Helm chart uses the system-update job (datahub-system-update) for database and search index setup; standalone `elasticsearchSetupJob` and `mysqlSetupJob` are disabled by default (`elasticsearchSetupJob.enabled: false`, `mysqlSetupJob.enabled: false`). SQL and index setup are controlled via `datahubSystemUpdate.sql.setup.enabled` and `datahubSystemUpdate.elasticsearch.setup.enabled` (default true). + +``` +NAME READY STATUS RESTARTS AGE +datahub-datahub-frontend-84c58df9f7-5bgwx 1/1 Running 0 4m2s +datahub-datahub-gms-58b676f77c-c6pfx 1/1 Running 0 4m2s +datahub-datahub-mae-consumer-7b98bf65d-tjbwx 1/1 Running 0 4m3s +datahub-datahub-mce-consumer-8c57d8587-vjv9m 1/1 Running 0 4m2s +datahub-datahub-system-update-xxxxx 0/1 Completed 0 4m50s +elasticsearch-master-0 1/1 Running 0 97m +elasticsearch-master-1 1/1 Running 0 97m +elasticsearch-master-2 1/1 Running 0 97m +prerequisites-cp-schema-registry-cf79bfccf-kvjtv 2/2 Running 1 99m +prerequisites-kafka-0 1/1 Running 2 97m +prerequisites-mysql-0 1/1 Running 1 97m +prerequisites-neo4j-community-0 1/1 Running 0 88m +prerequisites-zookeeper-0 1/1 Running 0 97m +``` + +You can run the following to expose the frontend locally. Note, you can find the pod name using the command above. In +this case, the datahub-frontend pod name was `datahub-datahub-frontend-84c58df9f7-5bgwx`. + +```(shell) +kubectl port-forward 9002:9002 +``` + +You should be able to access the frontend via http://localhost:9002. + +Once you confirm that the pods are running well, you can set up ingress for datahub-frontend to expose the 9002 port to +the public. + +## System update and upgrades + +The DataHub Helm chart runs a **system-update** Job on upgrades to apply schema changes, reindex search indices when needed, and run other blocking or non-blocking upgrade steps. When scale-down is enabled (see [Environment variables](environment-vars.md#kubernetes-scale-down-system-update)), the job can temporarily scale down deployments by label selector (e.g. MAE/MCE) and set environment variables on other deployments by label selector (e.g. GMS) before blocking upgrades (e.g. reindex), then restore them afterward. Rollout and scale-down operations run in parallel. Scale-down is conditional: it only runs when a blocking upgrade (such as BuildIndices when reindex is required) requests it. For upgrade behavior and potential downtime, see [Updating DataHub](../how/updating-datahub.md). + +## Other useful commands + +| Command | Description | +| ---------------------- | ----------------------- | +| helm uninstall datahub | Remove DataHub | +| helm ls | List of Helm charts | +| helm history | Fetch a release history | diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/deploy/telemetry.md b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/telemetry.md new file mode 100644 index 00000000..fce45419 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/deploy/telemetry.md @@ -0,0 +1,35 @@ +--- +title: DataHub Telemetry +sidebar_label: Telemetry +slug: /deploy/telemetry +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/deploy/telemetry.md +--- +# DataHub Telemetry + +## Overview of DataHub Telemetry + +To effectively build and maintain the DataHub Project, we must understand how end-users work within DataHub. Beginning in version 0.8.35, DataHub collects anonymous usage statistics and errors to inform our roadmap priorities and to enable us to proactively address errors. + +Both the DataHub backend and the ingestion framework collect telemetry. + +## DataHub Backend Telemetry + +Deployments are assigned a UUID which is sent along with event details, Java version, OS, and timestamp. +The source code is available [here](https://github.com/datahub-project/datahub/blob/master/metadata-service/factories/src/main/java/com/linkedin/gms/factory/telemetry/TelemetryUtils.java). + +## Ingestion Framework Telemetry + +The ingestion framework collects telemetry including CLI invocations, source/sink types, error types, versions, and timestamps. If you run with `datahub --debug`, all telemetry calls will be logged. + +On first invocation, the CLI will generate a randomized UUID, which will be sent alongside every telemetry event. This config is stored in `~/.datahub/telemetry-config.json`. + +The source code is available [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/telemetry/telemetry.py). + +## Disabling Telemetry + +Telemetry is enabled by default. While we are careful to anonymize all telemetry data and encourage users to keep it enabled so that we can improve DataHub, we understand that some users may wish to disable it. + +You can disable backend telemetry by setting the `DATAHUB_TELEMETRY_ENABLED` environment variable to `false`. You'll need to set this on both the datahub-gms and datahub-actions containers. + +If you're using the DataHub CLI, ingestion framework telemetry will be disabled when the `DATAHUB_TELEMETRY_ENABLED` environment variable is set to `false`. To persist this change for your machine, run `datahub telemetry disable`. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/agent-context.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/agent-context.md new file mode 100644 index 00000000..1560e1d0 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/agent-context.md @@ -0,0 +1,308 @@ +--- +title: Agent Context Kit +slug: /dev-guides/agent-context/agent-context +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/agent-context/agent-context.md +--- +# Agent Context Kit + +> **📚 Navigation**: [LangChain Integration →](./langchain.md) | [Google ADK Integration →](./google-adk.md) | [Google Vertex AI Integration →](./google-vertex-ai.md) | [Snowflake Integration →](./snowflake.md) | [Copilot Studio Integration →](./copilot-studio.md) + +## What Problem Does This Solve? + +When building AI agents that answer questions about data, agents often face these challenges: + +- **Hallucinate metadata**: Generate table or column names that don't exist +- **Lack context**: Can't discover related datasets, lineage, or business definitions +- **Missing ownership info**: Don't know who owns what data or how to contact them +- **No quality signals**: Can't distinguish certified datasets from deprecated ones + +**Agent Context Kit solves this** by giving AI agents real-time access to your DataHub metadata catalog, enabling them to provide accurate, contextual answers about your data ecosystem. + +### Example Use Cases + +- **Data Discovery**: "Show me all datasets owned by the analytics team" +- **Schema Exploration**: "What tables have a customer_id column?" +- **Lineage Tracing**: "Trace lineage from raw data to this dashboard" +- **Documentation Search**: "Find the business definition of 'churn rate'" +- **Compliance Queries**: "List all PII fields and their owners" + +## Overview + +**DataHub Agent Context** provides a collection of tools and utilities for building AI agents that interact with DataHub metadata. This package contains MCP (Model Context Protocol) tools that enable AI agents to search, retrieve, and manipulate metadata in DataHub. These can be used directly to create an agent, or be included in an MCP server such as DataHub's open source MCP server. + +### Quick Start Guide + +1. **New to Agent Context?** Start here with the basic example below +2. **Using LangChain?** See the [LangChain integration guide](./langchain.md) +3. **Using Google ADK?** See the [Google ADK integration guide](./google-adk.md) +4. **Using Google Vertex AI Agent Builder?** See the [Google Vertex AI integration guide](./google-vertex-ai.md) +5. **Using Snowflake Intelligence?** See the [Snowflake integration guide](./snowflake.md) + +## Installation + +```bash +pip install datahub-agent-context +``` + +### Prerequisites + +- DataHub instance (Cloud or self-hosted) +- Python 3.10 or higher +- DataHub personal access token (for authentication) + +## Basic Usage + +### Simple Example + +These tools are designed to be used with an AI agent and have the responses passed directly to an LLM, so the return schema is a simple dict, but they can be used independently if desired. + +```python +from datahub.sdk.main_client import DataHubClient +from datahub_agent_context.context import DataHubContext +from datahub_agent_context.mcp_tools.search import search +from datahub_agent_context.mcp_tools.entities import get_entities + +# Initialize DataHub client from environment (or specify server/token) +client = DataHubClient.from_env() +# Or: client = DataHubClient(server="http://localhost:8080", token="YOUR_TOKEN") + +# Use DataHubContext to set up the client for tool calls +with DataHubContext(client): + # Search for datasets + results = search( + query="user_data", + filters={"entity_type": ["dataset"]}, + num_results=10 + ) + + print(f"Found {len(results['searchResults'])} datasets") + for result in results["searchResults"]: + print(f"- {result['entity']['name']} ({result['entity']['urn']})") + + # Get detailed entity information + entity_urns = [result["entity"]["urn"] for result in results["searchResults"]] + entities = get_entities(urns=entity_urns) + + print(f"\nDetailed info for {len(entities['entities'])} entities:") + for entity in entities["entities"]: + print(f"- {entity['urn']}: {entity.get('properties', {}).get('description', 'No description')}") +``` + +## Key Concepts (Glossary) + +Before using Agent Context Kit, familiarize yourself with these DataHub concepts: + +- **Entity**: A metadata object in DataHub (e.g., Dataset, Dashboard, Chart, User). Think of these as the "nouns" of your data ecosystem. +- **URN (Uniform Resource Name)**: A unique identifier for an entity. Format: `urn:li:dataset:(urn:li:dataPlatform:mysql,mydb.users,PROD)`. This is like a primary key for metadata. +- **MCP (Model Context Protocol)**: A standard protocol for connecting AI agents to external data sources. These tools implement MCP for DataHub. +- **Client**: The underlying client used internally to query DataHub. For LangChain users, this is handled automatically by the builder. + +## Agent Platforms + +| Platform | Status | Guide | +| ------------------------ | ----------- | ----------------------------------------------- | +| Custom | Launched | See below | +| Langchain | Launched | [LangChain Guide](./langchain.md) | +| Snowflake | Launched | [Snowflake Guide](./snowflake.md) | +| Google ADK | Launched | [Google ADK Guide](./google-adk.md) | +| Google Vertex AI | Launched | [Google Vertex AI Guide](./google-vertex-ai.md) | +| Microsoft Copilot Studio | Launched | [Copilot Studio Guide](./copilot-studio.md) | +| Crew AI | Coming Soon | - | +| OpenAI | Coming Soon | - | + +## Available Tools + +#### Search Tools + +- **`search(client, query, filters, num_results)`** + + - **Use when**: Finding entities by keyword across DataHub + - **Returns**: List of matching entities with URNs, names, and descriptions + - **Example**: `search(client, "customer", {"entity_type": ["dataset"]}, 10)` to find datasets about customers + - **Filters**: Can filter by entity_type, platform, domain, tags, and more + +- **`search_documents(client, query, semantic_query, num_results)`** + + - **Use when**: Searching for documentation, business glossaries, or knowledge base articles + - **Returns**: Document entities with titles and content + - **Example**: `search_documents(client, "*", "data retention policy", 5)` to find policy documents + +- **`grep_documents(client, pattern, num_results)`** + - **Use when**: Searching for specific patterns or exact phrases in documentation + - **Returns**: Documents containing the pattern with matched excerpts + - **Example**: `grep_documents(client, "PII.*encrypted", 10)` to find docs mentioning PII encryption + +#### Entity Tools + +- **`get_entities(client, urns)`** + + - **Use when**: Retrieving detailed metadata for specific entities you already know the URNs for + - **Returns**: Full entity metadata including all aspects (schema, ownership, properties, etc.) + - **Example**: After search, use this to get complete details about the found entities + +- **`list_schema_fields(client, urn, filters)`** + - **Use when**: Exploring columns/fields in a dataset + - **Returns**: List of fields with names, types, descriptions, and tags + - **Example**: `list_schema_fields(client, dataset_urn, {"field_path": "customer_"})` to find customer-related columns + - **Filters**: Can filter by field name patterns, data types, or tags + +#### Lineage Tools + +- **`get_lineage(client, urn, direction, max_depth)`** + + - **Use when**: Understanding data flow and dependencies + - **Returns**: Upstream (sources) or downstream (consumers) entities + - **Example**: `get_lineage(client, dashboard_urn, "UPSTREAM", 3)` to trace data sources for a dashboard + - **Direction**: Use "UPSTREAM" for sources, "DOWNSTREAM" for consumers + +- **`get_lineage_paths_between(client, source_urn, destination_urn)`** + - **Use when**: Finding how data flows between two specific entities + - **Returns**: All paths connecting the entities with intermediate steps + - **Example**: Find how raw data flows to a specific dashboard + +#### Query Tools + +- **`get_dataset_queries(client, urn, column_name)`** + - **Use when**: Finding SQL queries that use a dataset or specific column + - **Returns**: List of queries with SQL text and metadata + - **Example**: `get_dataset_queries(client, dataset_urn, "email")` to see how the email column is used + - **Use cases**: Understanding data usage patterns, finding query examples + +#### Mutation Tools + +**Note**: These tools modify metadata. Use with caution in production environments. + +- **`add_tags(client, urn, tags)`** / **`remove_tags(client, urn, tags)`** + + - **Use when**: Categorizing or labeling entities + - **Example**: `add_tags(client, dataset_urn, ["PII", "Finance"])` to mark sensitive data + +- **`update_description(client, urn, description)`** + + - **Use when**: Adding or updating documentation for entities + - **Example**: Agents can auto-generate and update descriptions + +- **`set_domains(client, urn, domain_urns)`** / **`remove_domains(client, urn, domain_urns)`** + + - **Use when**: Organizing entities into business domains + - **Example**: Assign datasets to "Marketing" or "Finance" domains + +- **`add_owners(client, urn, owners)`** / **`remove_owners(client, urn, owners)`** + + - **Use when**: Assigning data ownership and accountability + - **Example**: `add_owners(client, dataset_urn, [{"owner": user_urn, "type": "TECHNICAL_OWNER"}])` + +- **`add_glossary_terms(client, urn, term_urns)`** / **`remove_glossary_terms(client, urn, term_urns)`** + + - **Use when**: Linking entities to business glossary definitions + - **Example**: Link a revenue column to the "Revenue" glossary term + +- **`add_structured_properties(client, urn, properties)`** / **`remove_structured_properties(client, urn, properties)`** + + - **Use when**: Adding custom metadata fields to entities + - **Example**: Add "data_retention_days" or "compliance_tier" properties + +- **`save_document(document_type, title, content, urn, topics, related_documents, related_assets)`** + + - **Use when**: Creating or updating standalone documents in DataHub's knowledge base (insights, decisions, FAQs, analysis, etc.) + - **Document types**: "Insight", "Decision", "FAQ", "Analysis", "Summary", "Recommendation", "Note", "Context" + - **Parameters**: + - `document_type`: Type of document (required) + - `title`: Document title (required) + - `content`: Full document content in markdown format (required) + - `urn`: URN of existing document to update (optional, creates new if not provided) + - `topics`: List of topic tags for categorization (optional) + - `related_documents`: URNs of related documents (optional) + - `related_assets`: URNs of related data assets like datasets or dashboards (optional) + - **Example**: `save_document("Insight", "High Null Rate in Customer Emails", "## Finding\\n\\n23% of customer records have null email...", topics=["data-quality", "customer-data"], related_assets=["urn:li:dataset:(urn:li:dataPlatform:snowflake,customers,PROD)"])` + - **Important**: Always confirm with user before saving. Documents are visible to all DataHub users. + - **Note**: For updating descriptions on data assets (datasets, dashboards), use `update_description` instead + +#### User Tools + +- **`get_me(client)`** + - **Use when**: Getting information about the authenticated user + - **Returns**: User details including name, email, and roles + - **Use cases**: Personalization, permission checks, audit logging + +## MCP Server + +It is also possible to connect your agent or tool directly to the **DataHub MCP Server**: [DataHub MCP Server](../../features/feature-guides/mcp.md) with your chosen framework. + +## Troubleshooting + +### Authentication Errors + +**Problem**: `Unauthorized` or `401` errors when calling tools + +**Solutions**: + +- Verify your DataHub token is valid: `datahub check metadata-service` +- Ensure the token has the required permissions (read access for search tools, write access for mutation tools) +- Check that the token hasn't expired + +### Connection Errors + +**Problem**: `Connection refused` or timeout errors + +**Solutions**: + +- Verify DataHub server URL is correct and accessible +- Check network connectivity: `curl -I https://your-datahub-instance.com/api/gms/health` +- Ensure firewall rules allow outbound connections to DataHub +- For self-hosted DataHub, verify the service is running + +### Empty or Unexpected Results + +**Problem**: Search returns no results or missing expected entities + +**Solutions**: + +- Verify entities exist in DataHub UI first +- Check that your search query isn't too restrictive +- Try removing filters to broaden the search +- Ensure entity types are spelled correctly (case-sensitive): `dataset`, not `Dataset` +- For schema fields, verify the dataset URN is correct + +### Import Errors + +**Problem**: `ModuleNotFoundError: No module named 'datahub_agent_context'` + +**Solutions**: + +- Ensure package is installed: `pip install datahub-agent-context` +- If using LangChain: `pip install datahub-agent-context[langchain]` +- If using Snowflake: `pip install datahub-agent-context[snowflake]` +- Verify you're using the correct Python environment + +### Rate Limiting + +**Problem**: `429 Too Many Requests` errors + +**Solutions**: + +- Implement exponential backoff and retry logic +- Reduce the frequency of API calls +- For batch operations, use pagination instead of large single requests +- Contact DataHub admin to adjust rate limits if needed + +### Debugging Tips + +Enable debug logging to see detailed API calls: + +```python +import logging +logging.basicConfig(level=logging.DEBUG) + +# Your agent code here +``` + +Check the DataHub server logs for more details on server-side errors. + +### Getting Help + +- **Documentation**: [DataHub Docs](/) +- **Community Slack**: [Join DataHub Slack](https://datahub.com/slack/) +- **GitHub Issues**: [Report bugs](https://github.com/datahub-project/datahub/issues) +- **Email Support**: For DataHub Cloud customers, contact support@acryl.io diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/copilot-studio.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/copilot-studio.md new file mode 100644 index 00000000..327fca6f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/copilot-studio.md @@ -0,0 +1,130 @@ +--- +title: Microsoft Copilot Studio Integration +slug: /dev-guides/agent-context/copilot-studio +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/agent-context/copilot-studio.md +--- +# Microsoft Copilot Studio Integration + +> **📚 Navigation**: [← Back to Agent Context Kit](./agent-context.md) | [LangChain Integration →](./langchain.md) | [Snowflake Integration →](./snowflake.md) + +## What Problem Does This Solve? + +[Microsoft Copilot Studio](https://www.microsoft.com/en-us/microsoft-copilot/microsoft-copilot-studio) lets you build custom AI agents that work across Microsoft 365, Teams, and other channels. Out of the box, these agents don't know anything about your organization's data assets — they can't search your data catalog, trace lineage, or answer questions about dataset ownership and quality. + +By connecting DataHub's MCP server to Copilot Studio, your agents gain direct access to your metadata catalog. Users can ask questions like _"What are the most queried datasets?"_ or _"Who owns the revenue table?"_ and get grounded answers drawn from real metadata. + +## Prerequisites + +- A [Microsoft Copilot Studio](https://copilotstudio.microsoft.com/) account +- A DataHub Cloud instance (v0.3.12+) or a self-hosted DataHub instance with the [MCP server](../../features/feature-guides/mcp.md) running +- A DataHub [personal access token](../../authentication/personal-access-tokens.md) + +## Setup Guide + +### Step 1: Create or Open an Agent + +1. Navigate to [Copilot Studio](https://copilotstudio.microsoft.com/) and sign in. +2. Create a new agent by clicking **+ Create a blank agent**, or open an existing one. + +

+ +

+ +### Step 2: Add an MCP Tool + +1. From your agent's overview page, scroll down to the **Tools** section and click **+ Add tool**. + +

+ +

+ +2. In the "Add tool" dialog, select **Model Context Protocol** under "Create new". + +

+ +

+ +### Step 3: Configure the MCP Server Connection + +Fill in the MCP server details: + +| Field | Value | +| ---------------------- | -------------------------------------------------------------- | +| **Server name** | `DataHub MCP Server` | +| **Server description** | `DataHub MCP server to connect to datasets, documents, & more` | +| **Server URL** | `https://.acryl.io/integrations/ai/mcp` | +| **Authentication** | `API key` | +| **Type** | `Header` | +| **Header name** | `Authorization` | +| **Header value** | `Bearer ` | + +Replace `` with your DataHub Cloud tenant name and `` with your [personal access token](../../authentication/personal-access-tokens.md). + +

+ +

+ +:::note Self-Hosted DataHub +If you're using a self-hosted DataHub instance, you'll need to run the [self-hosted MCP server](../../features/feature-guides/mcp.md#self-hosted-mcp-server-usage) and expose it via a publicly accessible URL. Use that URL as the **Server URL** above. +::: + +### Step 4: Add and Configure Tools + +1. After entering the connection details, click **Next**. Copilot Studio will discover the available tools from the DataHub MCP server. +2. Review the connection and click **Add and configure** to add the MCP server to your agent. + +

+ +

+ +3. You should now see all available DataHub tools listed. Toggle on the tools you want your agent to have access to. + +

+ +

+ +### Step 5: Test Your Agent + +Click the **Test** button in the top-right corner to open the test panel. Try asking your agent a question like: + +- _"Find the most important data at my organization"_ +- _"What datasets does the analytics team own?"_ +- _"Show me the lineage for the revenue dashboard"_ + +

+ +

+ +## Tips + +- **Add instructions**: Use the **Instructions** field on the agent overview page to guide how the agent uses DataHub tools — e.g., _"Always search DataHub before answering data questions"_ or _"Include dataset owners in your responses."_ +- **Add knowledge**: You can attach additional documents or data sources under the **Knowledge** section to complement DataHub metadata. +- **Publish your agent**: Once you're happy with the agent, click **Publish** to make it available in Teams, your website, or other channels. + +## Troubleshooting + +### Connection Errors + +If Copilot Studio can't connect to the MCP server: + +- Verify your DataHub Cloud instance URL is correct (`https://.acryl.io/integrations/ai/mcp`) +- Ensure the personal access token is valid and hasn't expired +- Confirm that authentication is set to **API key** (not OAuth 2.0) +- Make sure the header value includes the `Bearer ` prefix (with the trailing space) + +### Tools Not Appearing + +If tools don't appear after adding the MCP server: + +- Click the refresh button on the Tools page to re-discover tools +- Verify the MCP server is running and healthy +- Check that your token has the required permissions + +### Empty Results + +If the agent returns no results for queries: + +- Verify that your DataHub instance has ingested metadata +- Try broader search terms +- Check the [MCP server troubleshooting guide](../../features/feature-guides/mcp.md#troubleshooting) for more details diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-adk.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-adk.md new file mode 100644 index 00000000..8ec5006d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-adk.md @@ -0,0 +1,368 @@ +--- +title: Google ADK Integration +slug: /dev-guides/agent-context/google-adk +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/agent-context/google-adk.md +--- +# Google ADK Integration + +> **📚 Navigation**: [← Back to Agent Context Kit](./agent-context.md) | [LangChain Integration →](./langchain.md) + +## What Problem Does This Solve? + +Google ADK (Agent Development Kit) is Google's framework for building AI agents with Gemini models, but agents often struggle with data questions because they: + +- Don't have access to your organization's data catalog +- Hallucinate table names, schemas, and relationships +- Can't discover data ownership or documentation +- Have no context about data quality or lineage + +**The Google ADK integration** provides pre-built tools that give your Gemini-powered agents direct access to DataHub metadata, enabling them to answer data questions accurately using real metadata from your organization. + +### What You Can Build + +- **Data Discovery Chatbot**: "Show me all datasets about customers in the marketing domain" +- **Schema Explorer**: "What columns exist in the user_events table and what do they mean?" +- **Lineage Tracer**: "What dashboards use data from this raw table?" +- **Documentation Assistant**: "Find the business definition of 'monthly recurring revenue'" +- **Compliance Helper**: "List all tables with PII and their owners" + +## Overview + +The Google ADK optional add-on lets you connect DataHub's context tools directly to a Google ADK `Agent`. You can also connect via the DataHub MCP server using ADK's built-in `McpToolset` — useful if you prefer to run a standalone MCP server rather than embedding tools in your Python process. + +## Installation + +```bash +pip install datahub-agent-context[google-adk] +``` + +### Prerequisites + +- Python 3.10 or higher +- Google ADK (`pip install google-adk`) +- DataHub instance with access token +- Google API key (Gemini Developer API) **or** Google Cloud credentials (Vertex AI) + +## Quick Start + +### Basic Setup + +Build AI agents with pre-built Google ADK tools: + +```python +from datahub.sdk.main_client import DataHubClient +from datahub_agent_context.google_adk_tools import build_google_adk_tools + +# Initialize DataHub client from environment (recommended) +client = DataHubClient.from_env() +# Or specify server and token explicitly: +# client = DataHubClient(server="http://localhost:8080", token="YOUR_TOKEN") + +# Build all tools (read-only by default) +tools = build_google_adk_tools(client, include_mutations=False) + +# Or include mutation tools for tagging, descriptions, etc. +tools = build_google_adk_tools(client, include_mutations=True) +``` + +**Note**: `include_mutations=False` provides read-only tools (search, get entities, lineage). Set to `True` to enable tools that modify metadata (add tags, update descriptions, etc.). + +## Complete Working Example + +Here's a full example of a DataHub-powered Google ADK agent: + +```python +import asyncio +import os + +from google.adk.agents import Agent +from google.adk.runners import Runner +from google.adk.sessions import InMemorySessionService +from google.genai import types + +from datahub.sdk.main_client import DataHubClient +from datahub_agent_context.google_adk_tools import build_google_adk_tools + +# Initialize DataHub connection +datahub_gms_url = os.getenv("DATAHUB_GMS_URL") +if datahub_gms_url is None: + client = DataHubClient.from_env() +else: + client = DataHubClient( + server=datahub_gms_url, token=os.getenv("DATAHUB_GMS_TOKEN") + ) + +# Build DataHub tools (read-only) +tools = build_google_adk_tools(client, include_mutations=False) + +# Create agent +agent = Agent( + model="gemini-2.5-flash", + name="datahub_agent", + description="A data discovery assistant with access to DataHub metadata.", + instruction="""You are a helpful data catalog assistant with access to DataHub metadata. + +Use the available tools to: +- Search for datasets, dashboards, and other data assets +- Get detailed entity information including schemas and descriptions +- Trace data lineage to understand data flow +- Find documentation and business glossary terms + +Always provide URNs when referencing entities so users can find them in DataHub. +Be concise but thorough in your explanations.""", + tools=tools, +) + + +async def ask_datahub(question: str) -> str: + """Ask a question about your data catalog.""" + session_service = InMemorySessionService() + session = await session_service.create_session( + app_name="datahub_agent", + user_id="user", + ) + runner = Runner( + agent=agent, + app_name="datahub_agent", + session_service=session_service, + ) + + response_text = "" + async for event in runner.run_async( + user_id="user", + session_id=session.id, + new_message=types.Content(role="user", parts=[types.Part(text=question)]), + ): + if event.is_final_response() and event.content and event.content.parts: + response_text = event.content.parts[0].text + return response_text + + +# Example queries +if __name__ == "__main__": + # Find datasets + print(asyncio.run(ask_datahub("Find all datasets about customers"))) + + # Get schema information + print(asyncio.run(ask_datahub("What columns are in the user_events dataset?"))) + + # Trace lineage + print(asyncio.run(ask_datahub("Show me the upstream sources for the revenue dashboard"))) + + # Find documentation + print(asyncio.run(ask_datahub("What's the business definition of 'churn rate'?"))) +``` + +### Example Output + +``` +Query: Find all datasets about customers + +Agent: I found 3 datasets related to customers: + +1. **customer_profiles** (urn:li:dataset:(urn:li:dataPlatform:snowflake,prod.analytics.customer_profiles,PROD)) + - Description: Core customer profile data including demographics and preferences + - Platform: Snowflake + - Domain: Marketing + +2. **customer_transactions** (urn:li:dataset:(urn:li:dataPlatform:postgres,transactions.customer_orders,PROD)) + - Description: Historical customer purchase transactions + - Platform: PostgreSQL + - Domain: Finance + +3. **customer_360** (urn:li:dataset:(urn:li:dataPlatform:bigquery,analytics.customer_360,PROD)) + - Description: Unified customer view combining profile, transactions, and interactions + - Platform: BigQuery + - Domain: Analytics + +You can view these in DataHub at: https://your-datahub.acryl.io +``` + +## Advanced Usage + +### Interactive Agent + +For a conversational experience with multi-turn support, create a fresh session per turn to keep memory usage bounded: + +```python +async def interactive_agent(agent: Agent) -> None: + session_service = InMemorySessionService() + runner = Runner( + agent=agent, + app_name="datahub_agent", + session_service=session_service, + ) + + print("Interactive Mode - Type 'quit' to exit") + while True: + user_input = input("\nYou: ").strip() + if not user_input or user_input.lower() in ["quit", "exit", "q"]: + break + + # New session per turn keeps memory usage bounded + session = await session_service.create_session( + app_name="datahub_agent", + user_id="user", + ) + response = "" + async for event in runner.run_async( + user_id="user", + session_id=session.id, + new_message=types.Content(role="user", parts=[types.Part(text=user_input)]), + ): + if event.is_final_response() and event.content and event.content.parts: + response = event.content.parts[0].text + print(f"\nAgent: {response}") +``` + +### Connecting via DataHub MCP Server + +If you're running the [DataHub MCP Server](../../features/feature-guides/mcp.md), you can connect Google ADK directly to it using `McpToolset` instead of the Python tools builder: + +```python +import asyncio + +from google.adk.agents import Agent +from google.adk.runners import Runner +from google.adk.sessions import InMemorySessionService +from google.adk.tools.mcp_tool import McpToolset +from google.adk.tools.mcp_tool.mcp_session_manager import StreamableHTTPConnectionParams +from google.genai import types + +# Replace with your MCP server URL. +# Use https://.acryl.io/integrations/ai/mcp for Datahub cloud +MCP_URL = "http://localhost:8080/mcp" + +async def main() -> None: + session_service = InMemorySessionService() + session = await session_service.create_session( + app_name="datahub_mcp_agent", + user_id="user", + ) + + toolset = McpToolset( + connection_params=StreamableHTTPConnectionParams(url=MCP_URL), + headers={"Authorization": f"Bearer {YOUR_TOKEN}"}, + ) + # Eagerly initialize the MCP session so the AsyncExitStack is owned here — + # otherwise ADK creates it in a spawned task and close() raises an error. + await toolset.get_tools() + + agent = Agent( + model="gemini-2.5-flash", + name="datahub_agent", + instruction="You help users find datasets in DataHub. Provide clear, concise answers.", + tools=[toolset], + ) + + runner = Runner( + agent=agent, + app_name="datahub_mcp_agent", + session_service=session_service, + ) + + try: + async for event in runner.run_async( + user_id="user", + session_id=session.id, + new_message=types.Content( + role="user", + parts=[types.Part(text="Find datasets about users")], + ), + ): + if event.is_final_response() and event.content and event.content.parts: + print(f"Agent: {event.content.parts[0].text}") + finally: + await toolset.close() + + +if __name__ == "__main__": + asyncio.run(main()) +``` + +### Using Vertex AI Instead of Gemini Developer API + +To use Vertex AI (Application Default Credentials) instead of an API key: + +```python +import os + +# Do not set GOOGLE_API_KEY — ADK falls back to ADC automatically +# Ensure your environment is authenticated: gcloud auth application-default login + +agent = Agent( + model="gemini-2.5-flash", # or a Vertex AI model ID + name="datahub_agent", + instruction="...", + tools=tools, +) +``` + +## More Examples + +Complete examples are available in the datahub-project repo: + +- [Basic Agent](https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/examples/google_adk/basic_agent.py) +- [Simple Search](https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/examples/google_adk/simple_search.py) + +## Troubleshooting + +### Tool Execution Errors + +**Problem**: Agent tries to use tools but gets errors + +**Solutions**: + +- Verify DataHub connection: Test `client.config` is correctly set +- Check tool availability: Print `[tool.name for tool in tools]` to see available tools +- Validate token permissions: Ensure token has read access (and write access if using mutations) +- Set `GOOGLE_GENAI_USE_VERTEXAI=0` if mixing Gemini Developer API and Vertex AI configs + +### Agent Not Using Tools + +**Problem**: Agent responds without calling DataHub tools + +**Solutions**: + +- Improve the `instruction` prompt: explicitly instruct the agent to use tools +- Check model compatibility: Gemini 2.0+ models have the best tool-calling support +- Reduce temperature: ADK uses `temperature=0` by default for tool-heavy agents + +### AsyncExitStack / Task Errors + +**Problem**: `Attempted to exit cancel scope in a different task` when using `McpToolset` + +**Solution**: Call `await toolset.get_tools()` in the same async task that owns the toolset before passing it to the `Agent`, and call `await toolset.close()` in a `finally` block. + +### Performance Issues + +**Problem**: Agent responses are slow + +**Solutions**: + +- Limit search results: Use `num_results` filter in tool calls +- Use `gemini-2.0-flash` or `gemini-2.5-flash` for faster responses +- Cache results: Implement caching for frequently accessed metadata + +### Import Errors + +**Problem**: `ModuleNotFoundError` for Google ADK components + +**Solutions**: + +```bash +# Install all required dependencies +pip install datahub-agent-context[google-adk] +pip install google-adk + +# Or install from scratch +pip install datahub-agent-context google-adk google-genai +``` + +### Getting Help + +- **Google ADK Docs**: [Google ADK Documentation](https://google.github.io/adk-docs/) +- **DataHub Agent Context**: [Agent Context Kit Guide](./agent-context.md) +- **GitHub Issues**: [Report issues](https://github.com/datahub-project/datahub/issues) +- **Community Slack**: [Join DataHub Slack](https://datahub.com/slack/) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-vertex-ai.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-vertex-ai.md new file mode 100644 index 00000000..674b4b8b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/google-vertex-ai.md @@ -0,0 +1,86 @@ +--- +title: Google Vertex AI Integration +slug: /dev-guides/agent-context/google-vertex-ai +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/agent-context/google-vertex-ai.md +--- +# Google Vertex AI Integration + +> **📚 Navigation**: [← Back to Agent Context Kit](./agent-context.md) | [Google ADK Integration →](./google-adk.md) + +## What Problem Does This Solve? + +Google Vertex AI Agent Builder provides a visual, low-code environment for building AI agents — but agents often struggle with data questions because they: + +- Don't have access to your organization's data catalog +- Hallucinate table names, schemas, and relationships +- Can't discover data ownership or documentation +- Have no context about data quality or lineage + +**The Vertex AI integration** lets you connect the DataHub MCP server to your Vertex AI agent visually, enabling your agents to answer data questions accurately using real metadata from your organization. + +## Overview + +[Agent Designer](https://docs.cloud.google.com/agent-builder/agent-designer) is a low-code visual designer built into the Google Cloud Console as part of Vertex AI Agent Builder. It lets you design and test agents in the browser, then export the generated code for further development. + +> **Note**: Agent Designer is currently a **preview feature**. + +## Connecting DataHub via the Agent Designer UI + +### Prerequisites + +- A Google Cloud project with Vertex AI Agent Builder enabled +- A running DataHub instance with the [MCP server](../../features/feature-guides/mcp.md) enabled +- Your DataHub MCP server URL (e.g., `https://.acryl.io/integrations/ai/mcp`) + +### Steps + +1. Open the **Agent Designer** in the [Google Cloud Console](https://console.cloud.google.com/gen-app-builder/agents). +2. Click **"Create agent"** to open the visual canvas. +3. Configure the agent: + - Set a **name** and **description** for your agent. + - Write **instructions** to guide the agent's behavior (e.g., "You are a data catalog assistant. Use DataHub tools to find datasets, schemas, and lineage."). + - Select your preferred **model** (e.g., Gemini 2.5 Flash). +4. Click **"Add tools"** and select **"MCP Server"**. +5. Enter a display name (e.g., `DataHub`) and your DataHub MCP endpoint URL. +6. Click **"Save"** — Agent Designer will discover all available tools from the MCP server automatically. +7. Use the **Preview tab** to test your agent by chatting with it. + +### MCP Authentication Limitation + +> **Important**: The Agent Designer UI only supports MCP servers that do **not** require authentication. If your DataHub MCP server requires a bearer token (as DataHub Cloud does), the UI cannot pass authentication headers. + +To work around this, use the **"Get code"** button in Agent Designer to export the generated agent code, then add the `Authorization` header manually: + +```python +from google.adk.tools.mcp_tool import McpToolset +from google.adk.tools.mcp_tool.mcp_session_manager import StreamableHTTPConnectionParams + +MCP_URL = "https://.acryl.io/integrations/ai/mcp" +YOUR_TOKEN = "" + +toolset = McpToolset( + connection_params=StreamableHTTPConnectionParams(url=MCP_URL), + # Add the Authorization header here — the UI cannot do this for you + headers={"Authorization": f"Bearer {YOUR_TOKEN}"}, +) +``` + +See the [Google ADK Integration](./google-adk.md#connecting-via-datahub-mcp-server) page for a complete working example with authentication. + +## Exporting to Code + +Once you're happy with your agent design, click **"Get code"** to view the full source code representation of your agent. You can copy this code into your editor and continue development using: + +- The **Google ADK** — see the [Google ADK Integration](./google-adk.md) guide +- Frameworks such as LangChain or LangGraph + +This allows you to start with the visual designer for rapid prototyping and then transition to code for production-grade agents that require authentication, custom logic, or CI/CD deployment. + +## Getting Help + +- **Agent Designer Docs**: [Google Cloud Agent Designer](https://docs.cloud.google.com/agent-builder/agent-designer) +- **Google ADK Docs**: [Google ADK Documentation](https://google.github.io/adk-docs/) +- **DataHub Agent Context**: [Agent Context Kit Guide](./agent-context.md) +- **GitHub Issues**: [Report issues](https://github.com/datahub-project/datahub/issues) +- **Community Slack**: [Join DataHub Slack](https://datahub.com/slack/) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/langchain.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/langchain.md new file mode 100644 index 00000000..aeb17163 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/langchain.md @@ -0,0 +1,280 @@ +--- +title: LangChain Integration +slug: /dev-guides/agent-context/langchain +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/agent-context/langchain.md +--- +# LangChain Integration + +> **📚 Navigation**: [← Back to Agent Context Kit](./agent-context.md) | [Snowflake Integration →](./snowflake.md) + +## What Problem Does This Solve? + +LangChain is a popular framework for building AI agent applications, but agents often struggle with data questions because they: + +- Don't have access to your organization's data catalog +- Hallucinate table names, schemas, and relationships +- Can't discover data ownership or documentation +- Have no context about data quality or lineage + +**The LangChain integration** provides pre-built LangChain tools that give your agents direct access to DataHub metadata, enabling them to answer data questions accurately using real metadata from your organization. + +### What You Can Build + +- **Data Discovery Chatbot**: "Show me all datasets about customers in the marketing domain" +- **Schema Explorer**: "What columns exist in the user_events table and what do they mean?" +- **Lineage Tracer**: "What dashboards use data from this raw table?" +- **Documentation Assistant**: "Find the business definition of 'monthly recurring revenue'" +- **Compliance Helper**: "List all tables with PII and their owners" + +## Overview + +The LangChain optional add-on lets you connect DataHub's context tools directly to your LangChain Agent. + +## Installation + +```bash +pip install datahub-agent-context[langchain] +``` + +### Prerequisites + +- Python 3.10 or higher +- LangChain (`pip install langchain langchain-openai`) +- DataHub instance with access token +- OpenAI API key (or other LLM provider) + +## Quick Start + +### Basic Setup + +Build AI agents with pre-built LangChain tools: + +```python +from datahub.sdk.main_client import DataHubClient +from datahub_agent_context.langchain_tools import build_langchain_tools + +# Initialize DataHub client from environment (recommended) +client = DataHubClient.from_env() +# Or specify server and token explicitly: +# client = DataHubClient(server="http://localhost:8080", token="YOUR_TOKEN") + +# Build all tools (read-only by default) +tools = build_langchain_tools(client, include_mutations=False) + +# Or include mutation tools for tagging, descriptions, etc. +tools = build_langchain_tools(client, include_mutations=True) +``` + +**Note**: `include_mutations=False` provides read-only tools (search, get entities, lineage). Set to `True` to enable tools that modify metadata (add tags, update descriptions, etc.). + +## Complete Working Example + +Here's a full example of a DataHub-powered LangChain agent: + +```python +from datahub.sdk.main_client import DataHubClient +from datahub_agent_context.langchain_tools import build_langchain_tools +from langchain_openai import ChatOpenAI +from langchain.agents import AgentExecutor, create_tool_calling_agent +from langchain.prompts import ChatPromptTemplate + +# Initialize DataHub client +client = DataHubClient.from_env() +# Or: client = DataHubClient(server="http://localhost:8080", token="YOUR_TOKEN") + +# Build DataHub tools (read-only) +tools = build_langchain_tools(client, include_mutations=False) + +# Initialize LLM +llm = ChatOpenAI( + model="gpt-4", + temperature=0, + openai_api_key="YOUR_OPENAI_KEY" +) + +# Create agent prompt +prompt = ChatPromptTemplate.from_messages([ + ("system", """You are a helpful data catalog assistant with access to DataHub metadata. + + Use the available tools to: + - Search for datasets, dashboards, and other data assets + - Get detailed entity information including schemas and descriptions + - Trace data lineage to understand data flow + - Find documentation and business glossary terms + + Always provide URNs when referencing entities so users can find them in DataHub. + Be concise but thorough in your explanations."""), + ("human", "{input}"), + ("placeholder", "{agent_scratchpad}"), +]) + +# Create agent +agent = create_tool_calling_agent(llm, tools, prompt) +agent_executor = AgentExecutor( + agent=agent, + tools=tools, + verbose=True, + handle_parsing_errors=True +) + +# Use the agent +def ask_datahub(question: str): + """Ask a question about your data catalog.""" + result = agent_executor.invoke({"input": question}) + return result["output"] + +# Example queries +if __name__ == "__main__": + # Find datasets + print(ask_datahub("Find all datasets about customers")) + + # Get schema information + print(ask_datahub("What columns are in the user_events dataset?")) + + # Trace lineage + print(ask_datahub("Show me the upstream sources for the revenue dashboard")) + + # Find documentation + print(ask_datahub("What's the business definition of 'churn rate'?")) +``` + +### Example Output + +``` +User: Find all datasets about customers + +Agent: I found 3 datasets related to customers: + +1. **customer_profiles** (urn:li:dataset:(urn:li:dataPlatform:snowflake,prod.analytics.customer_profiles,PROD)) + - Description: Core customer profile data including demographics and preferences + - Platform: Snowflake + - Domain: Marketing + +2. **customer_transactions** (urn:li:dataset:(urn:li:dataPlatform:postgres,transactions.customer_orders,PROD)) + - Description: Historical customer purchase transactions + - Platform: PostgreSQL + - Domain: Finance + +3. **customer_360** (urn:li:dataset:(urn:li:dataPlatform:bigquery,analytics.customer_360,PROD)) + - Description: Unified customer view combining profile, transactions, and interactions + - Platform: BigQuery + - Domain: Analytics + +You can view these in DataHub at: https://your-datahub.acryl.io +``` + +## Advanced Usage + +### Using with Memory + +Add conversation memory to your agent: + +```python +from langchain.memory import ConversationBufferMemory + +memory = ConversationBufferMemory( + memory_key="chat_history", + return_messages=True +) + +agent_executor = AgentExecutor( + agent=agent, + tools=tools, + memory=memory, + verbose=True +) +``` + +### Filtering Tools + +Include only specific tool categories: + +```python +from datahub_agent_context.langchain_tools import ( + build_search_tools, + build_entity_tools, + build_lineage_tools +) + +# Only search and entity tools +search_tools = build_search_tools(client) +entity_tools = build_entity_tools(client) +tools = search_tools + entity_tools +``` + +### Custom Tool Configuration + +```python +# Configure specific tools +tools = build_langchain_tools( + client, + include_mutations=False, + max_search_results=20, # Increase search result limit + max_lineage_depth=5 # Deeper lineage traversal +) +``` + +## More Examples + +Complete examples are available in the datahub-project repo: + +- [Basic Agent](https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/examples/langchain/basic_agent.py) +- [Simple Search](https://github.com/datahub-project/datahub/blob/master/datahub-agent-context/examples/langchain/simple_search.py) + +## Troubleshooting + +### Tool Execution Errors + +**Problem**: Agent tries to use tools but gets errors + +**Solutions**: + +- Verify DataHub connection: Test `client.config` is correctly set +- Check tool availability: Print `[tool.name for tool in tools]` to see available tools +- Enable verbose mode: Set `verbose=True` in AgentExecutor to see tool calls +- Validate token permissions: Ensure token has read access (and write access if using mutations) + +### Agent Not Using Tools + +**Problem**: Agent responds without calling DataHub tools + +**Solutions**: + +- Improve system prompt: Explicitly instruct agent to use tools +- Use better examples: Add few-shot examples of tool usage to prompt +- Check model compatibility: Ensure LLM supports tool/function calling (GPT-4, GPT-3.5-turbo, Claude 3+) +- Reduce temperature: Set `temperature=0` for more deterministic tool usage + +### Performance Issues + +**Problem**: Agent responses are slow + +**Solutions**: + +- Limit search results: Reduce `max_search_results` in tool configuration +- Use streaming: Enable streaming responses with `stream=True` +- Cache results: Implement caching for frequently accessed metadata +- Use faster models: Consider gpt-3.5-turbo for simpler queries + +### Import Errors + +**Problem**: `ModuleNotFoundError` for LangChain components + +**Solutions**: + +```bash +# Install all required dependencies +pip install datahub-agent-context[langchain] +pip install langchain langchain-openai + +# Or install from scratch +pip install datahub-agent-context langchain langchain-openai langchain-community +``` + +### Getting Help + +- **LangChain Docs**: [LangChain Agent Documentation](https://python.langchain.com/docs/modules/agents/) +- **DataHub Agent Context**: [Agent Context Kit Guide](./agent-context.md) +- **GitHub Issues**: [Report issues](https://github.com/datahub-project/datahub/issues) +- **Community Slack**: [Join DataHub Slack](https://datahub.com/slack/) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/snowflake.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/snowflake.md new file mode 100644 index 00000000..fdf21492 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/agent-context/snowflake.md @@ -0,0 +1,372 @@ +--- +title: Snowflake Intelligence Integration +slug: /dev-guides/agent-context/snowflake +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/agent-context/snowflake.md +--- +# Snowflake Intelligence Integration + +> **📚 Navigation**: [← Back to Agent Context Kit](./agent-context.md) | [← LangChain Integration](./langchain.md) | [Copilot Studio Integration →](./copilot-studio.md) + +## What Problem Does This Solve? + +Snowflake Intelligence provides powerful text-to-SQL capabilities, but it only sees raw table and column names. Without business context, your Snowflake Intelligence agents: + +- ❌ Can't distinguish `customer_revenue` from `customer_revenue_archive` +- ❌ Don't understand business glossary terms like "churn" or "LTV" +- ❌ Can't discover "all tables about customers" across schemas +- ❌ Don't know which datasets are certified vs deprecated +- ❌ Have no context about data ownership or documentation + +**DataHub's Snowflake Context Connector** solves this by providing Snowflake Intelligence with access to your DataHub metadata through User-Defined Functions (UDFs). This enables: + +### What You Can Do + +- ✅ **Semantic Search**: "Find all revenue tables owned by finance" +- ✅ **Business Context**: "What's the business definition of 'churn'?" +- ✅ **Quality Signals**: "Show me certified customer datasets" +- ✅ **Documentation Access**: Search across data docs and descriptions +- ✅ **Ownership Info**: Discover who owns and maintains datasets + +### Example Queries DataHub Enables + +```sql +-- Agent can now answer: +"Find all tables with customer data in the marketing domain" +"What datasets are tagged as PII?" +"Show me the business definition from the glossary for MRR" +"Which tables does the finance team own?" +``` + +## Overview + +DataHub's Snowflake context connector allows creating [Snowflake Intelligence](https://www.snowflake.com/en/developers/guides/getting-started-with-snowflake-intelligence) agents with UDFs that can access DataHub Context to power text-to-SQL generation. The connector creates an integration with DataHub that can search DataHub for documents and assets, as well as read Snowflake data tables and generate queries to help answer questions. + +This guide and agent creation tool will create an end-to-end experience for setting up a Snowflake Intelligence agent that can be used in Snowflake Intelligence. + +## Setup + +### Install the add-on to acryl-datahub package + +```bash +pip install datahub-agent-context[snowflake] +``` + +### Permissions + +#### DataHub + +In order to interact with DataHub, you will need the following from your DataHub account + +- The URL of your DataHub Cloud instance e.g. `https://.acryl.io` +- A [personal access token](../../authentication/personal-access-tokens.md) + +#### Snowflake + +In order to execute the SQL in Snowflake, you will need a user with the `SNOWFLAKE_ADMIN` role to configure the rules and UDFs. This is necessary to set up the secrets and networking options. Once the initial setup is completed, the `SNOWFLAKE_INTELLIGENCE_ADMIN` role is required to do edits and further configuration. + +### Generate or Execute SQL for creating Snowflake Agent + +In order to use DataHub tools in Snowflake intelligence, we'll create a new agent that has access to UDFs that can use the context inside of DataHub to answer using business questions. We can either run the SQL to register the agent and tools ourselves, or let the DataHub Agents CLI execute it for us. + +#### Execute SQL for Snowflake directly + +Add `--execute` to automatically execute the SQL as your Snowflake user. + +Authentication: + +- Use `--sf-password` to automatically run the generated scripts with your password or PAT. +- SSO support using an external browser with `--sf-authenticator externalBrowser`. + +```bash +datahub agent create snowflake \ + --sf-account YOUR_ACCOUNT \ + --sf-user YOUR_USER \ + --sf-password YOUR_PASSWORD \ + --sf-role YOUR_ROLE \ + --sf-warehouse YOUR_WAREHOUSE \ + --sf-database YOUR_DATABASE \ + --sf-schema YOUR_SCHEMA \ + --datahub-url https://your-datahub.acryl.io \ + --datahub-token YOUR_TOKEN \ + --enable-mutations \ + --execute +``` + +This will automatically execute the commands in your Snowflake environment and output the results. This workflow is recommended for a hands off default experience. + +

+ +

+ +#### Generate SQL for Snowflake + +```bash +datahub agent create snowflake \ + --sf-account YOUR_ACCOUNT \ + --sf-user YOUR_USER \ + --sf-role YOUR_ROLE \ + --sf-warehouse YOUR_WAREHOUSE \ + --sf-database YOUR_DATABASE \ + --sf-schema YOUR_SCHEMA \ + --datahub-url https://your-datahub.acryl.io \ + --datahub-token YOUR_TOKEN \ + --enable-mutations +``` + +In this version you will need to execute 5 SQL files in your snowflake UI as a notebook. This is recommended for advanced workflows or if you want to review or make changes to the configuration before publishing. These configuration files can be pushed into version control if desired to maintain history. + +```sql +-- 1. Set up configuration and secrets +@00_configuration.sql; + +-- 2. Create network rules +@01_network_rules.sql; + +-- 3. Create DataHub UDFs +@02_datahub_udfs.sql; + +-- 4. Create stored procedure +@03_stored_procedure.sql; + +-- 5. Create Cortex Agent +@04_cortex_agent.sql; + +``` + +

+ +

+ +### Configure Snowflake agent in the UI + +Once your agent is created, you can further customize it's prompt settings, models, tools, & more inside the Snowflake user interface. It is recommended to make sure your prompt and model is tweaked for your specific use case and requirements. + +

+ +

+ +### Using your DataHub Agent + +Open [Snowflake Intelligence](https://ai.snowflake.com/) and select the DataHub Agent. + +

+ +

+ +## Updating UDFs + +Periodically we expect to add new tools and update existing tools. To update the tools for your DataHub Agent, simply run the following SQL snippets to update the tool definitions and SDK: + +```sql +-- 1. Create updated DataHub UDFs +@02_datahub_udfs.sql; + +-- 2. Create updated Cortex Agent +@04_cortex_agent.sql; +``` + +After running these updates, refresh your Snowflake Intelligence UI to see the new tools. + +## Agent Context + +For more info on the tools exposed and DataHub Context, see the [DataHub Agent Context Documentation](./agent-context.md). + +## Troubleshooting + +### Setup Issues + +#### Problem: Permission Denied Errors + +**Symptoms**: `Insufficient privileges to operate on database/schema` errors during setup + +**Solutions**: + +- Verify you're using the `ACCOUNTADMIN` or `SECURITYADMIN` role for initial setup +- Check that your user has `CREATE DATABASE` and `CREATE INTEGRATION` privileges +- Ensure the role has `USAGE` on the warehouse +- Run: `SHOW GRANTS TO USER ;` to verify permissions + +#### Problem: Network Rule Creation Fails + +**Symptoms**: Cannot create network rules for DataHub connection + +**Solutions**: + +- Verify your Snowflake account allows external network access +- Check that the DataHub URL is accessible from your network +- Ensure you're using the correct DataHub URL format: `https://your-instance.acryl.io` +- Contact your Snowflake account admin if external network access is restricted + +#### Problem: Secret Creation Fails + +**Symptoms**: Cannot create secret for DataHub token + +**Solutions**: + +- Verify the DataHub token is valid and not expired +- Check that the token format is correct (should start with `eyJ`) +- Ensure the secret name doesn't conflict with existing secrets +- Run: `SHOW SECRETS IN SCHEMA ;` to check existing secrets + +### Agent Execution Issues + +#### Problem: Agent Can't Find DataHub Tools + +**Symptoms**: Agent doesn't use DataHub UDFs or says "function not found" + +**Solutions**: + +- Verify UDFs were created: `SHOW USER FUNCTIONS IN SCHEMA ;` +- Check agent configuration includes DataHub tools +- Refresh the Snowflake Intelligence UI +- Recreate the agent using the latest `@04_cortex_agent.sql` + +#### Problem: UDF Execution Errors + +**Symptoms**: UDF calls fail with timeout or connection errors + +**Solutions**: + +- Verify network rules are configured correctly +- Test DataHub connectivity: Try calling a UDF directly in a SQL worksheet +- Check DataHub server is accessible and running +- Verify the DataHub token hasn't expired +- Review Snowflake query history for detailed error messages + +#### Problem: Empty or No Results from DataHub + +**Symptoms**: Agent says "no data found" even though data exists in DataHub + +**Solutions**: + +- Verify token has correct permissions in DataHub +- Check that entities exist and are searchable in DataHub UI +- Try the search directly in DataHub UI with the same query +- Verify the entity types exist (dataset, dashboard, etc.) +- Check DataHub search index is up to date + +### Agent Quality Issues + +#### Problem: Agent Doesn't Use DataHub Context + +**Symptoms**: Agent generates SQL without consulting DataHub metadata + +**Solutions**: + +- Update the agent system prompt to explicitly mention using DataHub tools +- Configure agent to require tool usage before generating SQL +- Add examples of DataHub tool usage to the agent prompt +- Verify tools are properly registered in the agent configuration + +#### Problem: Agent Hallucinates Table Names + +**Symptoms**: Agent suggests tables that don't exist + +**Solutions**: + +- Ensure agent is configured to search DataHub before querying Snowflake +- Update prompt to require validation of table names via DataHub +- Limit agent to only use tables found in DataHub search results +- Add a validation step that checks table existence + +### CLI Issues + +#### Problem: `datahub agent create snowflake` Command Not Found + +**Solutions**: + +```bash +# Ensure package is installed with CLI +pip install --upgrade 'acryl-datahub[cli]' +pip install datahub-agent-context[snowflake] + +# Verify installation +datahub version +datahub agent --help +``` + +#### Problem: Authentication Fails with `--execute` + +**Solutions**: + +- For password auth: Ensure password is properly quoted if it contains special characters +- For SSO: Use `--sf-authenticator externalbrowser` and ensure browser can open +- For key-pair auth: Use `--sf-private-key-path` with your key file +- Test connection manually: `snowsql -a -u ` + +### Performance Issues + +#### Problem: Slow Agent Responses + +**Solutions**: + +- Ensure warehouse size is appropriate for workload (start with SMALL or MEDIUM) +- Check DataHub API response times in DataHub logs +- Optimize DataHub searches by adding specific entity type filters +- Consider caching frequently accessed metadata +- Review Snowflake query profile for bottlenecks + +### Debugging Tips + +#### Enable Verbose Logging + +```bash +# When using --execute, add verbose flag +datahub agent create snowflake \ + --execute \ + --verbose \ + ...other flags... +``` + +#### Test UDF Directly + +```sql +-- Test search UDF +SELECT datahub_search('customer', {'entity_type': ['dataset']}, 10); + +-- Test document search +SELECT datahub_search_documents('data retention policy', 5); + +-- Check UDF definitions +SHOW USER FUNCTIONS LIKE 'datahub%'; +``` + +#### Check Agent Logs + +In Snowflake Intelligence UI: + +1. Open the agent conversation +2. Click "View Details" on a response +3. Check the "Tool Calls" section to see which UDFs were called +4. Review execution times and errors + +### Common Error Messages + +| Error | Cause | Solution | +| ------------------------- | ------------------------ | ----------------------------------------------------- | +| `Network rule violation` | DataHub URL not allowed | Update network rules to include DataHub domain | +| `Secret not found` | Secret name mismatch | Verify secret name matches in UDF and creation script | +| `Insufficient privileges` | Missing permissions | Grant required roles and privileges | +| `Invalid access token` | Token expired or invalid | Regenerate DataHub token and update secret | +| `Function does not exist` | UDF not created | Run `@02_datahub_udfs.sql` to create UDFs | + +### Getting Help + +- **DataHub Documentation**: [Agent Context Kit](./agent-context.md) +- **Snowflake Intelligence Docs**: [Getting Started Guide](https://www.snowflake.com/en/developers/guides/getting-started-with-snowflake-intelligence) +- **Community Support**: [DataHub Slack](https://datahub.com/slack/) +- **GitHub Issues**: [Report bugs](https://github.com/datahub-project/datahub/issues) +- **DataHub Cloud Support**: Email support@acryl.io + +### Verification Checklist + +After setup, verify everything works: + +- [ ] UDFs created successfully: `SHOW USER FUNCTIONS LIKE 'datahub%';` +- [ ] Network rules configured: `SHOW NETWORK RULES;` +- [ ] Secrets created: `SHOW SECRETS IN SCHEMA ;` +- [ ] Agent registered: Check Snowflake Intelligence UI +- [ ] Test UDF directly: `SELECT datahub_search('test', {}, 5);` +- [ ] Test agent: Ask a simple question in Snowflake Intelligence +- [ ] Verify DataHub tools appear in agent configuration diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/ARCHITECTURE.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/ARCHITECTURE.md new file mode 100644 index 00000000..b85de0ed --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/ARCHITECTURE.md @@ -0,0 +1,446 @@ +--- +title: Semantic Search Architecture +slug: /dev-guides/semantic-search/architecture +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/semantic-search/ARCHITECTURE.md +--- +# Semantic Search Architecture + +This document provides a detailed explanation of DataHub's semantic search architecture, design decisions, and implementation details. + +## Design Philosophy + +### Why Semantic Search? + +Traditional keyword search has limitations: + +1. **Vocabulary Mismatch**: Users may use different terms than those in documents +2. **Synonym Blindness**: "access request" won't match "permission request" +3. **Context Ignorance**: Keywords lack understanding of meaning + +Semantic search addresses these by understanding the _meaning_ of text through vector embeddings—numerical representations that capture semantic similarity. + +### Core Principles + +1. **Non-invasive**: Semantic search is additive; it doesn't replace keyword search +2. **Configurable**: Organizations choose which entities and models to use +3. **Extensible**: New embedding models can be added without architectural changes +4. **Async Processing**: Embedding generation happens asynchronously to not block ingestion + +## Index Architecture + +### Dual-Index Strategy + +For each entity type enabled for semantic search, two indices exist: + +``` +┌─────────────────────────────────┐ ┌─────────────────────────────────┐ +│ documentindex_v2 │ │ documentindex_v2_semantic │ +├─────────────────────────────────┤ ├─────────────────────────────────┤ +│ Standard OpenSearch index │ │ OpenSearch index with k-NN │ +│ │ │ │ +│ Fields: │ │ Fields: │ +│ - urn │ │ - urn │ +│ - title (text) │ │ - title (text) │ +│ - text (text) │ │ - text (text) │ +│ - browsePaths │ │ - browsePaths │ +│ - tags │ │ - tags │ +│ - ... │ │ - ... │ +│ │ │ │ +│ │ │ + embeddings (nested object): │ +│ │ │ - cohere_embed_v3: │ +│ │ │ - model_version │ +│ │ │ - generated_at │ +│ │ │ - chunks[] (nested): │ +│ │ │ - position │ +│ │ │ - text │ +│ │ │ - vector (knn_vector) │ +└─────────────────────────────────┘ └─────────────────────────────────┘ +``` + +### Why Separate Indices? (Transitional Architecture) + +The dual-index approach is a **transitional architecture**. The long-term plan is to: + +1. **Phase 1 (Current)**: Run both indices in parallel during transition +2. **Phase 2**: Migrate all search traffic to semantic indices +3. **Phase 3**: Retire `v2` indices entirely + +**Benefits of the transitional approach:** + +1. **Zero Downtime Migration**: Users continue using keyword search while semantic capabilities are built +2. **Gradual Validation**: Semantic search quality can be validated before full rollout +3. **Rollback Safety**: If issues arise, keyword search remains available +4. **Incremental Embedding Generation**: Embeddings can be backfilled without blocking operations + +**Future State:** + +Once the transition is complete, the `_semantic` indices will become the primary (and only) search indices. They will support both: + +- **Keyword search**: Using standard OpenSearch text matching on the same index +- **Semantic search**: Using k-NN vector similarity + +This unified index approach simplifies operations and reduces storage overhead. + +### Embeddings Schema + +The semantic index stores embeddings in a nested structure: + +```json +{ + "urn": "urn:li:document:example-doc", + "title": "Data Access Guide", + "text": "How to request access to datasets...", + "embeddings": { + "cohere_embed_v3": { + "model_version": "bedrock/cohere.embed-english-v3", + "generated_at": "2024-01-15T10:30:00Z", + "chunking_strategy": "sentence_boundary_400t", + "total_chunks": 3, + "total_tokens": 850, + "chunks": [ + { + "position": 0, + "text": "How to request access to datasets...", + "character_offset": 0, + "character_length": 450, + "token_count": 95, + "vector": [0.023, -0.041, 0.087, ...] // 1024 dimensions + }, + { + "position": 1, + "text": "For sensitive data, additional approval...", + "character_offset": 450, + "character_length": 380, + "token_count": 82, + "vector": [0.019, -0.055, 0.091, ...] + } + ] + } + } +} +``` + +### Multi-Model Support + +The embeddings structure supports multiple embedding models: + +```json +{ + "embeddings": { + "cohere_embed_v3": { ... }, + "openai_text_embedding_3": { ... }, + "custom_model": { ... } + } +} +``` + +This allows: + +- A/B testing different models +- Gradual migration between models +- Model-specific optimizations + +## Data Flow + +### Ingestion Flow + +The ingestion connector generates document embeddings and sends them to GMS along with the document content: + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ Ingestion Flow │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌─────────────┐ │ +│ │ Source │ 1. Extract documents │ +│ │ System │ │ +│ └──────┬──────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────┐ │ +│ │ Ingestion │ 2. Generate embeddings for document content │ +│ │ Connector │ (using connector's embedding provider) │ +│ └──────┬──────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────┐ 3. Send document + embeddings to GMS │ +│ │ GMS │ │ +│ └──────┬──────┘ │ +│ │ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────────────────────────┐ │ +│ │ OpenSearch │ │ +│ │ ┌─────────────────────┐ ┌─────────────────────────────────┐ │ │ +│ │ │ entityindex_v2 │ │ entityindex_v2_semantic │ │ │ +│ │ │ (keyword search) │ │ (keyword + vector search) │ │ │ +│ │ │ │ │ │ │ │ +│ │ │ - urn │ │ - urn │ │ │ +│ │ │ - title │ │ - title │ │ │ +│ │ │ - text │ │ - text │ │ │ +│ │ │ - ... │ │ - embeddings.model.chunks[]. │ │ │ +│ │ │ │ │ vector │ │ │ +│ │ └─────────────────────┘ └─────────────────────────────────┘ │ │ +│ └─────────────────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### Embedding Generation + +**Document Embeddings** are generated by the **ingestion connector** at ingestion time and sent to GMS via MCP (Metadata Change Proposal). This ensures: + +1. **Consistency**: Every ingested document has embeddings from the start +2. **Simplicity**: No separate backfill job to manage +3. **Freshness**: Embeddings are always up-to-date with document content +4. **Audit Trail**: Embeddings are tracked in the Metadata Change Log (MCL) +5. **Privacy Support**: Sensitive sources can generate embeddings locally and share only vectors + +#### MCP-Based Embedding Flow + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Ingestion Pipeline │ +├─────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────┐ │ +│ │ Source │───▶│ Ingestion │───▶│ DataHub GMS │ │ +│ │ System │ │ Connector │ │ │ │ +│ └──────────────┘ └──────┬───────┘ └──────────┬───────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌──────────────────┐ ┌──────────────────┐ │ +│ │ Generate document│ │ Process MCP and │ │ +│ │ embeddings │ │ write to semantic │ │ +│ │ (in connector) │ │ search index │ │ +│ └────────┬─────────┘ └──────────────────┘ │ +│ │ ▲ │ +│ │ MCP with │ │ +│ └─────SemanticContent─┘ │ +│ aspect │ +│ │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +#### SemanticContent Aspect + +Embeddings are stored as a proper DataHub aspect (`SemanticContent`), defined in PDL schema: + +```json +{ + "entityType": "document", + "entityUrn": "urn:li:document:my-doc", + "aspectName": "semanticContent", + "aspect": { + "embeddings": { + "cohere_embed_v3": { + "modelVersion": "bedrock/cohere.embed-english-v3", + "generatedAt": 1702234567890, + "totalChunks": 2, + "chunks": [ + { "position": 0, "vector": [...], "text": "..." }, + { "position": 1, "vector": [...], "text": "..." } + ] + } + } + } +} +``` + +#### Privacy-Sensitive Use Cases + +The `text` field in each chunk is **optional**. This supports scenarios where: + +- Source data contains sensitive information (PII, trade secrets) +- Customers want semantic search without storing source text in DataHub +- Embeddings are generated locally at the data source + +**Note:** Embeddings are one-way—original text cannot be reconstructed from vectors. + +**Query Embeddings** are generated by GMS at search time using the configured embedding provider (e.g., AWS Bedrock): + +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ GraphQL │───▶│ GMS │───▶│ Embedding │───▶│ OpenSearch │ +│ Client │ │ │ │ Provider │ │ k-NN Query │ +└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ + │ + ▼ + Query embedding + generated here + (for search only) +``` + +**Key Point:** The GMS embedding provider is used **only for query embedding**, not for document embedding. The ingestion connector is responsible for document embeddings. + +### Query Flow + +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ GraphQL │───▶│ GMS │───▶│ Embedding │───▶│ OpenSearch │ +│ Client │ │ │ │ Provider │ │ k-NN Query │ +└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘ + │ + semanticSearchAcrossEntities( │ + query: "how to access data" │ + ) │ + ▼ + ┌─────────────────────────────┐ + │ Nested k-NN Query: │ + │ │ + │ { │ + │ "nested": { │ + │ "path": "embeddings │ + │ .cohere_embed_v3 │ + │ .chunks", │ + │ "query": { │ + │ "knn": { │ + │ "...chunks.vector":│ + │ { "vector": [...], │ + │ "k": 10 } │ + │ } │ + │ } │ + │ } │ + │ } │ + └─────────────────────────────┘ +``` + +## Chunking Strategy + +### Why Chunk Documents? + +Embedding models have token limits (512 tokens for cohere's embed-english-v3.0). Long documents must be split into chunks: + +1. **Token Limits**: Models can't process unlimited text +2. **Precision**: Smaller chunks allow more precise matching +3. **Relevance**: A document may have one highly relevant section + +### Chunking Algorithm + +```python +def chunk_text(text, max_tokens=400): + """ + Chunk text at sentence boundaries, respecting token limits. + + 1. Split text into sentences + 2. Accumulate sentences until approaching limit + 3. Save chunk, start new accumulation + 4. Handle oversized sentences by character splitting + """ +``` + +**Parameters:** + +- `max_tokens`: Target chunk size (default: 400) +- `chars_per_token`: Estimation ratio (default: 4 characters ≈ 1 token) + +### Chunk Metadata + +Each chunk stores metadata for debugging and analysis: + +```json +{ + "position": 0, // Order in document + "text": "...", // Chunk content + "character_offset": 0, // Start position in original + "character_length": 450, // Length in characters + "token_count": 95, // Estimated tokens + "vector": [...] // Embedding vector +} +``` + +## k-NN Search Configuration + +### OpenSearch k-NN Settings + +The semantic index uses OpenSearch's k-NN plugin with FAISS engine: + +```json +{ + "settings": { + "index.knn": true + }, + "mappings": { + "properties": { + "embeddings": { + "type": "nested", + "properties": { + "cohere_embed_v3": { + "type": "nested", + "properties": { + "chunks": { + "type": "nested", + "properties": { + "vector": { + "type": "knn_vector", + "dimension": 1024, + "method": { + "name": "hnsw", + "engine": "faiss", + "space_type": "cosinesimil", + "parameters": { + "ef_construction": 128, + "m": 16 + } + } + } + } + } + } + } + } + } + } + } +} +``` + +### HNSW Parameters + +| Parameter | Value | Description | +| ----------------- | ----------- | -------------------------------------------------------------------- | +| `ef_construction` | 128 | Build-time accuracy (higher = more accurate, slower build) | +| `m` | 16 | Number of connections per node (higher = more accurate, more memory) | +| `space_type` | cosinesimil | Similarity metric (cosine similarity) | + +## Security Considerations + +### Data Privacy + +1. **Embedding Storage**: Vectors are stored alongside documents; same access controls apply +2. **External API Calls**: Embedding providers receive document text; ensure compliance +3. **Credential Management**: API keys/AWS credentials must be secured + +### Access Control + +Semantic search respects DataHub's existing access controls: + +- Users only see results they have permission to view +- Entity-level permissions are enforced before returning results + +## Performance Considerations + +### Indexing Performance + +- **Dual-write Impact**: ~10-20% increase in write latency +- **Embedding Generation**: Async; doesn't block ingestion +- **Batch Processing**: Embeddings generated in batches for efficiency + +### Query Performance + +- **k-NN Overhead**: ~50-200ms per query (depends on index size) +- **Embedding Generation**: ~100-300ms for query embedding +- **Total Latency**: Typically 200-500ms end-to-end + +### Scaling Recommendations + +| Index Size | Recommendation | +| -------------- | --------------------------------- | +| < 100K docs | Single node sufficient | +| 100K - 1M docs | Consider dedicated k-NN nodes | +| > 1M docs | Sharding and replicas recommended | + +## Future Enhancements + +1. **Hybrid Search**: Combine keyword and semantic scores for improved relevance +2. **Model Fine-tuning**: Domain-specific embedding models for better accuracy diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/CONFIGURATION.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/CONFIGURATION.md new file mode 100644 index 00000000..c99b9678 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/CONFIGURATION.md @@ -0,0 +1,545 @@ +--- +title: Semantic Search Configuration Guide +slug: /dev-guides/semantic-search/configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/semantic-search/CONFIGURATION.md +--- +# Semantic Search Configuration Guide + +This guide covers all configuration options for DataHub's semantic search, including embedding models, index settings, and environment variables. + +## Enabling Semantic Search + +### Environment Variables + +Set these in your deployment configuration (e.g., `docker/profiles/empty2.env`): + +```bash +# Enable semantic search feature +ELASTICSEARCH_SEMANTIC_SEARCH_ENABLED=true + +# Entity types to enable (comma-separated) +ELASTICSEARCH_SEMANTIC_SEARCH_ENTITIES=document + +# Vector dimensions (must match embedding model) +ELASTICSEARCH_SEMANTIC_VECTOR_DIMENSION=3072 +``` + +### Application Configuration + +In `metadata-service/configuration/src/main/resources/application.yaml`, you need to configure: + +1. **Semantic search settings** - Enable the feature and specify which entities support it +2. **Index model mappings** - Define the embedding model(s) and their vector dimensions +3. **Embedding provider** - Configure credentials for generating query embeddings + +#### Index Model Mappings + +The `models:` section defines the structure of the semantic index. **You must add a mapping for each embedding model you plan to use.** The model key (e.g., `text_embedding_3_large`) must match the key used when ingesting document embeddings. + +```yaml +elasticsearch: + entityIndex: + semanticSearch: + enabled: ${ELASTICSEARCH_SEMANTIC_SEARCH_ENABLED:false} + enabledEntities: ${ELASTICSEARCH_SEMANTIC_SEARCH_ENTITIES:document} + + # Define index mappings for each embedding model you use + models: + # Default: OpenAI text-embedding-3-large + text_embedding_3_large: + vectorDimension: 3072 + knnEngine: faiss + spaceType: cosinesimil + efConstruction: 128 + m: 16 +``` + +**Common model configurations:** + +| Model | Key | Dimensions | +| ----------------------------- | ------------------------ | ---------- | +| AWS Bedrock Cohere | `cohere_embed_v3` | 1024 | +| OpenAI text-embedding-3-small | `text_embedding_3_small` | 1536 | +| OpenAI text-embedding-3-large | `text_embedding_3_large` | 3072 | +| Cohere embed-english-v3.0 | `embed_english_v3_0` | 1024 | + +> **Important:** The model key is derived from the configured model ID using explicit mappings for known models, with a fallback that replaces dots, hyphens, and colons with underscores. For example: +> +> - `cohere.embed-english-v3` (AWS Bedrock, via `BEDROCK_EMBEDDING_MODEL`) → `cohere_embed_v3` +> - `embed-english-v3.0` (Cohere direct, via `COHERE_EMBEDDING_MODEL`) → `embed_english_v3_0` +> - `text-embedding-3-small` (OpenAI, via `OPENAI_EMBEDDING_MODEL`) → `text_embedding_3_small` +> +> The explicit mappings ensure AWS Bedrock Cohere models (`cohere.embed-english-v3`) and Cohere direct API models (`embed-english-v3.0`) produce distinct keys despite similar names. + +> **Matching with Ingestion:** The model key in the index mapping must match the key used in the `semanticContent` aspect when documents are ingested. The ingestion connector (e.g., `datahub-documents`) uses the configured model to determine this key. For example, if your index has `text_embedding_3_small`, the ingested `semanticContent` aspect must have embeddings under `embeddings.text_embedding_3_small`. + +#### Embedding Provider Configuration + +Configure the provider used to generate query embeddings at search time: + +```yaml +elasticsearch: + entityIndex: + semanticSearch: + # ... models section above ... + + embeddingProvider: + type: ${EMBEDDING_PROVIDER_TYPE:openai} + maxCharacterLength: ${EMBEDDING_PROVIDER_MAX_CHAR_LENGTH:2048} + bedrock: + awsRegion: ${BEDROCK_EMBEDDING_AWS_REGION:us-west-2} + model: ${BEDROCK_EMBEDDING_MODEL:cohere.embed-english-v3} + + # OpenAI configuration (used when type is "openai") + openai: + apiKey: ${OPENAI_API_KEY:} + model: ${OPENAI_EMBEDDING_MODEL:text-embedding-3-large} + endpoint: ${OPENAI_EMBEDDING_ENDPOINT:https://api.openai.com/v1/embeddings} + + # Cohere configuration (used when type is "cohere") + cohere: + apiKey: ${COHERE_API_KEY:} + model: ${COHERE_EMBEDDING_MODEL:embed-english-v3.0} + endpoint: ${COHERE_EMBEDDING_ENDPOINT:https://api.cohere.ai/v1/embed} +``` + +## Embedding Models + +### Understanding Embedding Providers + +There are **two separate embedding contexts** in DataHub's semantic search: + +| Context | When | Who Generates | Configuration | +| ----------------------- | ----------------- | ------------------- | ------------------------- | +| **Document Embeddings** | At ingestion time | Ingestion Connector | Configured in connector | +| **Query Embeddings** | At search time | GMS | Configured in GMS (below) | + +> **Important:** The GMS embedding provider (configured below) is used **only for query embedding**. +> Document embeddings are generated by the ingestion connector using its own embedding configuration. +> Both must use the **same embedding model** for semantic search to work correctly. + +### Ingestion via MCP (Metadata Change Proposal) + +Document embeddings are sent to DataHub via MCP using the `SemanticContent` aspect. This is the standard DataHub pattern for ingesting metadata. + +**MCP Payload Structure:** + +```json +{ + "entityType": "document", + "entityUrn": "urn:li:document:my-doc-123", + "aspectName": "semanticContent", + "aspect": { + "value": "{...JSON encoded SemanticContent...}", + "contentType": "application/json" + } +} +``` + +**SemanticContent Aspect:** + +```json +{ + "embeddings": { + "text_embedding_3_large": { + "modelVersion": "openai/text-embedding-3-large", + "generatedAt": 1702234567890, + "chunkingStrategy": "sentence_boundary_400t", + "totalChunks": 2, + "totalTokens": 450, + "chunks": [ + { + "position": 0, + "vector": [0.123, -0.456, ...], + "characterOffset": 0, + "characterLength": 512, + "tokenCount": 230, + "text": "First chunk of text..." + } + ] + } + } +} +``` + +**Privacy Option:** The `text` field in each chunk is optional. For sensitive data sources, you can omit the source text and send only the embedding vectors. + +### GMS Query Embedding Provider (Java) + +GMS uses an `EmbeddingProvider` implementation to generate query embeddings at search time. + +**Interface:** `com.linkedin.metadata.search.embedding.EmbeddingProvider` + +```java +public interface EmbeddingProvider { + /** + * Returns an embedding vector for the given text. + * @param text The text to embed + * @param model The model identifier (nullable, uses default if null) + * @return The embedding vector + */ + @Nonnull + float[] embed(@Nonnull String text, @Nullable String model); +} +``` + +**Built-in Implementations:** + +- `OpenAiEmbeddingProvider` - Uses OpenAI API (default) +- `AwsBedrockEmbeddingProvider` - Uses AWS Bedrock +- `NoOpEmbeddingProvider` - Throws exception if called (used when semantic search disabled) + +The following providers can be configured: + +#### AWS Bedrock + +AWS Bedrock provides managed access to embedding models: + +```bash +# Environment +EMBED_PROVIDER=bedrock +BEDROCK_EMBEDDING_MODEL=cohere.embed-english-v3 +AWS_REGION=us-west-2 + +# For local development +AWS_PROFILE=your-profile + +# For production (IAM role or explicit credentials) +AWS_ACCESS_KEY_ID=... +AWS_SECRET_ACCESS_KEY=... +``` + +**Available Bedrock Models:** + +| Model ID | Dimensions | Max Tokens | Notes | +| ------------------------------ | ------------ | ---------- | ----------------------- | +| `cohere.embed-english-v3` | 1024 | 512 | Best for English | +| `cohere.embed-multilingual-v3` | 1024 | 512 | Multi-language support | +| `amazon.titan-embed-text-v1` | 1536 | 8192 | Amazon's model | +| `amazon.titan-embed-text-v2:0` | 256/512/1024 | 8192 | Configurable dimensions | + +#### OpenAI (Default) + +OpenAI provides high-quality embedding models with simple API access: + +```bash +# Required +EMBEDDING_PROVIDER_TYPE=openai +OPENAI_API_KEY=sk-your-api-key-here + +# Optional - defaults shown +# OPENAI_EMBEDDING_MODEL: Model used to generate query embeddings via OpenAI API +OPENAI_EMBEDDING_MODEL=text-embedding-3-large +``` + +**Available OpenAI Models:** + +| Model ID | Dimensions | Max Tokens | Notes | +| ------------------------ | ---------- | ---------- | -------------------- | +| `text-embedding-3-small` | 1536 | 8191 | Fast, cost-effective | +| `text-embedding-3-large` | 3072 | 8191 | Higher quality | + +#### Cohere (Direct API) + +Use Cohere's embedding API directly (without AWS Bedrock): + +```bash +# Required +EMBEDDING_PROVIDER_TYPE=cohere +COHERE_API_KEY=your-cohere-api-key + +# Optional - defaults shown +# COHERE_EMBEDDING_MODEL: Model used to generate query embeddings via Cohere API +COHERE_EMBEDDING_MODEL=embed-english-v3.0 +``` + +**Available Cohere Models:** + +| Model ID | Dimensions | Notes | +| ------------------------- | ---------- | ----------------- | +| `embed-english-v3.0` | 1024 | English optimized | +| `embed-multilingual-v3.0` | 1024 | 100+ languages | + +#### Switching Between Providers + +When switching embedding providers, you must delete and recreate the semantic index because different models produce vectors with different dimensions. + +See **[SWITCHING_PROVIDERS.md](./SWITCHING_PROVIDERS.md)** for detailed step-by-step instructions. + +#### Future Providers + +Potential future built-in providers: + +- Azure OpenAI +- Google Vertex AI + +#### Custom/Self-Hosted Providers + +To add a custom embedding provider, implement the `EmbeddingProvider` interface and register it in the factory. + +**1. Implement the interface** (`metadata-io/src/main/java/com/linkedin/metadata/search/embedding/`): + +```java +package com.linkedin.metadata.search.embedding; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class CustomEmbeddingProvider implements EmbeddingProvider { + + private final String endpoint; + private final int dimensions; + + public CustomEmbeddingProvider(String endpoint, int dimensions) { + this.endpoint = endpoint; + this.dimensions = dimensions; + } + + @Override + @Nonnull + public float[] embed(@Nonnull String text, @Nullable String model) { + // Call your embedding service + // Return float array with `dimensions` elements + return callEmbeddingService(endpoint, text, model); + } +} +``` + +**2. Register in the factory** (`metadata-service/factories/.../EmbeddingProviderFactory.java`): + +```java +@Bean(name = "embeddingProvider") +@Nonnull +protected EmbeddingProvider getInstance() { + // ... existing code ... + + String providerType = config.getType(); + + if ("aws-bedrock".equalsIgnoreCase(providerType)) { + return new AwsBedrockEmbeddingProvider(...); + } else if ("custom".equalsIgnoreCase(providerType)) { + return new CustomEmbeddingProvider( + config.getCustomEndpoint(), + config.getCustomDimensions() + ); + } else { + throw new IllegalStateException("Unsupported provider: " + providerType); + } +} +``` + +**3. Add configuration** (`EmbeddingProviderConfiguration.java`): + +```java +@Data +public class EmbeddingProviderConfiguration { + private String type = "aws-bedrock"; + private String awsRegion = "us-west-2"; + private String modelId = "cohere.embed-english-v3"; + + // Add custom provider fields + private String customEndpoint; + private int customDimensions = 1024; +} +``` + +**4. Configure in application.yaml**: + +```yaml +elasticsearch: + search: + semanticSearch: + embeddingProvider: + type: custom + customEndpoint: http://your-embedding-service:8080/embed + customDimensions: 1024 +``` + +## Index Configuration + +### Adding a Model to the Index + +Each embedding model you use must have a corresponding entry in the `models:` section of application.yaml. The index mapping is created when GMS starts, so you must configure this **before** ingesting documents. + +**Example: Adding OpenAI text-embedding-3-small:** + +```yaml +models: + text_embedding_3_small: # Key derived from model name (dots/hyphens → underscores) + vectorDimension: 1536 # Must match the model's output dimensions + knnEngine: faiss + spaceType: cosinesimil + efConstruction: 128 + m: 16 +``` + +### k-NN Settings + +The semantic index uses OpenSearch's k-NN plugin. Key parameters: + +```yaml +models: + your_model_key: + # Vector size (must match model output) + vectorDimension: 1024 + + # k-NN engine: faiss (recommended) or nmslib + knnEngine: faiss + + # Similarity metric + # - cosinesimil: Cosine similarity (recommended for text) + # - l2: Euclidean distance + # - innerproduct: Dot product + spaceType: cosinesimil + + # HNSW graph parameters + efConstruction: 128 # Build-time accuracy (32-512) + m: 16 # Connections per node (4-64) +``` + +### Parameter Tuning + +| Parameter | Low | Medium | High | Trade-off | +| ---------------- | --- | ------ | ---- | ------------------------------- | +| `efConstruction` | 32 | 128 | 512 | Speed vs accuracy at build time | +| `m` | 4 | 16 | 64 | Memory vs accuracy | + +**Recommendations:** + +- **Development**: `efConstruction: 64, m: 8` +- **Production**: `efConstruction: 128, m: 16` +- **High Accuracy**: `efConstruction: 256, m: 32` + +### Multiple Models (Advanced) + +You can configure multiple embedding models in the index for migration scenarios or A/B testing. This allows you to have documents with embeddings from different models coexisting in the same index. + +> **Note:** Most deployments only need a single model. Only add multiple models if you're migrating between providers or testing different models. + +```yaml +models: + # Current production model (OpenAI default) + text_embedding_3_large: + vectorDimension: 3072 + knnEngine: faiss + spaceType: cosinesimil + efConstruction: 128 + m: 16 + + # Alternative model being tested + cohere_embed_v3: + vectorDimension: 1024 + knnEngine: faiss + spaceType: cosinesimil + efConstruction: 128 + m: 16 +``` + +When using multiple models, the configured provider model (`BEDROCK_EMBEDDING_MODEL`, `OPENAI_EMBEDDING_MODEL`, or `COHERE_EMBEDDING_MODEL`) determines which model is used for query embeddings at search time. + +## Query Configuration + +### GMS Query Settings + +GMS needs credentials to generate query embeddings at search time: + +```bash +# Mount AWS credentials in Docker +volumes: + - ${HOME}/.aws:/home/datahub/.aws:ro + +# Set profile +AWS_PROFILE=your-profile +``` + +### Query Parameters + +In GraphQL queries: + +```graphql +query SemanticSearch($input: SearchAcrossEntitiesInput!) { + semanticSearchAcrossEntities(input: $input) { + # ... + } +} +``` + +Variables: + +```json +{ + "input": { + "query": "your natural language query", + "types": ["DOCUMENT"], // Entity types to search + "start": 0, // Pagination start + "count": 10 // Results per page + } +} +``` + +## Chunking Configuration + +Chunking is handled by the **ingestion connector** when generating document embeddings. The typical strategy: + +- **Chunk size**: ~400 tokens per chunk +- **Boundary**: Split at sentence boundaries when possible +- **Overlap**: Optional overlap between chunks for context continuity + +The chunking parameters are configured in the ingestion connector, not in GMS. GMS receives pre-chunked embeddings from the connector. + +## Monitoring + +### Useful Queries + +**Check embedding coverage:** + +```bash +curl "http://localhost:9200/documentindex_v2_semantic/_search" \ + -H "Content-Type: application/json" \ + -d '{ + "size": 0, + "aggs": { + "with_embeddings": { + "filter": { "exists": { "field": "embeddings.text_embedding_3_large" } } + }, + "without_embeddings": { + "filter": { "bool": { "must_not": { "exists": { "field": "embeddings.text_embedding_3_large" } } } } + } + } + }' +``` + +**Check index health:** + +```bash +curl "http://localhost:9200/_cat/indices/*semantic*?v" +``` + +## Troubleshooting + +### Common Issues + +| Issue | Cause | Solution | +| --------------------------------------- | ----------------------- | --------------------------------------------------------------- | +| "Unable to locate credentials" | AWS creds not available | Mount `.aws` to `/home/datahub/.aws` | +| "Profile file contained no credentials" | SSO session expired | Run `aws sso login --profile your-profile` | +| Empty search results | No embeddings in index | Verify ingestion connector is generating embeddings | +| Wrong results | Model mismatch | Ensure ingestion connector and GMS use the same embedding model | + +### Debug Logging + +Enable debug logging in GMS: + +```yaml +logging: + level: + com.linkedin.metadata.search: DEBUG +``` + +Check logs for: + +``` +[DEBUG-DUALWRITE] shouldWriteToSemanticIndex returned: true +Semantic dual-write enabled=true, enabledEntities=[document] +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/README.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/README.md new file mode 100644 index 00000000..84b92036 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/README.md @@ -0,0 +1,183 @@ +--- +title: DataHub Semantic Search +sidebar_label: Semantic Search +slug: /dev-guides/semantic-search +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/semantic-search/README.md +--- +# DataHub Semantic Search + +This directory contains documentation for DataHub's semantic search capability, which enables natural language search across metadata entities using vector embeddings. + +> **Note:** This is developer documentation for the semantic search feature. For a working example, see the smoke test at `smoke-test/tests/semantic/test_semantic_search.py`. + +## Overview + +Traditional keyword search requires exact term matches, limiting discoverability. Semantic search uses AI-generated embeddings to understand the _meaning_ of queries and documents, returning relevant results even when exact keywords don't match. + +**Example:** + +- Query: "how to request data access permissions" +- Keyword search: ❌ No results (no exact match) +- Semantic search: ✅ Returns "Data Access Request Process" document + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ DataHub Semantic Search │ +├─────────────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────────────────┐ │ +│ │ Ingestion │ │ GMS │────▶│ OpenSearch │ │ +│ │ Connector │ │ │ │ │ │ +│ │ │ │ ┌──────────┐ │ │ ┌────────────────────┐ │ │ +│ │ 1. Generate │ │ │ Process │ │ │ │ entityindex_v2 │ │ │ +│ │ embeddings│ │ │ MCP + │ │ │ │ (keyword search) │ │ │ +│ │ │ │ │ Write to │ │ │ └────────────────────┘ │ │ +│ │ 2. Emit MCP │────▶│ │ indices │ │ │ │ │ +│ │ with │ │ └──────────┘ │ │ ┌────────────────────┐ │ │ +│ │ Semantic │ │ │ │ │ entityindex_v2_ │ │ │ +│ │ Embedding │ │ │ │ │ semantic │ │ │ +│ │ aspect │ │ │ │ │ (vector search) │ │ │ +│ └──────────────┘ └──────────────┘ │ └────────────────────┘ │ │ +│ └──────────────────────────┘ │ +│ │ │ +│ ┌──────────────┐ │ │ +│ │ GraphQL │◀───────────────────────────────────────┘ │ +│ │ Client │ semanticSearchAcrossEntities() │ +│ └──────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +## How It Works + +### 1. Data Ingestion + +Documents and other entities are ingested into DataHub using standard ingestion connectors. When semantic search is enabled, GMS performs a **dual-write**: + +- **Primary Index** (`entityindex_v2`): Standard keyword-searchable index +- **Semantic Index** (`entityindex_v2_semantic`): Vector-enabled index for semantic search + +> **Note:** The dual-index approach is transitional. The plan is to eventually retire `v2` indices and use `_semantic` indices exclusively for both keyword and semantic search. See [Architecture](./ARCHITECTURE.md) for details. + +### 2. Embedding Generation + +Embeddings are generated at two points: + +**Document Embeddings** (at ingestion time): + +- Generated by the ingestion connector +- Emitted via MCP (Metadata Change Proposal) as a `SemanticContent` aspect +- GMS processes the MCP and writes embeddings to the semantic index +- Supports privacy-sensitive use cases where only embeddings (not source text) are shared + +**Query Embeddings** (at search time): + +- Generated by GMS using the configured embedding provider +- Used to find similar documents via k-NN search + +### 3. Query Processing + +When a user performs a semantic search: + +1. The query text is converted to an embedding vector using the same model +2. OpenSearch performs k-NN (k-nearest neighbors) vector similarity search +3. Results are ranked by cosine similarity to the query embedding +4. Top matches are returned through the GraphQL API + +## Quick Start + +### Prerequisites + +- DataHub running with semantic search enabled +- OpenAI API key (default), or AWS credentials (for Bedrock), or Cohere API key + +### 1. Enable Semantic Search + +Set in your environment (e.g., `docker/profiles/empty2.env`): + +```bash +ELASTICSEARCH_SEMANTIC_SEARCH_ENABLED=true +ELASTICSEARCH_SEMANTIC_SEARCH_ENTITIES=document +``` + +### 2. Run the Smoke Test + +The best way to verify semantic search is working is to run the smoke test: + +```bash +cd smoke-test +ENABLE_SEMANTIC_SEARCH_TESTS=true pytest tests/semantic/test_semantic_search.py -v +``` + +This test: + +- Ingests sample documents via GraphQL +- Waits for indexing (20 seconds) +- Executes semantic search +- Verifies results + +## GraphQL API + +### Semantic Search Query + +```graphql +query SemanticSearch($input: SearchAcrossEntitiesInput!) { + semanticSearchAcrossEntities(input: $input) { + total + searchResults { + entity { + urn + type + ... on Document { + info { + title + contents { + text + } + } + } + } + } + } +} +``` + +**Variables:** + +```json +{ + "input": { + "query": "how to request data access", + "types": ["DOCUMENT"], + "start": 0, + "count": 10 + } +} +``` + +## Documentation Index + +| File | Description | +| ------------------------ | ----------------------------------------------- | +| `README.md` | This documentation - overview and quick start | +| `ARCHITECTURE.md` | Detailed architecture and design decisions | +| `CONFIGURATION.md` | Configuration options and embedding models | +| `SWITCHING_PROVIDERS.md` | Guide for switching between embedding providers | + +## Testing + +For a working example of semantic search: + +```bash +# Run the smoke test +cd smoke-test +ENABLE_SEMANTIC_SEARCH_TESTS=true pytest tests/semantic/test_semantic_search.py -v +``` + +## Further Reading + +- [Architecture Details](./ARCHITECTURE.md) - Deep dive into the design +- [Configuration Guide](./CONFIGURATION.md) - Embedding models and settings diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/SWITCHING_PROVIDERS.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/SWITCHING_PROVIDERS.md new file mode 100644 index 00000000..00fbd42c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/semantic-search/SWITCHING_PROVIDERS.md @@ -0,0 +1,172 @@ +--- +title: Switching Embedding Providers +slug: /dev-guides/semantic-search/switching_providers +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/semantic-search/SWITCHING_PROVIDERS.md +--- +# Switching Embedding Providers + +This guide explains how to migrate from one embedding provider to another. Switching providers requires deleting the semantic index and re-ingesting all documents because different models produce vectors with incompatible dimensions. + +> For initial setup of semantic search (including all provider configurations), see [Semantic Search Configuration](../../how-to/semantic-search-configuration.md). + +## When You Need This Guide + +- Switching from OpenAI to AWS Bedrock (or vice versa) +- Switching from one model to another with different vector dimensions +- Changing from Cohere direct API to AWS Bedrock-managed Cohere + +## Provider and Model Reference + +| Provider | Model | Model Key | Dimensions | +| ----------- | ------------------------- | ------------------------ | ---------- | +| OpenAI | `text-embedding-3-large` | `text_embedding_3_large` | 3072 | +| OpenAI | `text-embedding-3-small` | `text_embedding_3_small` | 1536 | +| AWS Bedrock | `cohere.embed-english-v3` | `cohere_embed_v3` | 1024 | +| Cohere | `embed-english-v3.0` | `embed_english_v3_0` | 1024 | + +> **Important:** The model key is derived from the model name by replacing `-` and `.` with `_`. Both the ingestion connector and GMS must use the same model to ensure query embeddings match document embeddings. + +## Migration Steps + +### Step 1: Stop DataHub Services + +Stop GMS and any ingestion jobs to prevent writes during migration: + +```bash +# Docker Compose +docker stop datahub-gms + +# Kubernetes +kubectl scale deployment datahub-gms --replicas=0 +``` + +### Step 2: Delete the Semantic Index + +Delete the existing semantic index from OpenSearch: + +```bash +# Check existing semantic indices +curl -s "http://localhost:9200/_cat/indices/*semantic*?v" + +# Delete the semantic index (adjust index name as needed) +curl -X DELETE "http://localhost:9200/documentindex_v2_semantic" +``` + +### Step 3: Update Provider Configuration + +Update your configuration with the new provider settings. See [Semantic Search Configuration](../../how-to/semantic-search-configuration.md) for the full configuration options for each provider (Helm charts and environment variables). + +Make sure to update: + +- **Provider type** (`EMBEDDING_PROVIDER_TYPE`) +- **API credentials** (API key or IAM role) +- **Vector dimension** (`ELASTICSEARCH_SEMANTIC_VECTOR_DIMENSION`) to match the new model + +### Step 4: Update Index Configuration + +If using `application.yaml`, update the model entry to match the new provider: + +```yaml +elasticsearch: + entityIndex: + semanticSearch: + models: + # Use the model key that matches your new provider + text_embedding_3_large: + vectorDimension: 3072 # Must match model output + knnEngine: faiss + spaceType: cosinesimil + efConstruction: 128 + m: 16 +``` + +Or via environment variable: + +```bash +ELASTICSEARCH_SEMANTIC_VECTOR_DIMENSION=3072 +``` + +### Step 5: Start DataHub + +Start GMS — the system update job will automatically recreate the semantic index: + +```bash +# Docker Compose +docker start datahub-gms + +# Kubernetes +kubectl scale deployment datahub-gms --replicas=1 +``` + +The system update job runs automatically on startup and will: + +1. Detect the missing semantic index +2. Create it with the correct mapping for your new embedding model +3. Log progress to the GMS logs + +### Step 6: Re-ingest Documents + +After the index is recreated, re-ingest your documents to generate new embeddings: + +```bash +datahub ingest -c your-recipe.yaml +``` + +**Important:** Make sure your ingestion recipe also uses the same embedding model. The ingestion connector generates document embeddings, while GMS generates query embeddings — both must use the same model. + +### Step 7: Verify + +```bash +# Check the index exists with correct mapping +curl -s "http://localhost:9200/documentindex_v2_semantic/_mapping?pretty" | head -50 + +# Check documents have embeddings +curl -s "http://localhost:9200/documentindex_v2_semantic/_search" \ + -H "Content-Type: application/json" \ + -d '{"size": 1, "_source": ["urn", "embeddings"]}' | head -30 + +# Test semantic search via GraphQL or the UI +``` + +## Troubleshooting + +### "No embeddings found" after switching + +**Cause:** Documents were ingested before the provider switch and have embeddings from the old model. + +**Solution:** Re-run ingestion to generate new embeddings with the new provider. + +### "Dimension mismatch" errors + +**Cause:** The index was created with a different vector dimension than the new model produces. + +**Solution:** Delete the semantic index and let it be recreated (Steps 2-5 above). + +### "Invalid API key" errors + +**Cause:** API key not set or incorrect. + +**Solution:** Verify your API key is correctly set in the environment: + +```bash +# Check the environment variable is set (in the container) +docker exec datahub-gms env | grep -E 'OPENAI_API_KEY|COHERE_API_KEY' +``` + +### Query returns no results but documents exist + +**Cause:** Model mismatch between ingestion and query time. + +**Solution:** Ensure both the ingestion connector AND GMS use the same embedding model. Check: + +- The provider-specific model env var (`BEDROCK_EMBEDDING_MODEL`, `OPENAI_EMBEDDING_MODEL`, or `COHERE_EMBEDDING_MODEL`) in GMS config +- Embedding model in your ingestion recipe + +## Best Practices + +1. **Use the same model everywhere**: Ensure ingestion connectors and GMS use identical embedding models. +2. **Test in development first**: Switch providers in a dev environment before production. +3. **Plan for re-ingestion**: Switching providers requires re-generating all embeddings, which can take time for large datasets. +4. **Monitor costs**: Different providers have different pricing. OpenAI and Cohere charge per token/request. +5. **Keep backups**: Before deleting indices, consider backing up if you might need to rollback. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/timeline.md b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/timeline.md new file mode 100644 index 00000000..e41c8e71 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/dev-guides/timeline.md @@ -0,0 +1,265 @@ +--- +title: Timeline API +sidebar_label: Timeline API +slug: /dev-guides/timeline +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/dev-guides/timeline.md +--- + +The Timeline API supports viewing version history of schemas, documentation, tags, glossary terms, and other updates +to entities. At present, the API only supports Datasets and Glossary Terms. + +## Compatibility + +The Timeline API is available in server versions `0.8.28` and higher. The `cli` timeline command is available in [pypi](https://pypi.org/project/acryl-datahub/) versions `0.8.27.1` onwards. + +# Concepts + +## Entity Timeline Conceptually + +For the visually inclined, here is a conceptual diagram that illustrates how to think about the entity timeline with categorical changes overlaid on it. + +

+ +

+ +## Change Event + +Each modification is modeled as a +[ChangeEvent](https://github.com/datahub-project/datahub/blob/master/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeEvent.java) +which are grouped under [ChangeTransactions](https://github.com/datahub-project/datahub/blob/master/metadata-service/services/src/main/java/com/linkedin/metadata/timeline/data/ChangeTransaction.java) +based on timestamp. A `ChangeEvent` consists of: + +- `changeType`: An operational type for the change, either `ADD`, `MODIFY`, or `REMOVE` +- `semVerChange`: A [semver](https://semver.org/) change type based on the compatibility of the change. This gets utilized in the computation of the transaction level version. Options are `NONE`, `PATCH`, `MINOR`, `MAJOR`, and `EXCEPTIONAL` for cases where an exception occurred during processing, but we do not fail the entire change calculation +- `target`: The high level target of the change. This is usually an `urn`, but can differ depending on the type of change. +- `category`: The category a change falls under, specific aspects are mapped to each category depending on the entity +- `elementId`: Optional, the ID of the element being applied to the target +- `description`: A human readable description of the change produced by the `Differ` type computing the diff +- `changeDetails`: A loose property map of additional details about the change +- `modificationCategory`: Specifies the type of modification made to a schema field within an Entity change event. Options are `RENAME`, `TYPE_CHANGE` and `OTHER`. + +### Change Event Examples + +- A tag was applied to a _field_ of a dataset through the UI: + - `changeType`: `ADD` + - `target`: `urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:,,),)` -> The field the tag is being added to + - `category`: `TAG` + - `elementId`: `urn:li:tag:` -> The ID of the tag being added + - `semVerChange`: `MINOR` + - `modificationCategory`: `OTHER` +- A tag was added directly at the top-level to a dataset through the UI: + - `changeType`: `ADD` + - `target`: `urn:li:dataset:(urn:li:dataPlatform:,,)` -> The dataset the tag is being added to + - `category`: `TAG` + - `elementId`: `urn:li:tag:` -> The ID of the tag being added + - `semVerChange`: `MINOR` + - `modificationCategory`: `OTHER` + +Note the `target` and `elementId` fields in the examples above to familiarize yourself with the semantics. + +## Change Transaction + +Each `ChangeTransaction` is assigned a computed semantic version based on the `ChangeEvents` that occurred within it, +starting at `0.0.0` and updating based on whether the most significant change in the transaction is a `MAJOR`, `MINOR`, or +`PATCH` change. The logic for what changes constitute a Major, Minor or Patch change are encoded in the category specific `Differ` implementation. +For example, the [SchemaMetadataDiffer](https://github.com/datahub-project/datahub/blob/master/metadata-io/src/main/java/com/linkedin/metadata/timeline/eventgenerator/SchemaMetadataChangeEventGenerator.java) has baked-in logic for determining what level of semantic change an event is based on backwards and forwards incompatibility. Read on to learn about the different categories of changes, and how semantic changes are interpreted in each. + +# Categories + +ChangeTransactions contain a `category` that represents a kind of change that happened. The `Timeline API` allows the caller to specify which categories of changes they are interested in. Categories allow us to abstract away the low-level technical change that happened in the metadata (e.g. the `schemaMetadata` aspect changed) to a high-level semantic change that happened in the metadata (e.g. the `Technical Schema` of the dataset changed). Read on to learn about the different categories that are supported today. + +The Dataset entity currently supports the following categories: + +## Technical Schema + +- Any structural changes in the technical schema of the dataset, such as adding, dropping, renaming columns. +- Driven by the `schemaMetadata` aspect. +- Changes are marked with the appropriate semantic version marker based on well-understood rules for backwards and forwards compatibility. + +**_NOTE_**: Changes in field descriptions are not communicated via this category, use the Documentation category for that. + +### Example Usage + +We have provided some example scripts that demonstrate making changes to an aspect within each category and use then use the Timeline API to query the result. +All examples can be found in [smoke-test/test_resources/timeline](https://github.com/datahub-project/datahub/blob/master/smoke-test/test_resources/timeline) and should be executed from that directory. + +```console +% ./test_timeline_schema.sh +[2022-02-24 15:31:52,617] INFO {datahub.cli.delete_cli:130} - DataHub configured with http://localhost:8080 +Successfully deleted urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD). 6 rows deleted +Took 1.077 seconds to hard delete 6 rows for 1 entities +Update succeeded with status 200 +Update succeeded with status 200 +Update succeeded with status 200 +http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TECHNICAL_SCHEMA&start=1644874316591&end=2682397800000 +2022-02-24 15:31:53 - 0.0.0-computed + ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:property_id): A forwards & backwards compatible change due to the newly added field 'property_id'. + ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service): A forwards & backwards compatible change due to the newly added field 'service'. + ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.type): A forwards & backwards compatible change due to the newly added field 'service.type'. + ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.provider): A forwards & backwards compatible change due to the newly added field 'service.provider'. + ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.provider.name): A forwards & backwards compatible change due to the newly added field 'service.provider.name'. + ADD TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.provider.id): A forwards & backwards compatible change due to the newly added field 'service.provider.id'. +2022-02-24 15:31:55 - 1.0.0-computed + MODIFY TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.provider.name): A backwards incompatible change due to native datatype of the field 'service.provider.id' changed from 'varchar(50)' to 'tinyint'. + MODIFY TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.provider.id): A forwards compatible change due to field name changed from 'service.provider.id' to 'service.provider.id2' +2022-02-24 15:31:55 - 2.0.0-computed + MODIFY TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.provider.id): A backwards incompatible change due to native datatype of the field 'service.provider.name' changed from 'tinyint' to 'varchar(50)'. + MODIFY TECHNICAL_SCHEMA dataset:hive:testTimelineDataset (field:service.provider.id2): A forwards compatible change due to field name changed from 'service.provider.id2' to 'service.provider.id' +``` + +## Ownership + +- Any changes in ownership of the dataset, adding an owner, or changing the type of the owner. +- Driven by the `ownership` aspect. +- All changes are currently marked as `MINOR`. + +### Example Usage + +We have provided some example scripts that demonstrate making changes to an aspect within each category and use then use the Timeline API to query the result. +All examples can be found in [smoke-test/test_resources/timeline](https://github.com/datahub-project/datahub/blob/master/smoke-test/test_resources/timeline) and should be executed from that directory. + +```console +% ./test_timeline_ownership.sh +[2022-02-24 15:40:25,367] INFO {datahub.cli.delete_cli:130} - DataHub configured with http://localhost:8080 +Successfully deleted urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD). 6 rows deleted +Took 1.087 seconds to hard delete 6 rows for 1 entities +Update succeeded with status 200 +Update succeeded with status 200 +Update succeeded with status 200 +http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874829027&end=2682397800000 +2022-02-24 15:40:26 - 0.0.0-computed + ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. + ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): A new owner 'jdoe' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:40:27 - 0.1.0-computed + REMOVE OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): Owner 'datahub' of the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. +2022-02-24 15:40:28 - 0.2.0-computed + ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. + REMOVE OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): Owner 'jdoe' of the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. +Update succeeded with status 200 +Update succeeded with status 200 +Update succeeded with status 200 +http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=OWNER&start=1644874831456&end=2682397800000 +2022-02-24 15:40:26 - 0.0.0-computed + ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. + ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): A new owner 'jdoe' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:40:27 - 0.1.0-computed + REMOVE OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): Owner 'datahub' of the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. +2022-02-24 15:40:28 - 0.2.0-computed + ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:datahub): A new owner 'datahub' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. + REMOVE OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): Owner 'jdoe' of the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. +2022-02-24 15:40:29 - 0.2.0-computed +2022-02-24 15:40:30 - 0.3.0-computed + ADD OWNERSHIP dataset:hive:testTimelineDataset (urn:li:corpuser:jdoe): A new owner 'jdoe' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:40:30 - 0.4.0-computed + MODIFY OWNERSHIP urn:li:corpuser:jdoe (DEVELOPER): The ownership type of the owner 'jdoe' changed from 'DATAOWNER' to 'DEVELOPER'. +``` + +## Tags + +- Any changes in tags applied to the dataset or to fields of the dataset. +- Driven by the `schemaMetadata`, `editableSchemaMetadata` and `globalTags` aspects. +- All changes are currently marked as `MINOR`. + +### Example Usage + +We have provided some example scripts that demonstrate making changes to an aspect within each category and use then use the Timeline API to query the result. +All examples can be found in [smoke-test/test_resources/timeline](https://github.com/datahub-project/datahub/blob/master/smoke-test/test_resources/timeline) and should be executed from that directory. + +```console +% ./test_timeline_tags.sh +[2022-02-24 15:44:04,279] INFO {datahub.cli.delete_cli:130} - DataHub configured with http://localhost:8080 +Successfully deleted urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD). 9 rows deleted +Took 0.626 seconds to hard delete 9 rows for 1 entities +Update succeeded with status 200 +Update succeeded with status 200 +Update succeeded with status 200 +http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=TAG&start=1644875047911&end=2682397800000 +2022-02-24 15:44:05 - 0.0.0-computed + ADD TAG dataset:hive:testTimelineDataset (urn:li:tag:Legacy): A new tag 'Legacy' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:44:06 - 0.1.0-computed + ADD TAG dataset:hive:testTimelineDataset (urn:li:tag:NeedsDocumentation): A new tag 'NeedsDocumentation' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:44:07 - 0.2.0-computed + REMOVE TAG dataset:hive:testTimelineDataset (urn:li:tag:Legacy): Tag 'Legacy' of the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. + REMOVE TAG dataset:hive:testTimelineDataset (urn:li:tag:NeedsDocumentation): Tag 'NeedsDocumentation' of the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. +``` + +## Documentation + +- Any changes to documentation at the dataset level or at the field level. +- Driven by the `datasetProperties`, `institutionalMemory`, `schemaMetadata` and `editableSchemaMetadata`. +- Addition or removal of documentation or links is marked as `MINOR` while edits to existing documentation are marked as `PATCH` changes. + +### Example Usage + +We have provided some example scripts that demonstrate making changes to an aspect within each category and use then use the Timeline API to query the result. +All examples can be found in [smoke-test/test_resources/timeline](https://github.com/datahub-project/datahub/blob/master/smoke-test/test_resources/timeline) and should be executed from that directory. + +```console +% ./test_timeline_documentation.sh +[2022-02-24 15:45:53,950] INFO {datahub.cli.delete_cli:130} - DataHub configured with http://localhost:8080 +Successfully deleted urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD). 6 rows deleted +Took 0.578 seconds to hard delete 6 rows for 1 entities +Update succeeded with status 200 +Update succeeded with status 200 +Update succeeded with status 200 +http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=DOCUMENTATION&start=1644875157616&end=2682397800000 +2022-02-24 15:45:55 - 0.0.0-computed + ADD DOCUMENTATION dataset:hive:testTimelineDataset (https://www.linkedin.com): The institutionalMemory 'https://www.linkedin.com' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:45:56 - 0.1.0-computed + ADD DOCUMENTATION dataset:hive:testTimelineDataset (https://www.google.com): The institutionalMemory 'https://www.google.com' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:45:56 - 0.2.0-computed + ADD DOCUMENTATION dataset:hive:testTimelineDataset (https://docs.datahub.com/docs): The institutionalMemory 'https://docs.datahub.com/docs' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. + ADD DOCUMENTATION dataset:hive:testTimelineDataset (https://docs.datahub.com/docs): The institutionalMemory 'https://docs.datahub.com/docs' for the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. + REMOVE DOCUMENTATION dataset:hive:testTimelineDataset (https://www.linkedin.com): The institutionalMemory 'https://www.linkedin.com' of the dataset 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. +``` + +## Glossary Terms + +- Any changes to applied glossary terms to the dataset or to fields in the dataset. +- Driven by the `schemaMetadata`, `editableSchemaMetadata`, `glossaryTerms` aspects. +- All changes are currently marked as `MINOR`. + +### Example Usage + +We have provided some example scripts that demonstrate making changes to an aspect within each category and use then use the Timeline API to query the result. +All examples can be found in [smoke-test/test_resources/timeline](https://github.com/datahub-project/datahub/blob/master/smoke-test/test_resources/timeline) and should be executed from that directory. + +```console +% ./test_timeline_glossary.sh +[2022-02-24 15:44:56,152] INFO {datahub.cli.delete_cli:130} - DataHub configured with http://localhost:8080 +Successfully deleted urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD). 6 rows deleted +Took 0.443 seconds to hard delete 6 rows for 1 entities +Update succeeded with status 200 +Update succeeded with status 200 +Update succeeded with status 200 +http://localhost:8080/openapi/v2/timeline/v1/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CtestTimelineDataset%2CPROD%29?categories=GLOSSARY_TERM&start=1644875100605&end=2682397800000 +1969-12-31 18:00:00 - 0.0.0-computed + None None : java.lang.NullPointerException:null +2022-02-24 15:44:58 - 0.1.0-computed + ADD GLOSSARY_TERM dataset:hive:testTimelineDataset (urn:li:glossaryTerm:SavingsAccount): The GlossaryTerm 'SavingsAccount' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been added. +2022-02-24 15:44:59 - 0.2.0-computed + REMOVE GLOSSARY_TERM dataset:hive:testTimelineDataset (urn:li:glossaryTerm:CustomerAccount): The GlossaryTerm 'CustomerAccount' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. + REMOVE GLOSSARY_TERM dataset:hive:testTimelineDataset (urn:li:glossaryTerm:SavingsAccount): The GlossaryTerm 'SavingsAccount' for the entity 'urn:li:dataset:(urn:li:dataPlatform:hive,testTimelineDataset,PROD)' has been removed. +``` + +## Explore the API + +The API is browse-able via the UI through through the dropdown. +Here are a few screenshots showing how to navigate to it. You can try out the API and send example requests. + +

+ +

+ +

+ +

+ +## Future Work + +- Supporting versions as start and end parameters as part of the call to the timeline API +- Supporting entities beyond Datasets +- Adding GraphQL API support +- Supporting materialization of computed versions for entity categories (compared to the current read-time version computation) +- Support in the UI to visualize the timeline in various places (e.g. schema history, etc.) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/developers.md b/docs-archive/versioned_docs/version-1.5.0/docs/developers.md new file mode 100644 index 00000000..a738a2f8 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/developers.md @@ -0,0 +1,485 @@ +--- +title: Local Development +sidebar_label: Local Development +slug: /developers +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/developers.md' +--- + +# DataHub Developer's Guide + +## Requirements + +- [Java 17 JDK](https://openjdk.org/projects/jdk/17/) +- [Python 3.11](https://www.python.org/downloads/latest/python3.11/) +- [Docker](https://www.docker.com/) +- [Node 22.x](https://nodejs.org/en/about/previous-releases) +- [Docker Compose >=2.20](https://docs.docker.com/compose/) +- [Yarn >=v1.22](https://yarnpkg.com/en/docs/cli/) for documentation building +- Docker engine with at least 8GB of memory to run tests. + +### Option 1: Using Homebrew on Mac + +On macOS, these can be installed using [Homebrew](https://brew.sh/). + +```shell +# Install Java +brew install openjdk@17 + +# Install Python +brew install python@3.11 # you may need to add this to your PATH +# alternatively, you can use pyenv to manage your python versions + +# Install docker and docker compose +brew install --cask docker +``` + +### Option 2: Using mise + +Alternatively you can use [mise en place](https://mise.jdx.dev/) for managing tool installation. +You can see the existing mise.toml file in the repository and let mise manage the tool versions. + +You can install mise cli by following the instructions on https://mise.jdx.dev/getting-started.html + +Fork and clone the repo if you haven't done so already. Refer: [Building the Project](#building-the-project) + +```shell +# Enter the root folder of the repo +> cd datahub + +# Needed the first time to allow mise to auto activate the tools +# mentioned in mise.toml +> mise trust + +# Needed once if the required tools haven't been installed via mise before +# or if a new tool is added or tool version changed since last use +> mise install +``` + +After this the required tools should be auto activated as soon as you enter the folder where the repo is cloned + +You can verify the tools are activated correctly by running + +```shell +# Check tool versions installed +❯ mise ls --local +Tool Version Source Requested +java 17.0.2 ~/path/to/datahub/mise.toml 17 +node 22.21.1 ~/path/to/datahub/mise.toml 22 +python 3.11.14 ~/path/to/datahub/mise.toml 3.11 +yarn 4.12.0 ~/path/to/datahub/mise.toml latest +``` + +## Building the Project + +Fork and clone the repository if haven't done so already + +```shell +git clone https://github.com/{username}/datahub.git +``` + +Change into the repository's root directory + +```shell +cd datahub +``` + +Use [gradle wrapper](https://docs.gradle.org/current/userguide/gradle_wrapper.html) to build the project + +```shell +./gradlew build +``` + +Note that the above will also run tests and a number of validations which makes the process considerably slower. + +We suggest partially compiling DataHub according to your needs: + +- Build DataHub's backend GMS (Generalized metadata service): + + ``` + ./gradlew :metadata-service:war:build + ``` + +- Build DataHub's frontend: + + ``` + ./gradlew :datahub-frontend:dist -x yarnTest -x yarnLint + ``` + +- Build DataHub's command line tool: + + ``` + ./gradlew :metadata-ingestion:installDev + ``` + +- Build DataHub's documentation: + + ``` + ./gradlew :docs-website:yarnLintFix :docs-website:build -x :metadata-ingestion:runPreFlightScript + # To preview the documentation + ./gradlew :docs-website:serve + ``` + +## Dependency Management + +### Dependency Locking + +DataHub uses Gradle's [dependency locking](https://docs.gradle.org/current/userguide/dependency_locking.html) to ensure reproducible builds across all environments. Dependency locks guarantee that the exact same dependency versions are used everywhere - in development, CI, and production - preventing unexpected behavior from transitive dependency updates. + +#### Why Dependency Locking? + +- **Reproducible Builds**: Same dependency versions across all environments +- **Security**: Prevents unexpected transitive dependency updates that could introduce vulnerabilities +- **Stability**: Explicit control over when dependencies change +- **Audit Trail**: Lock files in git provide clear history of dependency changes + +#### How It Works + +Each sub-project has a `gradle.lockfile` that records the exact version of every direct and transitive dependency used across all configurations (compile, runtime, test, etc.). When you build the project, Gradle uses these locked versions instead of resolving the latest versions. + +#### Viewing Locked Dependencies + +To see the locked dependencies for a project: + +```bash +./gradlew :project-name:dependencies --configuration runtimeClasspath +``` + +For example: + +```bash +./gradlew :metadata-io:dependencies --configuration runtimeClasspath +``` + +#### Updating Dependencies + +When you update dependencies in `build.gradle` files, you must regenerate the lock files. There are several ways to do this depending on your needs: + +##### Update All Lock Files (Complete Refresh) + +After changing dependencies in any `build.gradle` file, run: + +```bash +./gradlew resolveAndLockAll --write-locks +``` + +This resolves all configurations across all sub-projects and updates all lock files. This is the recommended approach when: + +- Adding or removing dependencies +- Changing dependency versions +- After pulling changes that modify `build.gradle` files + +##### Update a Specific Dependency Version + +To update a specific dependency to a newer version: + +```bash +./gradlew dependencies --update-locks group:artifact +``` + +For example, to update Jackson: + +```bash +./gradlew dependencies --update-locks com.fasterxml.jackson.core:jackson-databind +``` + +##### Update Lock Files for a Single Project + +If you only changed dependencies in one sub-project: + +```bash +./gradlew :project-name:dependencies --write-locks +``` + +Note: This only locks configurations that get resolved by the `dependencies` task. For complete coverage, use `resolveAndLockAll` instead. + +#### Common Workflows + +**Adding a new dependency:** + +1. Add the dependency to the appropriate `build.gradle` file +2. Run `./gradlew resolveAndLockAll --write-locks` +3. Verify the build works: `./gradlew :project-name:build` +4. Commit both the `build.gradle` change and updated lock files + +**Updating an existing dependency:** + +1. Change the version in `build.gradle` +2. Run `./gradlew resolveAndLockAll --write-locks` +3. Review the lock file changes to see what transitive dependencies changed +4. Test thoroughly, especially if major version upgrades +5. Commit the changes + +**After pulling changes:** +If someone else updated dependencies and you pull their changes, just build normally: + +```bash +./gradlew build +``` + +Gradle will automatically use the locked versions from the updated lock files. You don't need to regenerate locks unless you're making your own dependency changes. + +#### Troubleshooting + +**Build fails with dependency resolution error:** + +- Ensure you've run `./gradlew resolveAndLockAll --write-locks` after dependency changes +- Check that lock files are committed and up to date +- Try `./gradlew --refresh-dependencies build` to refresh Gradle's cache + +**Lock file is out of sync:** +If you see warnings about lock state, regenerate the locks: + +```bash +./gradlew resolveAndLockAll --write-locks +``` + +## Deploying Local Versions + +This guide explains how to set up and deploy DataHub locally for development purposes. + +### Initial Setup + +Before you begin, you'll need to install the local `datahub` CLI tool: + +```shell +cd metadata-ingestion/ +python3 -m venv venv +source venv/bin/activate +cd ../ +``` + +### Deploying the Full Stack + +Deploy the entire system using docker-compose: + +```shell +./gradlew quickstartDebug +``` + +:::note +Starting v1.5.0 the default values for `authentication.tokenService.signingKey` and `authentication.tokenService.salt` have been removed. It is recommened that users provided their own values for these. If no values are provided, new keys are generated the first time the above command is run. The generated keys are saved to docker/.local-secrets.env and reused in subsequent runs as long as the file exists. + +To provide your own values, either update the above keys in `metadata-service/configuration/src/main/resources/application.yaml` or set the following environment variables + +``` +DATAHUB_TOKEN_SERVICE_SIGNING_KEY +DATAHUB_TOKEN_SERVICE_SALT +``` + +If you are upgrading from an earlier version Personal Access Tokens created before will be invalidated and will need to be created again. +::: +Access the DataHub UI at `http://localhost:9002` + +### Refreshing the Frontend + +To run and update the frontend with local changes, open a new terminal and run: + +```shell +cd datahub-web-react +yarn install && yarn start +``` + +The frontend will be available at `http://localhost:3000` and will automatically update as you make changes to the code. + +### Refreshing components of quickStart + +To refresh any of the running system started by `./gradlew quickstartDebug`, run + +```shell +./gradlew debugReload +``` + +This will build any changed components and restart those containers that had changes. +There are a few other quickstart\* variants, like quickstartDebugMin, quickstartDebugConsumers + +For each of those variants, there is a corresponding reloadTask. + +For `./gradlew quickstartDebugConsumers`, the reload command is `./gradlew debugConsumersReload` +For `./gradlew quickstartDebugMin`, the reload command is `./gradlew debugMinReload` + +A full restart using `./gradlew quickstartDebug` is recommended if there are significant changes and the setup/system update containers need to be run again. +For incremental changes, the `debugReload*` variants can be used. + +### Cleaning up containers and volumes + +To completely remove containers and volumes for a specific project, you can use the nuke tasks: + +```shell +# Remove containers and volumes for specific projects +./gradlew quickstartDebugNuke # For debug project +./gradlew quickstartCypressNuke # For cypress project (dh-cypress) +``` + +> **Note**: These are Gradle nuke tasks. For CLI-based cleanup, see `datahub docker nuke` in the [quickstart guide](quickstart.md). + +### Using .env to configure settings of services started by quickstart + +To start datahub with a customized set of environment variables, .env files can be created in the docker/profiles folder. +For example, an env file `my-settings.env` can be created in docker/profiles folder and loaded using + +```shell +DATAHUB_LOCAL_COMMON_ENV=my-settings.env ./gradlew quickstartDebug +``` + +To refresh the containers due to code changes, `debugReload` task can be used. +To change the env and reload containers, use the task `debugReloadEnv` + +```shell +DATAHUB_LOCAL_COMMON_ENV=my-other-settings.env ./gradlew debugReloadEnv +``` + +This will build any container artifacts were changed and all reloadable containers are re-created to use the new env settings. + +### Refreshing the CLI + +If you haven't set up the CLI for local development yet, run: + +```commandline +./gradlew :metadata-ingestion:installDev +cd metadata-ingestion +source venv/bin/activate +``` + +Once you're in `venv`, your local changes will be reflected automatically. +For example, you can run `datahub ingest -c ` to test local changes in ingestion connectors. + +To verify that you're using the local version, run: + +```commandline +datahub --version +``` + +Expected Output: + +```commandline +acryl-datahub, version unavailable (installed in develop mode) +``` + +### Building All Docker images + +Running `./gradlew quickstart` or one of its variants builds images required for that variant and also starts datahub. +If you want to build all images without starting datahub, run + +```commandline +./gradlew :docker:build +``` + +You can optionally pass the following additional args when executing `:docker:build` task + +- `-Ptag=customTag` to use the custom tag when generating the image tag. +- `-PdockerRegistry=customRegistry` to use the custom registry when generating the full image tag. + +## IDE Support + +The recommended IDE for DataHub development is [IntelliJ IDEA](https://www.jetbrains.com/idea/). + +### Required IntelliJ Plugins + +DataHub requires the following IntelliJ plugins for proper development: + +1. **Lombok Plugin** - Essential for Lombok annotation processing + - Install: Settings → Plugins → Search "Lombok" → Install + +### Setup Steps + +1. Install required plugins (see above) +2. Generate the IntelliJ project file (re-run after dependency changes): + ```shell + ./gradlew idea + ``` +3. Open `datahub.ipr` in IntelliJ + +## Windows Compatibility + +For optimal performance and compatibility, we strongly recommend building on a Mac or Linux system. +Please note that we do not actively support Windows in a non-virtualized environment. + +If you must use Windows, one workaround is to build within a virtualized environment, such as a VM(Virtual Machine) or [WSL(Windows Subsystem for Linux)](https://learn.microsoft.com/en-us/windows/wsl). +This approach can help ensure that your build environment remains isolated and stable, and that your code is compiled correctly. + +## Common Build Issues + +#### Getting `Unsupported class file major version 57` + +You're probably using a Java version that's too new for gradle. Run the following command to check your Java version + +```shell +java --version +``` + +While it may be possible to build and run DataHub using newer versions of Java, we currently only support [Java 17](https://openjdk.org/projects/jdk/17/) (aka Java 17). + +#### Getting `cannot find symbol` error for `javax.annotation.Generated` + +Similar to the previous issue, please use Java 17 to build the project. +You can install multiple version of Java on a single machine and switch between them using the `JAVA_HOME` environment variable. See [this document](https://docs.oracle.com/cd/E21454_01/html/821-2531/inst_jdk_javahome_t.html) for more details. + +#### `:metadata-models:generateDataTemplate` task fails with `java.nio.file.InvalidPathException: Illegal char <:> at index XX` or `Caused by: java.lang.IllegalArgumentException: 'other' has different root` error + +This is a [known issue](https://github.com/linkedin/rest.li/issues/287) when building the project on Windows due a bug in the Pegasus plugin. Please refer to [Windows Compatibility](/docs/developers.md#windows-compatibility). + +#### Various errors related to `generateDataTemplate` or other `generate` tasks + +As we generate quite a few files from the models, it is possible that old generated files may conflict with new model changes. When this happens, a simple `./gradlew clean` should resolve the issue. + +#### `Execution failed for task ':metadata-service:restli-servlet-impl:checkRestModel'` + +This generally means that an [incompatible change](https://linkedin.github.io/rest.li/modeling/compatibility_check) was introduced to the rest.li API in GMS. You'll need to rebuild the snapshots/IDL by running the following command once + +```shell +./gradlew :metadata-service:restli-servlet-impl:build -Prest.model.compatibility=ignore +``` + +#### `java.io.IOException: No space left on device` + +This means you're running out of space on your disk to build. Please free up some space or try a different disk. + +#### `Build failed` for task `./gradlew :datahub-frontend:dist -x yarnTest -x yarnLint` + +This could mean that you need to update your [Yarn](https://yarnpkg.com/getting-started/install) version + +#### `:buildSrc:compileJava` task fails with `package com.linkedin.metadata.models.registry.config does not exist` and `cannot find symbol` error for `Entity` + +There are currently two symbolic links within the [buildSrc](https://github.com/datahub-project/datahub/tree/master/buildSrc) directory for the [com.linkedin.metadata.aspect.plugins.config](https://github.com/datahub-project/datahub/blob/master/buildSrc/src/main/java/com/linkedin/metadata/aspect/plugins/config) and [com.linkedin.metadata.models.registry.config](https://github.com/datahub-project/datahub/blob/master/buildSrc/src/main/java/com/linkedin/metadata/models/registry/config) packages, which points to the corresponding packages in the [entity-registry](https://github.com/datahub-project/datahub/tree/master/entity-registry) subproject. + +When the repository is checked out using Windows 10/11 - even if WSL is later used for building using the mounted Windows filesystem in `/mnt/` - the symbolic links might have not been created correctly, instead the symbolic links were checked out as plain files. Although it is technically possible to use the mounted Windows filesystem in `/mnt/` for building in WSL, it is **strongly recommended** to checkout the repository within the Linux filesystem (e.g., in `/home/`) and building it from there, because accessing the Windows filesystem from Linux is relatively slow compared to the Linux filesystem and slows down the whole building process. + +To be able to create symbolic links in Windows 10/11 the [Developer Mode](https://learn.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development) has to be enabled first. Then the following commands can be used to enable [symbolic links in Git](https://git-scm.com/docs/git-config#Documentation/git-config.txt-coresymlinks) and recreating the symbolic links: + +```shell +# enable core.symlinks config +git config --global core.symlinks true + +# check the current core.symlinks config and scope +git config --show-scope --show-origin core.symlinks + +# in case the core.symlinks config is still set locally to false, remove the local config +git config --unset core.symlinks + +# reset the current branch to recreate the missing symbolic links (alternatively it is also possibly to switch branches away and back) +git reset --hard +``` + +See also [here](https://stackoverflow.com/questions/5917249/git-symbolic-links-in-windows/59761201#59761201) for more information on how to enable symbolic links on Windows 10/11 and Git. + +## Security Testing + +### Configuration Property Classification Test + +**Location**: `metadata-io/src/test/java/com/linkedin/metadata/system_info/collectors/PropertiesCollectorConfigurationTest.java` + +This test ensures all configuration properties are explicitly classified as either sensitive (redacted) or non-sensitive (visible in system info). It prevents accidental exposure of secrets through DataHub's system information endpoints. + +**When you add new configuration properties:** + +1. The test will fail if your property is unclassified +2. Follow the test failure message to add your property to the appropriate classification list +3. When in doubt, classify as sensitive - it's safer to over-redact than expose secrets + +**Run the test:** + +```bash +./gradlew :metadata-io:test --tests "*.PropertiesCollectorConfigurationTest" +``` + +Refer to the test file itself for comprehensive documentation on classification lists, template syntax, and examples. This is a mandatory security guardrail that protects against credential leaks. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-design.md b/docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-design.md new file mode 100644 index 00000000..a3e5ec48 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-design.md @@ -0,0 +1,1278 @@ +--- +title: DataHub Java SDK V2 Design Document +sidebar_label: Java SDK V2 Design Document +slug: /developers/java-sdk-v2-design +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/developers/java-sdk-v2-design.md +--- +# DataHub Java SDK V2 Design Document + +## Executive Summary + +This document describes the design of DataHub Java SDK V2, a modern, user-friendly Java client library that provides feature parity with the Python SDK V2. The new SDK addresses feedback from enterprise Java customers who require a first-class SDK experience comparable to Python developers. + +This document is organized into two main sections: + +- **Part 1 - User-Facing API Design**: The public API, patterns, and behaviors visible to SDK users +- **Part 2 - Developer-Facing Implementation**: Internal architecture and implementation details for contributors + +> **Why Hand-Crafted?** For a deep dive into why we chose to hand-craft this SDK instead of using OpenAPI code generation, see [Java SDK V2 Philosophy](java-sdk-v2-philosophy.md). + +## Background + +### Problem Statement + +Currently, DataHub's Java SDK (`datahub-client`) provides only low-level emission capabilities: + +- Manual MCP (Metadata Change Proposal) construction required +- No high-level entity builders for Dataset, Chart, Dashboard, etc. +- No client for CRUD operations (read, update, delete) +- No patch capabilities for granular updates +- Significantly inferior developer experience compared to Python SDK V2 + +This gap has created issues with enterprise customers, particularly Java shops who feel like "second-class citizens" when compared to Python developers. + +### Goals + +1. **Feature Parity**: Match Python SDK V2 capabilities for entity management +2. **Backward Compatibility**: Maintain 100% compatibility with existing Java SDK +3. **Namespace Separation**: Use `datahub.client.v2.*` namespace for new APIs +4. **Builder Pattern**: Fluent, type-safe API for entity construction +5. **Patch Support**: Granular updates without full entity replacement +6. **CRUD Operations**: Support create, read, update, upsert operations (delete/exists deferred) +7. **Comprehensive Testing**: Unit and integration tests validating all functionality + +### Non-Goals + +- Rewriting existing emitter infrastructure (leverage existing) +- 100% feature parity with Python SDK (focus on core entities first) +- GraphQL client implementation (focus on REST/OpenAPI) +- Search client (future enhancement) +- Lineage client (future enhancement) + +--- + +# Part 1: User-Facing API Design + +This section describes the public API that SDK users interact with - the patterns, behaviors, and interfaces that define the developer experience. + +## Design Principles + +### 1. Fluent Builder Pattern + +Intuitive entity construction through method chaining: + +```java +Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("my_table") + .env("PROD") + .description("My dataset") + .build(); + +// Fluent metadata operations with type-safe method chaining +dataset.addTag("pii") + .addOwner("urn:li:corpuser:jdoe", OwnershipType.TECHNICAL_OWNER) + .setDomain("urn:li:domain:Analytics") + .setStructuredProperty("io.acryl.dataQuality.qualityScore", 95.5); + +client.entities().upsert(dataset); +``` + +### 2. Type Safety and Compile-Time Checking + +Leverage Java's strong typing: + +- Strongly-typed URNs (`DatasetUrn`, `ChartUrn`, etc.) +- Generic types for entity operations +- CRTP (Curiously Recurring Template Pattern) for type-safe mixin interfaces +- Builder validation at construction time + +### 3. Mode-Aware Behavior + +**SDK Mode vs INGESTION Mode** for proper separation of concerns: + +- **SDK Mode (default)**: User edits → `editableDatasetProperties` +- **INGESTION Mode**: Pipeline writes → `datasetProperties` +- Getters intelligently prefer editable aspects over system aspects + +```java +// SDK mode - user edits go to editable aspects +DataHubClientV2 client = DataHubClientV2.builder() + .server("http://localhost:8080") + .mode(OperationMode.SDK) // Default + .build(); + +// INGESTION mode - pipeline writes go to system aspects +DataHubClientV2 ingestionClient = DataHubClientV2.builder() + .server("http://localhost:8080") + .mode(OperationMode.INGESTION) + .build(); +``` + +### 4. Patch-First Philosophy + +**Design Decision: Prioritize patches over full aspect replacement** + +The SDK V2 is designed around patch-based operations because they represent the most common and intuitive way to make metadata changes: + +```java +Dataset dataset = client.entities().get(datasetUrn); +Dataset mutable = dataset.mutable(); // Get mutable copy + +// These create patches internally - no server calls yet +mutable.addTag("pii") + .addTag("sensitive") + .addOwner(ownerUrn, OwnershipType.TECHNICAL_OWNER); + +// Single call emits all accumulated patches atomically +client.entities().update(mutable); +``` + +**Why patches?** + +- **Simplicity**: Users think "add a tag" not "fetch all tags, add one, PUT entire tag aspect back" +- **Safety**: Patches don't overwrite concurrent changes from other users +- **Efficiency**: Only changed fields are transmitted and processed +- **Common use case**: Most metadata operations are incremental additions/removals + +**When to use low-level SDK:** +If you need to completely replace an aspect (full PUT/upsert semantics), use the V1 SDK's `RestEmitter` directly with `MetadataChangeProposalWrapper`. The V2 SDK focuses on making common operations simple, not exposing every low-level primitive. + +### 5. Composition Through Mixin Interfaces + +Shared metadata operations via type-safe mixin interfaces: + +- `HasTags` - Add, remove, set tags +- `HasOwners` - Manage ownership +- `HasGlossaryTerms` - Associate glossary terms +- `DomainOperations` - Domain assignment +- `HasContainer` - Parent-child hierarchies + +All mixins use CRTP pattern for type-safe method chaining that returns the concrete entity type. + +## Architecture + +### Package Structure (Actual Implementation) + +``` +datahub-client/ +├── src/main/java/ +│ ├── datahub/client/ # Existing v1 (unchanged) +│ │ ├── Emitter.java +│ │ ├── rest/RestEmitter.java +│ │ └── ... +│ │ +│ └── datahub/client/v2/ # New v2 namespace +│ ├── DataHubClientV2.java # Main client entry point +│ │ +│ ├── entity/ # Entity classes +│ │ ├── Entity.java # Base entity class (490 lines) +│ │ ├── AspectCache.java # Unified cache with dirty tracking (184 lines) +│ │ ├── CachedAspect.java # Aspect wrapper with metadata (68 lines) +│ │ ├── AspectSource.java # SERVER vs LOCAL enum (23 lines) +│ │ ├── ReadMode.java # ALLOW_DIRTY vs SERVER_ONLY (28 lines) +│ │ ├── Dataset.java # Dataset entity (564 lines) +│ │ ├── Chart.java # Chart entity (587 lines) +│ │ ├── Dashboard.java # Dashboard entity (671 lines) +│ │ ├── DataJob.java # DataJob entity (597 lines) +│ │ ├── DataFlow.java # DataFlow entity (467 lines) +│ │ ├── Container.java # Container entity (500 lines) +│ │ ├── MLModel.java # ML Model entity NEW +│ │ ├── MLModelGroup.java # ML Model Group entity NEW +│ │ ├── HasTags.java # Tag operations mixin +│ │ ├── HasOwners.java # Ownership operations mixin +│ │ ├── HasGlossaryTerms.java # Terms operations mixin +│ │ ├── HasDomains.java # Domain operations mixin +│ │ ├── HasContainer.java # Container hierarchy mixin +│ │ └── HasStructuredProperties.java # Structured properties mixin +│ │ +│ ├── operations/ # CRUD operation clients +│ │ └── EntityClient.java # Entity CRUD operations (570 lines) +│ │ +│ └── config/ # Configuration +│ └── DataHubClientConfigV2.java # Config with mode support +│ +└── src/test/java/ # Tests mirror structure + └── datahub/client/v2/ + ├── DataHubClientV2Test.java # Client tests + ├── entity/ # 378 unit tests + │ ├── AspectCacheTest.java # 30 tests (cache infrastructure) + │ ├── CachedAspectTest.java # 13 tests (cache infrastructure) + │ ├── DatasetTest.java # 37 tests + │ ├── ChartTest.java # 43 tests + │ ├── DashboardTest.java # 52 tests + │ ├── DataJobTest.java # 45 tests + │ ├── DataFlowTest.java # 40 tests + │ ├── ContainerTest.java # 40 tests + │ ├── MLModelTest.java # 44 tests + │ └── MLModelGroupTest.java # 38 tests + └── integration/ # 79 integration tests + ├── DatasetIntegrationTest.java + ├── ChartIntegrationTest.java + ├── DashboardIntegrationTest.java + ├── DataJobIntegrationTest.java + ├── DataFlowIntegrationTest.java + ├── ContainerIntegrationTest.java + ├── MLModelIntegrationTest.java + └── MLModelGroupIntegrationTest.java +``` + +**Key Design Decisions:** + +- No separate `patch/` package - patches accumulate internally within entities +- Mixin interfaces in `entity/` package using CRTP pattern for type safety +- Support for 8 entity types including ML entities (MLModel, MLModelGroup) +- Mode-aware configuration for SDK vs INGESTION behavior + +### Core Classes + +#### 1. DataHubClientV2 (Main Entry Point) + +**File**: `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/DataHubClientV2.java` (266 lines) + +```java +package datahub.client.v2; + +/** + * Main entry point for DataHub Java SDK V2. + * Provides high-level operations for entity management with mode-aware behavior. + * + *

Example usage: + * 

+ * DataHubClientV2 client = DataHubClientV2.builder()
+ *     .server("http://localhost:8080")
+ *     .token("my-token")
+ *     .mode(OperationMode.SDK)  // SDK or INGESTION mode
+ *     .build();
+ *
+ * Dataset dataset = Dataset.builder()
+ *     .platform("snowflake")
+ *     .name("my_table")
+ *     .env("PROD")
+ *     .description("My dataset")
+ *     .build();
+ *
+ * client.entities().upsert(dataset);
+ *
+ */ +public class DataHubClientV2 implements AutoCloseable { + private final RestEmitter emitter; + private final DataHubClientConfigV2 config; + private final EntityClient entityClient; + + // Builder for client configuration + public static Builder builder() { ... } + + // Entity operations + public EntityClient entities() { return entityClient; } + + // Low-level emitter access (for advanced users) + public RestEmitter emitter() { return emitter; } + + // Configuration access + public DataHubClientConfigV2 config() { return config; } + + @Override + public void close() throws IOException { ... } + + public static class Builder { + public Builder server(String serverUrl) { ... } + public Builder token(String token) { ... } + public Builder timeout(int timeoutMs) { ... } + public Builder mode(OperationMode mode) { ... } // NEW + public Builder config(DataHubClientConfigV2 config) { ... } + public DataHubClientV2 build() { ... } + } +} +``` + +**Design Features:** + +- Mode-aware behavior (SDK vs INGESTION) for proper aspect routing +- Environment variable support for configuration +- Builder pattern with sensible defaults +- AutoCloseable interface for resource management + +#### 2. Entity (Base Class) - User-Facing API + +**File**: `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/entity/Entity.java` (490 lines) + +The Entity base class provides a unified interface for all DataHub entities. From a user perspective, all entities support: + +**Public API Methods:** + +```java +// URN access +public Urn getUrn() +public abstract String getEntityType() + +// Convert to MCPs for emission (primarily internal) +public List toMCPs() +``` + +**Entity Construction:** + +Entities are constructed via fluent builders: + +```java +Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("my_table") + .env("PROD") + .description("My dataset") + .build(); +``` + +**Fluent Metadata Operations:** + +All entities support method chaining for metadata operations (via mixin interfaces): + +```java +dataset.addTag("pii") + .addOwner(ownerUrn, OwnershipType.TECHNICAL_OWNER) + .setDomain(domainUrn) + .addTerm(termUrn); +``` + +**Lazy Loading:** + +Entities loaded from the server fetch aspects on-demand: + +```java +Dataset dataset = client.entities().get(datasetUrn); // Only URN loaded +String description = dataset.getDescription(); // Aspect fetched now +List tags = dataset.getTags(); // Another aspect fetch +``` + +**Patch Accumulation:** + +Metadata operations create patches that accumulate until save: + +```java +Dataset dataset = client.entities().get(datasetUrn); +Dataset mutable = dataset.mutable(); // Get mutable copy +mutable.addTag("pii"); // Creates patch (not sent yet) +mutable.addTag("sensitive"); // Another patch (not sent yet) +client.entities().update(mutable); // Emits all patches atomically +``` + +**Immutability-by-Default:** + +Entities fetched from the server are read-only to prevent accidental mutations: + +```java +Dataset dataset = client.entities().get(datasetUrn); +dataset.isReadOnly(); // true +dataset.isMutable(); // false + +// Attempting mutation throws ReadOnlyEntityException +// dataset.addTag("pii"); // ERROR! + +// Get mutable copy for updates +Dataset mutable = dataset.mutable(); +mutable.isMutable(); // true +mutable.addTag("pii"); // Works +client.entities().upsert(mutable); +``` + +**Entity Lifecycle:** + +1. **Builder-created entities** - Mutable from creation + + ```java + Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("my_table") + .build(); + dataset.isMutable(); // true - can mutate immediately + ``` + +2. **Server-fetched entities** - Immutable by default + + ```java + Dataset dataset = client.entities().get(urn); + dataset.isReadOnly(); // true - must call .mutable() + ``` + +3. **Mutable copies** - Created via `.mutable()` + ```java + Dataset mutable = dataset.mutable(); + mutable.isMutable(); // true - can mutate + ``` + +**The .mutable() method:** + +- Creates a shallow copy with independent mutability flags +- Shares aspect cache with original (read-your-own-writes semantics) +- Idempotent - returns self if already mutable +- Original entity remains read-only after creating mutable copy + +**Why immutability-by-default?** + +- Makes mutations explicit and intentional +- Prevents accidental modification when passing entities between functions +- Clear separation between read and write workflows +- Enables safe entity sharing across threads +- Common pattern in modern APIs (Rust, Python, Java immutable collections) + +See "Developer-Facing Implementation Design" section below for internal architecture details. + +#### 3. Supported Entities + +The SDK V2 implements 8 entity types with full metadata support: + +**Data Entities:** + +- **Dataset** - Tables, views, files with schema support +- **Container** - Databases, schemas, folders (hierarchical structures) + +**Pipeline Entities:** + +- **DataFlow** - Pipelines, workflows (Airflow DAGs, Spark jobs, dbt projects) +- **DataJob** - Individual tasks with inlet/outlet lineage + +**Visualization Entities:** + +- **Chart** - Visualizations with input dataset lineage +- **Dashboard** - Dashboards with chart relationships and input datasets + +**ML Entities:** + +- **MLModel** - Machine learning models with metrics, hyperparameters, training jobs +- **MLModelGroup** - Model families with version management + +**Common Entity Operations:** + +All entities support these fluent operations (via mixin interfaces): + +```java +// Tags +entity.addTag("pii") + .removeTag("deprecated") + .setTags(Arrays.asList("tag1", "tag2")) + .clearTags() + +// Owners +entity.addOwner(ownerUrn, OwnershipType.TECHNICAL_OWNER) + .removeOwner(ownerUrn) + .setOwners(ownerList) + .clearOwners() + +// Glossary Terms +entity.addTerm(termUrn) + .removeTerm(termUrn) + .setTerms(termList) + .clearTerms() + +// Domains +entity.setDomain(domainUrn) + .removeDomain(domainUrn) + .clearDomains() + +// Container (for hierarchical entities) +entity.setContainer(containerUrn) + .clearContainer() + +// Structured Properties (custom typed metadata) +entity.setStructuredProperty("io.acryl.dataManagement.replicationSLA", "24h") + .setStructuredProperty("io.acryl.dataQuality.qualityScore", 95.5) + .setStructuredProperty("io.acryl.dataManagement.certifications", + Arrays.asList("SOC2", "HIPAA", "GDPR")) + .setStructuredProperty("io.acryl.privacy.retentionDays", 90, 180, 365) + .removeStructuredProperty("io.acryl.dataManagement.deprecated") +``` + +**Entity-Specific Documentation:** + +See comprehensive guides in `metadata-integration/java/docs/sdk-v2/`: + +- `dataset-entity.md` - Dataset with schema support +- `chart-entity.md` - Chart with lineage +- `dashboard-entity.md` - Dashboard with chart relationships +- `container-entity.md` - Container hierarchies +- `dataflow-entity.md` - DataFlow pipelines +- `datajob-entity.md` - DataJob with inlet/outlet lineage +- `mlmodel-entity.md` - MLModel with metrics +- `mlmodelgroup-entity.md` - MLModelGroup with versions + +#### 4. EntityClient (CRUD Operations) + +**File**: `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/operations/EntityClient.java` (570 lines) + +```java +package datahub.client.v2.operations; + +/** + * Client for entity CRUD operations. + * Provides create, read, update, and upsert operations. + */ +public class EntityClient { + private final RestEmitter emitter; + private final DataHubClientConfigV2 config; + + /** + * Create a new entity (convenience method - same as upsert). + */ + public void create(T entity) throws IOException, ExecutionException, InterruptedException { + upsert(entity); + } + + /** + * Upsert an entity (create or update). + * Emits all aspects and accumulated patches. + */ + public void upsert(T entity) throws IOException, ExecutionException, InterruptedException { + List mcps = entity.toMCPs(); + // Emit all MCPs asynchronously and wait for completion + // ... + } + + /** + * Update an existing entity. + * Emits only accumulated patches (not full aspects). + */ + public void update(T entity) throws IOException, ExecutionException, InterruptedException { + // Emit only pending patches + // ... + } + + /** + * Get an entity by URN. + * Returns entity with lazy-loaded aspects. + */ + public T get(Urn urn, Class entityClass) throws IOException { + // Fetch entity aspects from server + // Construct entity with lazy loading support + // ... + } + + // Note: delete(Urn) and exists(Urn) operations deferred to future releases +} +``` + +**Supported Operations:** + +- `create()` - Create new entities (wrapper for upsert) +- `upsert()` - Create or update entities (emits all aspects + patches) +- `update()` - Update existing entities (emits only patches) +- `get()` - Retrieve entities with lazy loading +- `delete()` and `exists()` - Deferred to future releases + +**Patch Behavior:** + +Patches are accumulated **inside entities** during metadata operations and emitted automatically during `upsert()`/`update()`: + +```java +Dataset dataset = client.entities().get(datasetUrn); +Dataset mutable = dataset.mutable(); // Get mutable copy +mutable.addTag("pii"); // Creates internal patch +mutable.addTag("sensitive"); // Creates another internal patch +client.entities().update(mutable); // Emits both patches atomically +``` + +There is **no separate `patch()` method** - patches are managed internally by entities. + +#### 5. Mixin Interfaces (CRTP Pattern) + +**Files**: `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/entity/Has*.java` + +Mixin interfaces provide reusable metadata operations across entities using the **Curiously Recurring Template Pattern (CRTP)** for type-safe method chaining: + +```java +/** + * Interface for entities that support tags. + * Uses CRTP for type-safe method chaining. + */ +public interface HasTags> { + + /** + * Add a tag to this entity. + * Creates a patch that will be emitted on save. + */ + default T addTag(@Nonnull String tagUrn) { + // Implementation creates patch internally + return (T) this; + } + + default T removeTag(@Nonnull String tagUrn) { ... } + default T setTags(@Nonnull List tagUrns) { ... } + default T clearTags() { ... } + + // Getter methods + default List getTags() { ... } +} +``` + +**Available Mixin Interfaces:** + +1. **`HasTags`** - Tag operations (`addTag`, `removeTag`, `setTags`, `clearTags`) +2. **`HasOwners`** - Ownership operations (`addOwner`, `removeOwner`, `setOwners`, `clearOwners`) +3. **`HasGlossaryTerms`** - Glossary term operations (`addTerm`, `removeTerm`, `setTerms`, `clearTerms`) +4. **`DomainOperations`** - Domain operations (`setDomain`, `removeDomain`, `clearDomains`) +5. **`HasContainer`** - Container hierarchy (`setContainer`, `clearContainer`) +6. **`HasStructuredProperties`** - Structured properties operations (`setStructuredProperty`, `removeStructuredProperty`) + +**Why CRTP?** + +The CRTP pattern enables type-safe method chaining that returns the concrete entity type: + +```java +// Without CRTP: returns Entity +Entity entity = dataset.addTag("pii"); // Loses Dataset type! + +// With CRTP: returns Dataset +Dataset dataset = dataset.addTag("pii") + .addOwner(ownerUrn, type) // Still Dataset type! + .setDomain(domainUrn); // Still Dataset type! +``` + +**Entity Implementations:** + +Entities implement mixin interfaces by declaring them in the class signature: + +```java +public class Dataset extends Entity + implements HasTags, + HasOwners, + HasGlossaryTerms, + DomainOperations, + HasContainer, + HasStructuredProperties { + // Mixin methods provided by default implementations +} +``` + +--- + +# Part 2: Developer-Facing Implementation Design + +This section describes the internal architecture and implementation details for developers contributing to the SDK. + +## Internal Architecture + +### Entity Base Class - Internal Implementation + +**File**: `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/entity/Entity.java` (490 lines) + +The Entity base class implements three core subsystems: + +#### 1. AspectCache System with Read-Your-Own-Writes + +**Unified Cache Architecture**: The SDK uses a unified `AspectCache` that provides read-your-own-writes semantics with proper dirty tracking. This architecture fixes bugs where fetched aspects would override patches. + +**Core Implementation Files:** + +- `AspectCache.java` (184 lines) - Main cache with dirty tracking +- `CachedAspect.java` (68 lines) - Aspect wrapper with metadata +- `AspectSource.java` (23 lines) - Enum for SERVER vs LOCAL aspects +- `ReadMode.java` (28 lines) - Enum for ALLOW_DIRTY vs SERVER_ONLY reads + +**Key Architectural Features:** + +1. **AspectSource Tracking**: Distinguishes between SERVER-fetched aspects (subject to TTL) and LOCAL-created aspects (no expiration) + +2. **Dirty Tracking**: Explicit marking of aspects that need write-back to server via `markDirty()` method + +3. **Read-Your-Own-Writes**: Default `ReadMode.ALLOW_DIRTY` returns local modifications immediately, `SERVER_ONLY` mode skips dirty aspects + +4. **TTL Management**: 60-second TTL enforced only for SERVER-sourced aspects, LOCAL aspects never expire + +5. **Thread Safety**: Uses `ConcurrentHashMap` for safe concurrent access + +**Internal State (Entity.java):** + +``` +protected final AspectCache cache; // Unified cache with dirty tracking +protected final Map> pendingPatches; +private DataHubClientV2 boundClient = null; +``` + +**Cache Operations:** + +- `getAspectLazy()` - Lazy loads from server, stores as clean SERVER-sourced aspect +- `getOrCreateAspect()` - Gets from cache or creates new LOCAL-sourced aspect (marked dirty) +- `markAspectDirty()` - Marks aspect dirty after in-place modification (used by domain operations) +- `toMCPs()` - Returns **only dirty aspects** for emission (excludes clean fetched aspects) + +**Why This Architecture?** + +The unified cache solves a critical bug: when entities are fetched from the server and then patch operations are applied (e.g., `removeTerm()`), the cached aspect would be included in `toMCPs()` and override the patches. With dirty tracking, `toMCPs()` only returns modified aspects, allowing patches to work correctly. + +#### 2. Patch Accumulation and MCP Generation + +Metadata operations create patches that accumulate until emission. The system supports two types of operations: + +**Patch-Based Operations** (incremental updates): + +- Tags, owners, glossary terms use `PatchBuilder` classes +- Patches accumulate in `pendingPatches` map (aspect name → list of patches) +- Multiple operations on same aspect create multiple patches + +**Cache-Based Operations** (full aspect replacement): + +- Domains, custom properties modify aspects in cache +- Aspects marked dirty via `markAspectDirty()` after modification +- Dirty aspects included in `toMCPs()` output + +**MCP Generation:** + +The `toMCPs()` method returns **only dirty aspects** and accumulated patches: + +``` +public List toMCPs() { + // 1. Add dirty aspects from cache (excludes clean fetched aspects) + for (Map.Entry entry : cache.getDirtyAspects().entrySet()) { + mcps.add(createMCP(entry.getKey(), entry.getValue())); + } + + // 2. Add accumulated patches + for (PatchBuilder builder : patchBuilders.values()) { + mcps.add(builder.build()); + } + + // 3. Add pending MCPs + mcps.addAll(pendingMCPs); + + return mcps; +} +``` + +**Critical Design Point**: `toMCPs()` uses `cache.getDirtyAspects()` instead of all cached aspects. This ensures that fetched aspects don't override patches - only locally modified aspects are emitted. + +#### 3. Mode-Aware Aspect Routing + +SDK mode vs INGESTION mode for proper aspect selection: + +````java +/** + * Get aspect name based on operation mode. + * SDK mode: prefer editable aspects + * INGESTION mode: use system aspects + */ +protected String getAspectName(Class aspectClass, OperationMode mode) { + if (mode == OperationMode.SDK) { + // Check if editable variant exists + String editableAspectName = getEditableAspectName(aspectClass); + if (editableAspectName != null) { + return editableAspectName; + } + } + return aspectClass.getSimpleName(); +} + +/** + * Get getter preference order: editable aspects first, then system aspects. + */ +protected T getAspectWithPreference( + Class editableClass, + Class systemClass +) { + // Try editable aspect first + T editable = getAspectLazy(editableClass); + if (editable != null) { + return editable; + } + + // Fall back to system aspect + return getAspectLazy(systemClass); +} + +## Implementation Phases + +### Phase 1: Core Framework + +Base functionality for all entities: + +- Base `Entity` class with aspect management, lazy loading, and patch accumulation +- `DataHubClientV2` main client class with mode-aware behavior +- `EntityClient` with create, read, update, upsert operations +- Configuration classes with environment variable support +- Mixin interfaces using CRTP pattern for type safety + +### Phase 2: Dataset Entity + +Reference implementation demonstrating all patterns: + +- `Dataset` entity with fluent builder +- Dataset-specific aspects (properties, schema, lineage) +- Mixin interface implementations +- Comprehensive unit tests + +### Phase 3: Additional Entities + +Seven additional entity types: + +- `Chart` - Visualizations with lineage +- `Dashboard` - Dashboards with chart relationships +- `Container` - Hierarchical data structures +- `DataJob` - Pipeline tasks with inlet/outlet lineage +- `DataFlow` - Pipeline workflows +- `MLModel` - Machine learning models +- `MLModelGroup` - ML model families + +### Phase 4: Patch Capabilities + +Patch-based updates for efficient metadata changes: + +- Internal patch accumulation within entities (not separate patch builders) +- Automatic patch emission on `update()` and `upsert()` +- Leverages existing `PatchBuilder` classes from entity-registry module +- Patches tested via entity unit tests + +### Phase 5: Testing & Documentation + +Comprehensive validation and user guides: + +- Integration tests with live DataHub server +- API documentation (Javadoc) and 13 comprehensive Markdown guides +- 19 working example files demonstrating real-world usage +- Migration guide from V1 +- Design principles document +- Patch operations deep-dive +- Entity-specific guides for all 8 entities + +## Testing Strategy + +### Unit Tests + +Each entity and component has comprehensive unit tests: + +- Builder validation (required fields, optional fields, validation logic) +- Aspect management (getters, setters, mode-aware routing) +- MCP generation (full aspects + patches) +- Patch operations (accumulation, emission) +- Fluent API chaining (type safety via CRTP) +- Mixin operations (tags, owners, terms, domains) + +**Test Coverage by Entity:** +- Dataset: 37 tests +- Chart: 43 tests +- Dashboard: 52 tests +- DataJob: 45 tests +- DataFlow: 40 tests +- Container: 40 tests +- MLModel: 44 tests +- MLModelGroup: 38 tests + +### Integration Tests + +Full end-to-end tests against a real DataHub instance: + +```java +@Test +public void testDatasetCreateAndRead() throws Exception { + // Create client + DataHubClientV2 client = DataHubClientV2.builder() + .server(TEST_SERVER) + .token(TEST_TOKEN) + .build(); + + // Create dataset + Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("db.schema.test_table_" + System.currentTimeMillis()) + .env("PROD") + .description("Test dataset created by Java SDK V2") + .build(); + + dataset.addTag("test-tag") + .addOwner("urn:li:corpuser:datahub", OwnershipType.TECHNICAL_OWNER); + + // Upsert + client.entities().upsert(dataset); + + // Read back + Dataset retrieved = client.entities().get(dataset.getUrn(), Dataset.class); + assertNotNull(retrieved); + assertEquals("Test dataset created by Java SDK V2", retrieved.getDescription()); +} + +@Test +public void testDatasetPatchOperations() throws Exception { + DataHubClientV2 client = DataHubClientV2.builder() + .server(TEST_SERVER) + .token(TEST_TOKEN) + .build(); + + // Create dataset first + Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("db.schema.test_table_patch_" + System.currentTimeMillis()) + .env("PROD") + .build(); + client.entities().upsert(dataset); + + // Retrieve and apply patches + Dataset retrieved = client.entities().get(dataset.getUrn(), Dataset.class); + Dataset mutable = retrieved.mutable(); // Get mutable copy + mutable.addTag("pii") // Creates patch + .addTag("sensitive") // Another patch + .addTerm("urn:li:glossaryTerm:CustomerData"); // Another patch + + // All patches emitted atomically + client.entities().update(mutable); + + // Verify patches were applied + Dataset verified = client.entities().get(dataset.getUrn(), Dataset.class); + assertTrue(verified.getTags().contains("urn:li:tag:pii")); +} +```` + +**Integration Test Coverage:** + +- Entity creation and retrieval +- Tag, owner, term, domain operations +- Lineage relationships (charts → datasets, jobs → datasets) +- Custom properties +- Full metadata workflows +- Batch operations +- Patch accumulation and emission + +**Running Integration Tests:** + +```bash +export DATAHUB_SERVER=http://localhost:8080 +export DATAHUB_TOKEN=your_token + +./gradlew :metadata-integration:java:datahub-client:test --tests "*Integration*" +``` + +### Test Coverage Results + +- Unit test coverage: **>80%** for new code (378 unit tests + 79 integration tests = 457 total) +- All public APIs covered +- Edge cases tested (null values, invalid inputs, mode switching) +- Async operations tested with proper synchronization +- Cache infrastructure thoroughly tested (43 tests for AspectCache + CachedAspect) +- Full end-to-end integration tests (79 tests) + +## API Documentation + +All public classes and methods have comprehensive Javadoc plus extensive Markdown documentation: + +**Javadoc Coverage:** + +- Class-level documentation explaining purpose and usage +- Method-level documentation with parameters, returns, exceptions +- Code examples for common use cases +- Links to related classes and methods + +**Markdown Documentation (13 files):** + +Located in `metadata-integration/java/docs/sdk-v2/`: + +1. **getting-started.md** - Quick start guide for new users +2. **design-principles.md** - Architecture and design decisions +3. **dataset-entity.md** - Dataset operations and schema support +4. **chart-entity.md** - Chart operations and lineage +5. **dashboard-entity.md** - Dashboard operations and relationships +6. **container-entity.md** - Container hierarchies +7. **dataflow-entity.md** - DataFlow pipeline operations +8. **datajob-entity.md** - DataJob inlet/outlet lineage +9. **mlmodel-entity.md** - MLModel metrics and hyperparameters +10. **mlmodelgroup-entity.md** - MLModelGroup version management +11. **patch-operations.md** - Deep dive into patch-based updates +12. **migration-from-v1.md** - Migration guide from V1 SDK +13. **java-sdk-v2-design.md** - This comprehensive design document + +**Working Examples (19 files):** + +Located in `metadata-integration/java/examples/src/main/java/io/datahubproject/examples/v2/`: + +- Dataset examples: DatasetCreateExample, DatasetFullExample, DatasetPatchExample +- Chart examples: ChartCreateExample, ChartFullExample, ChartLineageExample +- Dashboard examples: DashboardCreateExample, DashboardFullExample, DashboardLineageExample +- DataFlow examples: DataFlowCreateExample, DataFlowFullExample +- DataJob examples: DataJobCreateExample, DataJobFullExample, DataJobLineageExample +- Container examples: ContainerCreateExample, ContainerFullExample, ContainerHierarchyExample +- MLModel examples: MLModelCreateExample, MLModelFullExample +- MLModelGroup examples: MLModelGroupCreateExample, MLModelGroupFullExample + +## Migration Guide + +For users of the existing Java SDK: + +### Before (V1): + +```java +RestEmitter emitter = RestEmitter.create(b -> b.server("http://localhost:8080")); + +DatasetUrn urn = new DatasetUrn( + new DataPlatformUrn("postgres"), + "my_table", + FabricType.PROD +); + +DatasetProperties props = new DatasetProperties(); +props.setDescription("My dataset"); + +MetadataChangeProposalWrapper mcpw = MetadataChangeProposalWrapper.builder() + .entityType("dataset") + .entityUrn(urn) + .upsert() + .aspect(props) + .build(); + +emitter.emit(mcpw).get(); +``` + +### After (V2): + +```java +DataHubClientV2 client = DataHubClientV2.builder() + .server("http://localhost:8080") + .build(); + +Dataset dataset = Dataset.builder() + .platform("postgres") + .name("my_table") + .description("My dataset") + .build(); + +client.entities().upsert(dataset); +``` + +## Decision Log + +### 1. Use Pegasus Models vs OpenAPI Models + +**Decision**: Use Pegasus models (`com.linkedin.*`) for aspect classes. + +**Rationale**: + +- Pegasus models are the canonical representation in DataHub +- Already used by v1 SDK, maintains consistency +- Generated from PDL schemas, always in sync with backend +- OpenAPI models are less mature and have fewer utilities + +**Result**: Proven correct - seamless integration with existing infrastructure. + +### 2. Namespace Separation + +**Decision**: Use `datahub.client.v2.*` namespace. + +**Rationale**: + +- Clear separation from v1 API +- Allows side-by-side usage +- Follows semantic versioning principles +- Easy to deprecate v1 in future + +**Result**: 100% backward compatibility achieved - v1 code unchanged. + +### 3. Builder Pattern + +**Decision**: Use nested static Builder classes. + +**Rationale**: + +- Idiomatic Java pattern +- Type-safe construction +- Optional parameters handled cleanly +- Better than telescoping constructors + +**Result**: Excellent developer experience with fluent API. + +### 4. Synchronous vs Async + +**Decision**: Provide synchronous API that wraps async operations. + +**Rationale**: + +- Simpler for most users +- Matches Python SDK V2 API +- Can expose async API later for advanced users +- RestEmitter already provides async primitives + +**Result**: Simplified API widely adopted in examples and tests. + +### 5. Error Handling + +**Decision**: Throw checked exceptions for I/O operations. + +**Rationale**: + +- Forces callers to handle errors +- Consistent with Java conventions +- Clear distinction between programmer errors and runtime failures + +**Result**: Clear error handling patterns in all code. + +**Exception Hierarchy:** + +The SDK introduces custom exceptions for common error conditions: + +**ReadOnlyEntityException** - Thrown when attempting to mutate a read-only entity: + +```java +try { + Dataset dataset = client.entities().get(urn); + dataset.addTag("pii"); // Throws ReadOnlyEntityException +} catch (ReadOnlyEntityException e) { + // Exception message explains the issue and provides fix + System.err.println(e.getMessage()); + + // Fix: Get mutable copy first + Dataset mutable = dataset.mutable(); + mutable.addTag("pii"); + client.entities().upsert(mutable); +} +``` + +**PendingMutationsException** - Thrown when reading from entity with pending mutations: + +```java +Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("my_table") + .build(); + +dataset.setDescription("New description"); +// dataset.getDescription(); // Throws PendingMutationsException! + +// Fix: Save first, then read +client.entities().upsert(dataset); // Clears dirty flag +String desc = dataset.getDescription(); // Now works +``` + +**Why these restrictions?** + +- **ReadOnlyEntityException**: Makes mutations explicit, prevents accidental changes when passing entities between functions +- **PendingMutationsException**: Prevents reading stale cached data, enforces explicit save-then-fetch workflow + +Both restrictions enforce clear separation between read and write workflows. These may be relaxed in future versions as the API matures and usage patterns emerge. + +### 6. Patch-First over Full Aspect Replacement + +**Decision**: Prioritize patch-based operations as the primary API, defer full aspect replacement to V1 SDK. + +**Rationale**: + +- **User mental model**: "Add a tag" is more natural than "fetch all tags, modify list, PUT entire aspect" +- **Safety**: Patches don't clobber concurrent changes from other users/systems +- **Simplicity**: Most metadata operations are incremental (add owner, remove tag, etc.) +- **Efficiency**: Only changed fields transmitted and processed by server +- **Escape hatch exists**: Users needing full PUT semantics can use V1 SDK's `RestEmitter` directly + +**Why not both?** +V2 SDK focuses on making common operations simple, not exposing every low-level primitive. This keeps the API focused and prevents confusion about when to use patches vs full replacement. + +**Result**: Clean, intuitive API for 95% of use cases. Power users can drop to V1 SDK for remaining 5%. + +### 7. Internal Patch Accumulation vs External Patch Builders + +**Decision**: Accumulate patches **inside entities** rather than separate patch builder classes. + +**Rationale**: + +- More intuitive API - metadata operations just work +- Patches automatically emitted on save +- Reduces API surface area +- Simplifies user code + +**Original Design**: Separate `DatasetPatch`, `ChartPatch` builder classes + +**Actual Implementation**: Patches accumulate in `Entity.pendingPatches` and emit via `toMCPs()` + +**Result**: Superior developer experience - no need to learn separate patch API. + +### 8. CRTP Pattern for Mixin Interfaces + +**Decision**: Use Curiously Recurring Template Pattern for type-safe mixin interfaces. + +**Rationale**: + +- Type-safe method chaining returns concrete entity type +- Compile-time type checking +- No casting required in user code +- Idiomatic Java generics pattern + +**Original Design**: Simple interfaces returning `Entity` + +**Actual Implementation**: + +```java +public interface HasTags> { + default T addTag(String tagUrn) { return (T) this; } +} +``` + +**Result**: Excellent type safety and developer experience. + +### 9. Mode-Aware Behavior (SDK vs INGESTION) + +**Decision**: Support SDK mode and INGESTION mode for aspect routing. + +**Rationale**: + +- Proper separation of user edits vs pipeline writes +- SDK mode → editable aspects (user overrides) +- INGESTION mode → system aspects (pipeline data) +- Getters prefer editable over system + +**Original Design**: Not specified + +**Actual Implementation**: `OperationMode` enum with aspect routing logic + +**Result**: Clear separation of concerns, aligns with DataHub's aspect model. + +### 10. Lazy Loading for GET Operations + +**Decision**: Implement lazy loading for aspects when entities are retrieved. + +**Rationale**: + +- Performance - only fetch aspects when accessed +- Client binding enables on-demand fetching +- Cache management with timestamps + +**Original Design**: Not specified (GET deferred) + +**Actual Implementation**: Full lazy loading with `getAspectLazy()` and client binding + +**Result**: Efficient entity retrieval with on-demand aspect fetching. + +## Design Questions and Resolutions + +1. **GET operation implementation**: Should we implement REST client for reading entities, or defer to future? + + - **Resolution**: Implemented with lazy loading support + +2. **Search client**: Should we include search functionality in V2? + + - **Resolution**: Deferred to future (out of scope for V2) + +3. **Lineage client**: Should we include lineage management? + + - **Resolution**: Basic lineage on Dataset, Chart, Dashboard, DataJob entities + +4. **Schema field builders**: Should we provide fluent builders for schema fields? + - **Resolution**: Yes, schema field support in Dataset entity + +## References + +- [Python SDK V2 Implementation](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion/src/datahub/sdk) +- [Existing Java SDK](https://github.com/datahub-project/datahub/tree/master/metadata-integration/java/datahub-client) +- [DataHub Metadata Model](https://github.com/datahub-project/datahub/tree/master/metadata-models) + +## Quick Links for Reviewers + +**Start Here:** + +1. `metadata-integration/java/docs/sdk-v2/getting-started.md` - Quick start guide +2. `metadata-integration/java/docs/sdk-v2/design-principles.md` - Architecture overview + +**Core Implementation:** 3. `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/entity/Entity.java` (490 lines) - Base entity class 4. `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/operations/EntityClient.java` (570 lines) - CRUD operations 5. `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/DataHubClientV2.java` (266 lines) - Main client + +**Sample Entities:** 6. `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/entity/Dataset.java` (564 lines) - Reference implementation 7. `metadata-integration/java/datahub-client/src/main/java/datahub/client/v2/entity/HasTags.java` (145 lines) - CRTP mixin example + +**Examples:** 8. `metadata-integration/java/examples/src/main/java/io/datahubproject/examples/v2/DatasetFullExample.java` - Complete workflow 9. `metadata-integration/java/examples/src/main/java/io/datahubproject/examples/v2/ChartLineageExample.java` - Lineage relationships + +**Tests:** 10. `metadata-integration/java/datahub-client/src/test/java/datahub/client/v2/entity/DatasetTest.java` (37 unit tests) 11. `metadata-integration/java/datahub-client/src/test/java/datahub/client/v2/integration/DatasetIntegrationTest.java` - End-to-end validation + +--- + +**Document Status**: Design document reflecting implemented architecture (includes AspectCache refactoring) +**Author**: DataHub OSS Team +**Last Updated**: 2025-01-06 diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-philosophy.md b/docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-philosophy.md new file mode 100644 index 00000000..f2dae9ca --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/developers/java-sdk-v2-philosophy.md @@ -0,0 +1,372 @@ +--- +title: Why We Hand-Crafted the Java SDK V2 (Instead of Generating It) +slug: /developers/java-sdk-v2-philosophy +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/developers/java-sdk-v2-philosophy.md +--- +# Why We Hand-Crafted the Java SDK V2 (Instead of Generating It) + +## The Question + +When building DataHub's Java SDK V2, we faced a choice that every API platform eventually confronts: should we generate our SDK from OpenAPI specs, or hand-craft it? + +OpenAPI code generation is seductive. Tools like OpenAPI Generator promise instant SDKs in dozens of languages. Run a command, get a client—complete with type-safe models, proper serialization, and comprehensive endpoint coverage. Why would anyone choose to write thousands of lines of code by hand? + +We chose to hand-craft. This document explains why. + +## When Code Generation Works Beautifully + +Let's be clear: code generation isn't wrong. It's incredibly effective when your abstraction boundary aligns with your wire protocol. + +**CRUD APIs**: If your API exposes resources like `GET /users/{id}`, `POST /users`, `DELETE /users/{id}`, a generated client is perfect: + +```java +User user = client.getUser(123); +client.createUser(newUser); +client.deleteUser(456); +``` + +The user's mental model—"I want to fetch/create/delete a user"—maps directly to HTTP operations. There's no translation needed. + +**Protocol Buffers**: Google's protobuf generators are exemplary because the `.proto` file **is** the contract: + +```protobuf +service UserService { + rpc GetUser(UserId) returns (User); + rpc ListUsers(ListRequest) returns (UserList); +} +``` + +The service definition becomes the client API with perfect fidelity. What you define is what users get. + +**The Pattern**: Code generation excels when **the API's conceptual model matches user mental models**, and the wire protocol fully captures domain semantics. + +## The Semantic Gap: Why DataHub Is Different + +DataHub doesn't fit this mold. Our metadata platform has a semantic gap between what users want to do and what the HTTP API exposes. + +### The Aspect-Based Model + +DataHub stores metadata as discrete "aspects"—properties, tags, ownership, schemas. But users don't think in aspects. They think: + +- "I want to add a 'PII' tag to this dataset" +- "I need to assign ownership to John" +- "This table should be in the Finance domain" + +An OpenAPI-generated client would expose: + +```java +// What the API provides +client.updateGlobalTags(entityUrn, globalTagsPayload); +client.updateOwnership(entityUrn, ownershipPayload); +``` + +But to use this, you need to know: + +- What is `GlobalTags`? How do I construct it? +- Should I use PUT (full replacement) or PATCH (incremental update)? +- How do I avoid race conditions when multiple systems update tags? +- Where do tags even live—in system aspects or editable aspects? + +This is expert-level knowledge pushed onto every user. + +### The Patch Complexity + +DataHub supports both full aspect replacement (PUT) and JSON Patch (incremental updates). The generated client would expose both: + +```java +// Full replacement +void putGlobalTags(Urn entityUrn, GlobalTags tags); + +// JSON Patch +void patchGlobalTags(Urn entityUrn, JsonPatch patch); +``` + +Now users must decide when to use each. Patches are safer (no race conditions), but how do you construct a JsonPatch? Do you use a PatchBuilder? Hand-write JSON? + +Every user solves this problem independently, reinventing best practices. + +### The Mode Problem + +DataHub has dual aspects: **system aspects** (written by ingestion pipelines) and **editable aspects** (written by humans via UI/SDK). Users editing metadata should write to editable aspects, but pipelines should write to system aspects. + +A generated client doesn't understand this distinction. It just exposes endpoints. Users must learn DataHub's aspect model to route correctly. + +## Five Principles of Hand-Crafted SDKs + +Our hand-crafted SDK addresses these gaps through five design principles. + +### 1. Semantic Layers Translate Domain Concepts + +The SDK provides operations that match how users think: + +```java +Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("fact_revenue") + .build(); + +// Think "add a tag", not "construct and PUT a GlobalTags aspect" +dataset.addTag("pii"); + +// Think "assign ownership", not "build an Ownership aspect" +dataset.addOwner("urn:li:corpuser:jdoe", OwnershipType.TECHNICAL_OWNER); + +client.entities().upsert(dataset); +``` + +The SDK translates `addTag()` into the correct: + +- Aspect type (GlobalTags) +- Operation type (JSON Patch for safety) +- Aspect variant (editable, in SDK mode) +- JSON path (into the aspect structure) + +This is **semantic translation**—mapping domain intent to wire protocol. Generators can't do this because the semantics live in institutional knowledge, not OpenAPI specs. + +### 2. Opinionated APIs: The 95/5 Rule + +We optimized for the 95% case and provided escape hatches for the 5%. + +**The 95% case**: Incremental metadata changes—add a tag, update ownership, set a domain. + +```java +dataset.addTag("sensitive") + .addOwner(ownerUrn, type) + .setDomain(domainUrn); + +client.entities().update(dataset); +``` + +Users never think about PUT vs PATCH, aspect construction, or batch strategies. It just works. + +**The 5% case**: Complete aspect replacement, custom MCPs, or operations V2 doesn't support. + +```java +// Drop to V1 SDK for full control +RestEmitter emitter = client.emitter(); +MetadataChangeProposalWrapper mcpw = /* custom logic */; +emitter.emit(mcpw).get(); +``` + +This philosophy—**make simple things trivial, complex things possible**—requires intentional API design. Generators produce flat API surfaces where every operation has equal weight. + +### 3. Encoding Expert Knowledge + +Every platform accumulates tribal knowledge: + +- "Always use patches for concurrent-safe updates" +- "Editable aspects override system aspects in SDK mode" +- "Batch operations to avoid Kafka load spikes" +- "Schema field names don't always match aspect names" + +A generated client leaves this knowledge in Slack threads and documentation. Users discover best practices through painful trial and error. + +The hand-crafted SDK **encodes** this knowledge: + +```java +// Users call addTag(), SDK internally: +// - Creates a JSON Patch (not full replacement) +// - Targets the editable aspect in SDK mode +// - Accumulates patches for atomic emission +// - Uses the correct field paths +``` + +The SDK becomes **executable documentation** of best practices. This scales better than tribal knowledge. + +### Why Not an ORM Approach? + +Tools like Hibernate, SQLAlchemy, and Pydantic+ORM excel at managing complex object graphs in transactional applications. Why didn't we use this pattern? + +**Metadata operations follow different patterns than OLTP workloads:** + +1. **Bulk mutations** - "Tag 50 datasets as PII" requires only URNs and the operation, not loading full object graphs +2. **Point lookups** - "Get this dataset's schema before querying" is a direct fetch, no relationship navigation needed +3. **Read-modify-write** - "Infer quality scores from schema statistics" involves fetching an aspect, transforming it, and patching it back + +ORMs optimize for relationship traversal (`dataset.container.database.catalog`), session lifecycle management, and automatic dirty tracking. But: + +- **Relationship traversal** is handled by DataHub's search and graph query APIs, not in-memory navigation +- **Explicit patches** are central to our design—we want `addTag()` visible in code, not hidden behind session flush +- **Session complexity** adds cognitive overhead without benefit for metadata's bulk/point/patch patterns + +The result: a simpler, more explicit API that matches how developers actually work with metadata. + +### 4. Centralized Maintenance vs Distributed Pain + +Generated clients push maintenance costs onto users. When we improve DataHub: + +- **Add a new endpoint**: Users regenerate their client. Breaking change? Every team upgrades simultaneously. +- **Change error handling**: Regenerate. Update all call sites. +- **Optimize batch operations**: Can't—that logic lives in user code, reinvented by every team. + +Hand-crafted SDKs centralize expertise: + +- **Add convenience methods**: Users pull the SDK update. No code changes required. +- **Improve retry logic**: Fixed once in the SDK. All users benefit immediately. +- **Optimize batching**: Built into the SDK. Users get better performance automatically. + +The total maintenance cost is **lower** because we fix problems once instead of every team solving them independently. + +### 5. Progressive Disclosure + +Generated clients are flat—every endpoint is equally visible. Hand-crafted SDKs enable **progressive disclosure**: simple tasks are simple, complexity is opt-in. + +**Day 1 user**: Create and tag a dataset + +```java +Dataset dataset = Dataset.builder() + .platform("snowflake") + .name("my_table") + .build(); + +dataset.addTag("pii"); +client.entities().upsert(dataset); +``` + +No need to understand aspects, patches, or modes. + +**Week 1 user**: Manage governance + +```java +dataset.addOwner(ownerUrn, type) + .setDomain(domainUrn) + .addTerm(termUrn); +``` + +Still pure domain operations. + +**Month 1 user**: Understand update vs upsert + +```java +// update() emits only patches (for existing entities) +Dataset existing = client.entities().get(urn); +Dataset mutable = existing.mutable(); // Get writable copy +mutable.addTag("sensitive"); +client.entities().update(mutable); + +// upsert() emits full aspects + patches +Dataset newEntity = Dataset.builder()...; +client.entities().upsert(newEntity); +``` + +Complexity revealed **when needed**, not upfront. + +### 6. Immutability by Default + +Entities fetched from the server are **read-only by default**, enforcing explicit mutation intent. + +**The Problem:** + +Traditional SDKs allow silent mutation of fetched objects: + +```java +Dataset dataset = client.get(urn); +// Pass to function - might it mutate dataset? Who knows! +processDataset(dataset); +// Is dataset still the same? Must read all code to know +``` + +**The Solution:** + +Immutable-by-default makes mutation intent explicit: + +```java +Dataset dataset = client.get(urn); +// dataset is read-only - safe to pass anywhere +processDataset(dataset); + +// Want to mutate? Make it explicit +Dataset mutable = dataset.mutable(); +mutable.addTag("updated"); +client.entities().upsert(mutable); +``` + +**Benefits:** + +- **Safety:** Can't accidentally mutate shared references +- **Clarity:** `.mutable()` call signals write intent +- **Debugging:** Easier to track where mutations happen +- **Concurrency:** Safe to share read-only entities across threads + +**Design Inspiration:** + +This pattern is common in modern APIs because immutability scales better than defensive copying: + +- **Rust's ownership model** - mut vs immutable borrows +- **Python's frozen dataclasses** - `@dataclass(frozen=True)` +- **Java's immutable collections** - `Collections.unmodifiableList()` +- **Functional programming principles** - immutable data structures + +When you see `.mutable()` in our SDK, you're seeing battle-tested patterns from languages designed for safety and concurrency. + +## What This Costs (And Why It's Worth It) + +Hand-crafting isn't free: + +- **3,000+ lines of code** across entity classes, caching, and operations +- **457 tests** validating workflows, not just HTTP mechanics +- **13 documentation guides** teaching patterns, not just parameters +- **Ongoing maintenance** as DataHub evolves + +But this investment compounds. Every hour we spend on the SDK saves hundreds of hours across our user community. The SDK makes metadata management **effortless** instead of just **possible**. + +Compare total cost of ownership: + +| Approach | Initial Dev | User Onboarding | Ongoing Support | Innovation Speed | +| ---------------- | ----------- | --------------- | --------------- | ---------------- | +| Generated Client | Hours | High (steep) | High (repeated) | Slow (coupled) | +| Hand-Crafted SDK | Weeks | Low (gradual) | Low (central) | Fast (buffered) | + +After 6-12 months, the hand-crafted SDK becomes cheaper because centralized expertise scales better than distributed tribal knowledge. + +## The Philosophy: What SDKs Should Be + +This isn't about generated vs hand-crafted code. It's about what we believe SDKs **should be**. + +**SDKs are not just API wrappers.** They are: + +- **Semantic layers** that translate domain concepts to wire protocols +- **Knowledge repositories** that encode best practices +- **Usability interfaces** that optimize for human cognition +- **Evolution buffers** that allow internals to improve while APIs remain stable + +Code generation is perfect when **the API is the abstraction**. But for domain-rich platforms where users think in terms of datasets, lineage, and governance—not HTTP verbs and JSON payloads—hand-crafted SDKs aren't just better. They're necessary. + +## When Should You Generate? When Should You Craft? + +**Generate when**: + +- Your API's conceptual model matches user mental models +- The wire protocol fully captures domain semantics +- Operations are mostly stateless CRUD +- You prioritize API coverage over workflow optimization + +**Hand-craft when**: + +- Domain concepts require translation to wire protocol +- Users need guidance on best practices +- Stateful workflows matter (accumulate changes, emit atomically) +- You prioritize usability over feature completeness + +DataHub falls firmly in the second category. Our users don't want to learn aspect models, patch formats, or mode routing. They want to **add a tag to a dataset** and have it just work. + +That's what the hand-crafted SDK delivers. + +## Conclusion: Empathy at Scale + +In an era of automation, there's pressure to generate everything. But some problems demand craftsmanship. + +The hand-crafted SDK is an act of **empathy at scale**. It says: "We understand your problems. We've encoded the solutions. You shouldn't have to become a DataHub expert to use DataHub." + +A generated client says: "Here's our API. Figure it out." + +A hand-crafted SDK says: "Here's how to solve your problems." + +That difference is why we invested in hand-crafting. And it's why our users can focus on their data, not our API internals. + +--- + +**Document Status**: Design Philosophy +**Author**: DataHub OSS Team +**Last Updated**: 2025-01-06 diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/docker/development.md b/docs-archive/versioned_docs/version-1.5.0/docs/docker/development.md new file mode 100644 index 00000000..b2625726 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/docker/development.md @@ -0,0 +1,167 @@ +--- +title: Using Docker Images During Development +slug: /docker/development +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/docker/development.md +--- +# Using Docker Images During Development + +We've created a special `docker-compose.dev.yml` override file that should configure docker images to be easier to use +during development. + +Normally, you'd rebuild your images from scratch with a combination of gradle and docker compose commands. However, +this takes way too long for development and requires reasoning about several layers of docker compose configuration +yaml files which can depend on your hardware (Apple M1). + +The `docker-compose.dev.yml` file bypasses the need to rebuild docker images by mounting binaries, startup scripts, +and other data. These dev images, tagged with `debug` will use your _locally built code_ with gradle. +Building locally and bypassing the need to rebuild the Docker images should be much faster. + +We highly recommend you just invoke `./gradlew quickstartDebug` task. + +```shell +./gradlew quickstartDebug +``` + +This task is defined in `docker/build.gradle` and executes the following steps: + +1. Builds all required artifacts to run DataHub. This includes both application code such as the GMS war, the frontend + distribution zip which contains javascript, as well as secondary support docker containers. + +1. Locally builds Docker images with the expected `debug` tag required by the docker compose files. + +1. Runs the special `docker-compose.dev.yml` and supporting docker-compose files to mount local files directly in the + containers with remote debugging ports enabled. + +Once the `debug` docker images are constructed you'll see images similar to the following: + +```shell +acryldata/datahub-frontend-react debug e52fef698025 28 minutes ago 763MB +acryldata/datahub-gms debug ea2b0a8ea115 56 minutes ago 408MB +acryldata/datahub-upgrade debug 322377a7a21d 56 minutes ago 463MB +``` + +At this point it is possible to view the DataHub UI at `http://localhost:9002` as you normally would with quickstart. + +Like `quickstartDebug`, there are a few other tasks that bring up a different set of containers, for example +`quickstartDebugConsumers` will also bring up mce-consumer and mae-consumer. + +## Reloading + +Next, perform the desired modifications + +To see these changes in the deployment, a rebuilt of modified artifacts and a restart of the container(s) is required to run with the updated code. +The restart can be performed using following gradle task. + +```shell +./gradlew :docker:reload +``` + +This single task will build the artifacts that were modified and restart only those containers that were affected by the rebuilt artifacts. + +`reload` is generally much faster than re-running `quickstartDebug` and is recommended after an initial bringup of all services via `quickstartDebug` followed +by loading the incremental changes using `reload`. + +If there are significant changes to the code, for example due to pulling the latest code, it is recommended to start with a `quickstartDebug` and then iterate using `reload` + +# Setting environment variables via env files + +You can define different sets of environment variables for all the containers in an env file. The env files must be located in the `docker/profiles` folder. +To use the env file, run + +```shell +DATAHUB_LOCAL_COMMON_ENV=my-settings.env ./gradlew quickstartDebug +``` + +The `reload` process continues to work, but the restarted containers will use the same settings that were present at the time of running `./gradlew quickstartDebug`. + +If you need to reload the containers with a different env file or changes made to the env file, a task `reloadEnv` builds the artifacts that have code changes +and recreates all the containers that refer to these the env file via the DATAHUB_LOCAL_COMMON_ENV environment variable. + +The `reload` and `reloadEnv` tasks can only be run after running one of the debug variants of a auickstart task like `quickstartDebug` + +## Start/Stop + +The following commands can pause the debugging environment to release resources when not needed. + +Pause containers and free resources. + +```shell +docker compose -p datahub stop +``` + +Resume containers for further debugging. + +```shell +docker compose -p datahub start +``` + +## Cleanup + +To completely remove containers and volumes for a specific project, you can use the nuke tasks: + +```shell +# Remove containers and volumes for specific projects +./gradlew quickstartNuke # For default project (datahub) +./gradlew quickstartDebugNuke # For debug project (datahub) +./gradlew quickstartCypressNuke # For cypress project (dh-cypress) +``` + +> **Note**: These are Gradle nuke tasks. For CLI-based cleanup, see `datahub docker nuke` in the [quickstart guide](../quickstart.md). + +## Debugging + +The default debugging process uses your local code and enables debugging by default for both GMS and the frontend. Attach +to the instance using your IDE by using its Remote Java Debugging features. + +Environment variables control the debugging ports for GMS and the frontend. + +- `DATAHUB_MAPPED_GMS_DEBUG_PORT` - Default: 5001 +- `DATAHUB_MAPPED_FRONTEND_DEBUG_PORT` - Default: 5002 + +### IntelliJ Remote Debug Configuration + +The screenshot shows an example configuration for IntelliJ using the default GMS debugging port of 5001. + +

+ +

+ +## Tips for People New To Docker + +### Accessing Logs + +It is highly recommended you use [Docker Desktop's dashboard](https://www.docker.com/products/docker-desktop) to access service logs. If you double click an image it will pull up the logs for you. + +### Quickstart Conflicts + +If you run quickstart, use `./gradlew quickstartDebug` to return to using the debugging containers. + +### Docker Prune + +If you run into disk space issues and prune the images & containers you will need to execute the `./gradlew quickstartDebug` +again. + +### System Update + +The `datahub-upgrade` job will not block the startup of the other containers as it normally +does in a quickstart or production environment. Normally this is process is required when making updates which +require Elasticsearch reindexing. If reindexing is required, the UI will render but may temporarily return errors +until this job finishes. + +### Running a specific service + +`docker-compose up` will launch all services in the configuration, including dependencies, unless they're already +running. If you, for some reason, wish to change this behavior, check out these example commands. + +``` +docker-compose -p datahub -f docker-compose.yml -f docker-compose.override.yml -f docker-compose-without-neo4j.m1.yml -f docker-compose.dev.yml up datahub-gms +``` + +Will only start `datahub-gms` and its dependencies. + +``` +docker-compose -p datahub -f docker-compose.yml -f docker-compose.override.yml -f docker-compose-without-neo4j.m1.yml -f docker-compose.dev.yml up --no-deps datahub-gms +``` + +Will only start `datahub-gms`, without dependencies. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/domains.md b/docs-archive/versioned_docs/version-1.5.0/docs/domains.md new file mode 100644 index 00000000..e2b7406c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/domains.md @@ -0,0 +1,275 @@ +--- +title: Domains +slug: /domains +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/domains.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Domains + + + +Starting in version `0.8.25`, DataHub supports grouping data assets into logical collections called **Domains**. Domains are curated, top-level folders or categories where related assets can be explicitly grouped. Management of Domains can be centralized, or distributed out to Domain owners Currently, an asset can belong to only one Domain at a time. + +## Domains Setup, Prerequisites, and Permissions + +What you need to create and add domains: + +- **Manage Domains** platform privilege to add domains at the entity level + +You can create this privileges by creating a new [Metadata Policy](./authorization/policies.md). + +## Using Domains + +### Creating a Domain + +To create a Domain, first navigate to the **Domains** tab in the top-right menu of DataHub. + +

+ +

+ +Once you're on the Domains page, you'll see a list of all the Domains that have been created on DataHub. Additionally, you can +view the number of entities inside each Domain. + +

+ +

+ +To create a new Domain, click '+ New Domain'. + +

+ +

+ +Inside the form, you can choose a name for your Domain. Most often, this will align with your business units or groups, for example +'Platform Engineering' or 'Social Marketing'. You can also add an optional description. Don't worry, this can be changed later. + +#### Advanced: Setting a Custom Domain id + +Click on 'Advanced' to show the option to set a custom Domain id. The Domain id determines what will appear in the DataHub 'urn' (primary key) +for the Domain. This option is useful if you intend to refer to Domains by a common name inside your code, or you want the primary +key to be human-readable. Proceed with caution: once you select a custom id, it cannot be easily changed. + +

+ +

+ +By default, you don't need to worry about this. DataHub will auto-generate a unique Domain id for you. + +Once you've chosen a name and a description, click 'Create' to create the new Domain. + +### Assigning an Asset to a Domain + +You can assign assets to Domain using the UI or programmatically using the API or during ingestion. + +#### UI-Based Assignment + +To assign an asset to a Domain, simply navigate to the asset's profile page. At the bottom left-side menu bar, you'll +see a 'Domain' section. Click 'Set Domain', and then search for the Domain you'd like to add to. When you're done, click 'Add'. + +

+ +

+ +To remove an asset from a Domain, click the 'x' icon on the Domain tag. + +> Notice: Adding or removing an asset from a Domain requires the `Edit Domain` Metadata Privilege, which can be granted +> by a [Policy](authorization/policies.md). + +#### Ingestion-time Assignment + +All SQL-based ingestion sources support assigning domains during ingestion using the `domain` configuration. Consult your source's configuration details page (e.g. [Snowflake](./generated/ingestion/sources/snowflake.md)), to verify that it supports the Domain capability. + +:::note + +Assignment of domains during ingestion will overwrite domains that you have assigned in the UI. A single table can only belong to one domain. + +::: + +Here is a quick example of a snowflake ingestion recipe that has been enhanced to attach the **Analytics** domain to all tables in the **long_tail_companions** database in the **analytics** schema, and the **Finance** domain to all tables in the **long_tail_companions** database in the **ecommerce** schema. + +```yaml +source: + type: snowflake + config: + username: ${SNOW_USER} + password: ${SNOW_PASS} + account_id: + warehouse: COMPUTE_WH + role: accountadmin + database_pattern: + allow: + - "long_tail_companions" + schema_pattern: + deny: + - information_schema + profiling: + enabled: False + domain: + Analytics: + allow: + - "long_tail_companions.analytics.*" + Finance: + allow: + - "long_tail_companions.ecommerce.*" +``` + +:::note + +When bare domain names like `Analytics` is used, the ingestion system will first check if a domain like `urn:li:domain:Analytics` is provisioned, failing that; it will check for a provisioned domain that has the same name. If we are unable to resolve bare domain names to provisioned domains, then ingestion will refuse to proceeed until the domain is provisioned on DataHub. + +::: + +You can also provide fully-qualified domain names to ensure that no ingestion-time domain resolution is needed. For example, the following recipe shows an example using fully qualified domain names: + +```yaml +source: + type: snowflake + config: + username: ${SNOW_USER} + password: ${SNOW_PASS} + account_id: + warehouse: COMPUTE_WH + role: accountadmin + database_pattern: + allow: + - "long_tail_companions" + schema_pattern: + deny: + - information_schema + profiling: + enabled: False + domain: + "urn:li:domain:6289fccc-4af2-4cbb-96ed-051e7d1de93c": + allow: + - "long_tail_companions.analytics.*" + "urn:li:domain:07155b15-cee6-4fda-b1c1-5a19a6b74c3a": + allow: + - "long_tail_companions.ecommerce.*" +``` + +### Searching by Domain + +Once you've created a Domain, you can use the search bar to find it. + +

+ +

+ +Clicking on the search result will take you to the Domain's profile, where you +can edit its description, add / remove owners, and view the assets inside the Domain. + +

+ +

+ +Once you've added assets to a Domain, you can filter search results to limit to those Assets +within a particular Domain using the left-side search filters. + +

+ +

+ +On the homepage, you'll also find a list of the most popular Domains in your organization. + +

+ +

+ +## Additional Resources + +### Videos + +**Supercharge Data Mesh with Domains in DataHub** + +

+ +

+ +### GraphQL + +- [domain](../graphql/queries.md#domain) +- [listDomains](../graphql/queries.md#listdomains) +- [createDomains](../graphql/mutations.md#createdomain) +- [setDomain](../graphql/mutations.md#setdomain) +- [unsetDomain](../graphql/mutations.md#unsetdomain) + +#### Examples + +**Creating a Domain** + +```graphql +mutation createDomain { + createDomain( + input: { name: "My New Domain", description: "An optional description" } + ) +} +``` + +This query will return an `urn` which you can use to fetch the Domain details. + +## Create a Nested Domain + +```graphql +mutation createDomain { + createDomain( + input: { + name: "Verticals" + description: "An optional description" + parentDomain: "urn:li:domain:marketing" + } + ) +} +``` + +This query will create a new domain, "Verticals", under the "Marketing" domain. + +**Fetching a Domain by Urn** + +```graphql +query getDomain { + domain(urn: "urn:li:domain:engineering") { + urn + properties { + name + description + } + entities { + total + } + } +} +``` + +**Adding a Dataset to a Domain** + +```graphql +mutation setDomain { + setDomain( + entityUrn: "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)" + domainUrn: "urn:li:domain:engineering" + ) +} +``` + +> Pro Tip! You can try out the sample queries by visiting `/api/graphiql`. + +### DataHub Blog + +- [Just Shipped: UI-Based Ingestion, Data Domains & Containers, Tableau support, and MORE!](https://medium.com/datahub-project/just-shipped-ui-based-ingestion-data-domains-containers-and-more-f1b1c90ed3a) + +## FAQ and Troubleshooting + +**What is the difference between DataHub Domains, Tags, and Glossary Terms?** + +DataHub supports Tags, Glossary Terms, & Domains as distinct types of Metadata that are suited for specific purposes: + +- **Tags**: Informal, loosely controlled labels that serve as a tool for search & discovery. Assets may have multiple tags. No formal, central management. +- **Glossary Terms**: A controlled vocabulary, with optional hierarchy. Terms are typically used to standardize types of leaf-level attributes (i.e. schema fields) for governance. E.g. (EMAIL_PLAINTEXT) +- **Domains**: A set of top-level categories. Usually aligned to business units / disciplines to which the assets are most relevant. Central or distributed management. Single Domain assignment per data asset. + +### Related Features + +- [Glossary Terms](./glossary/business-glossary.md) +- [Tags](./tags.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features.md b/docs-archive/versioned_docs/version-1.5.0/docs/features.md new file mode 100644 index 00000000..32bdc103 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features.md @@ -0,0 +1,56 @@ +--- +hide_title: true +slug: /features +title: What is DataHub? +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/features.md' +--- + +import FeatureCardSection from '@site/src/pages/docs/\_components/FeatureCardSection'; +import QuickstartCTA from '@site/src/pages/docs/\_components/QuickstartCTA'; + +# What is DataHub? + +DataHub is a modern data catalog designed to streamline metadata management, data discovery, and data governance. It enables users to efficiently explore and understand their data, track data lineage, profile datasets, and establish data contracts. +This extensible metadata management platform is built for developers to tame the complexity of their rapidly evolving data ecosystems and for data practitioners to leverage the total value of data within their organization. + +

+ DataHub Integrations +

+ +## Quickstart + + + +## Key Features + + + +## Get Started + +### Deployment + +To get started with DataHub, you can use a simple CLI command. Alternatively, you can deploy the instance to production using Docker or Helm charts. + +- [Quickstart](quickstart.md) +- [Self-hosting DataHub with Kubernetes](deploy/kubernetes.md) +- [DataHub Cloud](managed-datahub/managed-datahub-overview.md) + +### Ingestion + +DataHub supports ingestion by UI and CLI. + +- [UI-based Ingestion](ui-ingestion.md) +- [CLI-based Ingestion](../metadata-ingestion/cli-ingestion.md) + +## Join the Community + +For additional information and assistance, feel free to visit one of these channels! + +- [Slack](https://datahubspace.slack.com) +- [Blog](https://medium.com/datahub-project/) +- [LinkedIn](https://www.linkedin.com/company/acryl-data/) +- Our champions - [Data Practitioners Guild](https://datahub.com/guild/) & [DataHub Champions](https://datahub.com/champions) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/dataset-usage-and-query-history.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/dataset-usage-and-query-history.md new file mode 100644 index 00000000..65502f01 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/dataset-usage-and-query-history.md @@ -0,0 +1,88 @@ +--- +title: Dataset Usage & Query History +slug: /features/dataset-usage-and-query-history +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/dataset-usage-and-query-history.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Dataset Usage & Query History + + + +Dataset Usage & Query History can give dataset-level information about the top queries which referenced a dataset. + +Usage data can help identify the top users who probably know the most about the dataset and top queries referencing this dataset. +You can also get an overview of the overall number of queries and distinct users. +In some sources, column level usage is also calculated, which can help identify frequently used columns. + +With sources that support usage statistics, you can collect Dataset, Dashboard, and Chart usages. + +## Dataset Usage & Query History Setup, Prerequisites, and Permissions + +To ingest Dataset Usage & Query History data, you should check first on the specific source doc +if it is supported by the DataHub source and how to enable it. + +You can validate this on the DataHub source's capabilities section: + +

+ +

+ +Some sources require a separate, usage-specific recipe to ingest Usage and Query History metadata. In this case, it is noted in the capabilities summary, like so: + +

+ +

+ +Please, always check the usage prerequisities page if the source has as it can happen you have to add additional +permissions which only needs for usage. + +## Using Dataset Usage & Query History + +After successful ingestion, the Queries and Stats tab will be enabled on datasets with any usage. + +

+ +

+ +On the Queries tab, you can see the top 5 most often run queries which referenced this dataset. + +

+ +

+ +On the Stats tab, you can see the top 5 users who run the most queries which referenced this dataset + +

+ +

+ +With the collected usage data, you can even see column-level usage statistics (Redshift Usage doesn't supported this yet): + +

+ +

+ +## Additional Resources + +### Videos + +**DataHub 101: Data Profiling and Usage Stats 101** + +

+ +

+ +### GraphQL + +- +- +- +- + +## FAQ and Troubleshooting + +### Why is my Queries/Stats tab greyed out? + +Queries/Stats tab is greyed out if there is no usage statistics for that dataset or there were no ingestion with usage extraction run before. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/access-roles.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/access-roles.md new file mode 100644 index 00000000..e2d722ff --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/access-roles.md @@ -0,0 +1,238 @@ +--- +title: Data Access Roles +slug: /features/feature-guides/access-roles +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/access-roles.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Data Access Roles + + + +> **Note**: This feature is under active development and subject to significant change. + +## Introduction + +DataHub's Access Roles feature allows you to ingest external roles from your source systems with your data assets in DataHub so that users can understand which roles they need in order to access a given asset. + +Whereas [Data Access Workflows](../../managed-datahub/workflows/access-workflows.md) enable you to create workflows for requesting and reviewing access for tables, dashboards, etc, this feature enables users to understand the roles +that already have access for a given asset, and redirect to an external platform to request access to the role. + +This creates a unified view of access control across your data ecosystem, helping data consumers: + +1. **Discover available access** - Find what roles are already provisioned for them across different data platforms +2. **Request appropriate access** - Easily identify and request to join the appropriate role for the access they need +3. **Simplify governance** - Streamline the access management process by centralizing role information in DataHub + +By integrating your external roles into DataHub, teams can reduce access request friction and ensure users have the right level of access to the data they need. + +## Configuration + +### Self-hosted DataHub + +For self-hosted DataHub deployments, the Access Management feature is _disabled_ by default. To enable it, +simply set the `SHOW_ACCESS_MANAGEMENT` environment variable for the `datahub-gms` service container +to `true`. For example in your `docker/datahub-gms/docker.env`, you'd configure: + +``` +SHOW_ACCESS_MANAGEMENT=true +``` + +### DataHub Cloud + +If you're using DataHub Cloud, enabling the Access Management feature just requires contacting your DataHub Cloud CustomerSuccess representative. They can enable this feature for your environment without any configuration changes on your part. + +## UI Location + +Under a dataset, the new tab "Access Management" should appear if configured correctly. + +

+ +

+ +## Data Model + +Access management introduces a new entity in DataHub's metadata model called a Role. +A Role is comprised of: + +- A unique key (URN) +- Properties of the role (name, description, type, request URL) +- A list of users that have been provisioned the role + +This role must then be associated with datasets through a new aspect called access. + +:::note Important Note +Currently, only Dataset entities support Access Management. +::: + +:::caution Do not confuse role with datahubrole +The "role" entity refers to an external role definition that exists in your source systems (like Snowflake or BigQuery), while "datahubrole" is for the management of privileges within DataHub itself (i.e., the admin role can accept proposed metadata changes). +::: + +## Managing Access Through DataHub + +You can set up Access Management through either the CLI or Python API. Here's how to complete the three main steps: + +### Creating External Roles + + + + +```bash +datahub put --urn "urn:li:role:reader" --aspect roleProperties -d - <<-EOF +{ + "name": "Snowflake Reader Role", + "description": "Description for Snowflake Reader Role", + "type": "READ", + "requestUrl": "http://custom-url-for-redirection.com" +} +EOF +``` + + + + +```python +import datahub.emitter.mce_builder as builder +from datahub.emitter.rest_emitter import DatahubRestEmitter +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.metadata.schema_classes import RolePropertiesClass, ChangeTypeClass + +# Create a role properties aspect +role_properties = RolePropertiesClass( + name="Snowflake Reader Role", + description="Description for Snowflake Reader Role", + type="READ", + requestUrl="http://custom-url-for-redirection.com" +) + +# Create a metadata change proposal +mcp = MetadataChangeProposalWrapper( + changeType=ChangeTypeClass.UPSERT, + entityUrn="urn:li:role:reader", + aspectName="roleProperties", + aspect=role_properties +) + +# Emit the metadata +emitter = DatahubRestEmitter(gms_server="http://localhost:8080") +emitter.emit(mcp) +``` + + + + +### Assigning Users to Roles (Optional) + + + + +```bash +datahub put --urn "urn:li:role:reader" --aspect actors -d - <<-EOF +{ + "users": [ + {"user": "urn:li:corpuser:datahubuser"} + ] +} +EOF +``` + + + + +```python +from datahub.metadata.schema_classes import ActorsClass, ActorClass + +# Create an actors aspect +actors = ActorsClass( + users=[ + ActorClass(user="urn:li:corpuser:datahubuser") + ] +) + +# Create a metadata change proposal +mcp = MetadataChangeProposalWrapper( + changeType=ChangeTypeClass.UPSERT, + entityUrn="urn:li:role:reader", + aspectName="actors", + aspect=actors +) + +# Emit the metadata +emitter.emit(mcp) +``` + + + + +### Assigning Roles to Datasets + + + + +```bash +datahub put --urn "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" --aspect access -d - <<-EOF +{ + "roles": [ + {"urn": "urn:li:role:reader"}, + {"urn": "urn:li:role:writer"} + ] +} +EOF +``` + + + + +```python +from datahub.metadata.schema_classes import AccessClass, RoleAssociationClass + +dataset_urn = "urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)" + +# Create an access aspect with multiple roles +access_aspect = AccessClass( + roles=[ + RoleAssociationClass(urn="urn:li:role:reader"), + RoleAssociationClass(urn="urn:li:role:writer") + ] +) + +# Create a metadata change proposal +mcp = MetadataChangeProposalWrapper( + changeType=ChangeTypeClass.UPSERT, + entityUrn=dataset_urn, + aspectName="access", + aspect=access_aspect +) + +# Emit the metadata +emitter.emit(mcp) +``` + + + + +## Use Cases + +Here are some common scenarios where integrating external roles into DataHub is valuable: + +1. **Unified Access View** - Data engineers can see all users with access to sensitive data across multiple platforms from a single interface +2. **Self-Service Access Requests** - Analysts can discover what roles they need to access specific datasets and request them directly from DataHub +3. **Access Auditing** - Compliance teams can review who has access to which datasets through which roles +4. **Onboarding Acceleration** - New team members can quickly discover what access they need for their role + +## Demo and Examples + +To see Access Management in action, check out our [DataHub Townhall demo](https://youtu.be/mXsn33tALCA?t=1333) where we showcase how to use this feature in a real-world scenario. + +## What's Next for Access Management + +Future enhancements planned for Access Management include: + +- Modeling external policies in addition to just roles +- Automatically extracting roles/policies from sources like BigQuery, Snowflake, etc. +- Extending support to more entity types beyond datasets +- Advanced access request workflows with approvals diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/applications.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/applications.md new file mode 100644 index 00000000..0fc3659b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/applications.md @@ -0,0 +1,124 @@ +--- +title: About DataHub Applications +sidebar_label: Applications +slug: /features/feature-guides/applications +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/applications.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# About DataHub Applications + + + +DataHub Cloud v1.2.0 introduces **Applications**, an approach to grouping similar assets within DataHub that align with a specific business purpose. + +Applications sit between Domains and Data Products in DataHub's metadata model hierarchy: + +- **Domains** (largest): Business areas with multiple purposes +- **Applications** (middle): Single business purpose within the Domain +- **Data Products** (smallest): Specific data offerings within the Application + +For example, an ecommerce company might have the following structure: + +- **Payments Domain**: Contains everything related to processing payments across the business +- **Refund Application**: Groups all assets specifically used for handling customer refunds within the Payments Domain, including: + - "Daily Refund Payments" **Data Product** + - "Daily Refund Failures" **Data Product** + - Supporting assets like payment pipelines, refund processing datasets, etc. +- **Fraud Detection Application**: Groups all assets related to identifying and preventing fraudulent transactions within the Payments Domain, including: + - "Real-time Fraud Alerts" **Data Product** + - "Weekly Fraud Analytics Report" **Data Product** + - Supporting assets like ML models, risk scoring datasets, etc. + +Each Application serves its specific business purpose while remaining part of the broader Payments Domain. + +## Why Use Applications? + +Applications work best for large organizations that already use this concept internally and need additional organization beyond Domains and Data Products. For most teams, we recommend starting with Domains and Data Products first. + +Keep in mind the following: + +- **Applications aren't part of Data Mesh architecture** — they're an additional organizational layer that some large organizations find helpful, but are not suitable for teams adopting Data Mesh +- **Applications differ from software services** — while a service is a single piece of software, an Application within DataHub groups multiple data assets around one business purpose + +:::note +**Most teams don't need Applications**. We find that Domains and Data Products frequently provide enough organizational support for DataHub users. +::: + +## Prerequisites and Permissions + +Users will need the following DataHub Privileges to use Applications: + +- **Edit Entity** privilege on Applications to create new Applications or edit existing ones. +- **Edit Entity Applications** privilege to assign Applications to data assets. +- **Manage Features** platform privilege to control whether Applications are visible to users. + +Learn more about creating a Custom Policy to leverage these privileges [here](../../authorization/policies.md). + +## Using Applications + +### Enabling Applications + +To enable Applications, you will need to work with your DataHub rep to configure the feature. Once this is complete, navigate to **Settings** > **Appearance** and toggle on **Show Applications**. + +:::note +Applications are disabled by default. Please reach out to your DataHub rep to enable the feature. +::: + +

+ +

+ +### Creating an Application + +To create an Application, first navigate to the **Applications** tab in the left-side navigation menu of DataHub. + +Once you're on the Applications page, you'll see a list of all the Applications that have been created on DataHub. + +

+ +

+ +To create a new Application, click '+ Create Application'. + +

+ +

+ +Inside the form, you can choose a name for your Application, along with description and owner. Learn more about creating Applications via API in [this tutorial](../../api/tutorials/applications.md). + +### Assigning Assets to an Application + +You can assign assets to Applications using the UI or programmatically using the API. To assign in the UI, you can add an Application via the Entity sidebar, or from an Applications page. + +

+ +

+ +Learn more about assigning assets to Applications via API in [this tutorial](../../api/tutorials/applications.md). + +### Searching by Application + +Once you've created an Application, you can use the search bar to find it. + +

+ +

+ +Clicking on the search result will take you to the Application's profile, where you can edit its description, manage owners, and view the assets inside the Application. + +

+ +

+ +Once you've added assets to an Application, you can filter search results to limit to those assets within a particular Application using search filters. + +### API Tutorials + +- [Applications](../../api/tutorials/applications.md) + +### Related Features + +- [Domains](../../domains.md) +- [Data Products](../../dataproducts.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/bigquery.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/bigquery.md new file mode 100644 index 00000000..d8d0d22e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/bigquery.md @@ -0,0 +1,151 @@ +--- +title: BigQuery Plugin +slug: /features/feature-guides/ask-datahub-plugins/bigquery +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub-plugins/bigquery.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# BigQuery Plugin + + + +> **Note**: Ask DataHub Plugins is currently in **Private Beta**. To enable this feature, please reach out to your DataHub customer support representative. + +The **BigQuery Plugin** connects Ask DataHub to [Google BigQuery](https://cloud.google.com/bigquery) via the [BigQuery MCP server](https://docs.cloud.google.com/bigquery/docs/use-bigquery-mcp), enabling conversational analytics directly from the chat. Like the Snowflake and Databricks plugins, DataHub acts as the **semantic layer** — finding the right data — while BigQuery serves as the **data layer** — executing queries and returning results. + +## Why Connect BigQuery? + +With the BigQuery plugin enabled, Ask DataHub can: + +- **Conversational analytics** — ask questions in plain language and get answers backed by real data. DataHub finds the right tables using metadata (descriptions, ownership, lineage), then BigQuery executes the query. +- **Data exploration** — understand dataset structure, preview sample data, and inspect column values to evaluate data quality or fitness for a use case. +- **Data debugging** — investigate data issues detected by DataHub assertions by inspecting the actual data: check for nulls, duplicates, unexpected values, or freshness problems at the source. + +Because the plugin uses **Google OAuth**, each user authenticates with their own Google account. Users only see data they are authorized to access via IAM — no need to manage separate policies in DataHub. + +**Example prompts:** + +- _"Query the top 10 customers by revenue this quarter"_ +- _"How many null values are in the email column of the customers table?"_ +- _"Show me 5 sample rows from the orders table"_ +- _"List all datasets in my project"_ + +## Prerequisites + +- A Google Cloud project with the BigQuery API enabled +- The [BigQuery MCP server enabled](https://docs.cloud.google.com/bigquery/docs/use-bigquery-mcp) in the project (see Step 1 below) +- Users must have the [required IAM roles](https://docs.cloud.google.com/bigquery/docs/use-bigquery-mcp#required_roles) on the project (including `roles/bigquery.dataViewer`, `roles/bigquery.jobUser`, and `roles/mcp.toolUser`) +- DataHub Cloud with Ask DataHub Plugins enabled +- Platform admin access in DataHub to configure the plugin + +## Admin Setup + +The BigQuery plugin uses **User OAuth** authentication — each user authenticates with their own Google account. Setup involves enabling the BigQuery MCP server, creating an OAuth client in Google Cloud, then configuring the plugin in DataHub. + +### Step 1: Enable the BigQuery MCP Server + +Enable the BigQuery MCP server in your Google Cloud project using the [`gcloud` CLI](https://docs.cloud.google.com/sdk/docs/install): + +```bash +gcloud beta services mcp enable bigquery.googleapis.com \ + --project=YOUR_PROJECT_ID +``` + +:::tip gcloud version +This command requires a recent version of the gcloud CLI beta component. If you see `Invalid choice: 'mcp'`, run `gcloud components update` to update to the latest version. +::: + +For more details, see the [Google Cloud documentation](https://docs.cloud.google.com/mcp/enable-disable-mcp-servers). + +### Step 2: Create an OAuth Client and Collect Credentials + +Create a **Web application** OAuth client in [Google Auth Platform > Clients](https://console.cloud.google.com/auth/clients). When configuring the client, add your DataHub OAuth callback URL as an **Authorized redirect URI**: + +``` +https:///integrations/oauth/callback +``` + +

+ +

+ +:::caution Redirect URI +The **Authorized redirect URI** must match the OAuth Callback URL shown in the DataHub plugin creation form exactly (e.g. `https://your-org.acryl.io/integrations/oauth/callback`). You can copy this URL from the DataHub form when creating the plugin in Step 3. +::: + +After creating the client, copy the **Client ID** and **Client Secret** immediately — the client secret won't be shown again. + +

+ +

+ +For detailed instructions, see [Obtain OAuth 2.0 credentials](https://developers.google.com/identity/protocols/oauth2#1.-obtain-oauth-2.0-credentials-from-the-dynamic_data.setvar.console_name.) in the Google documentation. + +### Step 3: Create Plugin in DataHub + +1. Navigate to **Settings > AI > Plugins** in DataHub +2. Click **+ Create** and select **Custom MCP** +3. Fill in the plugin details: + +| Field | Value | +| ----------------------- | -------------------------------------- | +| **Name** | `BigQuery` | +| **Description** | A description for the plugin | +| **MCP Server URL** | `https://bigquery.googleapis.com/mcp` | +| **Authentication Type** | `User OAuth (Each user authenticates)` | + +4. DataHub will automatically discover the OAuth configuration from Google. Provide the following: + +| Field | Value | +| ------------------ | ------------------------------------------ | +| **Client ID** | From Step 2 | +| **Client Secret** | From Step 2 | +| **Default Scopes** | `https://www.googleapis.com/auth/bigquery` | + +

+ +

+ +:::info OAuth Scopes +The `https://www.googleapis.com/auth/bigquery` scope grants read and write access to BigQuery. For a full list of available scopes, see [BigQuery OAuth scopes](https://developers.google.com/identity/protocols/oauth2/scopes#bigquery). +::: + +5. Optionally add **Instructions for the AI Assistant**, ensure **Enable for Ask DataHub** is on, and click **Create** + +## User Setup + +Navigate to **Settings > My AI Settings**, find the **BigQuery** plugin, and click **Connect**. You'll be redirected to Google to authenticate, then back to DataHub. See the [overview](./overview.md#user-setup-enabling-plugins) for more details. + +:::note OAuth Consent Screen +If your OAuth consent screen is set to **Internal**, only users within your Google Workspace organization can authenticate. If set to **External** and not yet verified, users may see a warning — they can proceed by clicking **Advanced > Go to \**. For production use, consider [verifying the consent screen](https://support.google.com/cloud/answer/10311615). +::: + +## Capabilities + +Once connected, Ask DataHub can use the BigQuery MCP server to: + +- **Run SQL queries** — execute SELECT queries against your BigQuery data (read-only; write operations are not supported) +- **Browse datasets and tables** — list datasets in a project and tables in a dataset +- **Inspect table metadata** — view schema, row counts, and other table details + +For the full list of available tools, see the [BigQuery MCP reference](https://docs.cloud.google.com/bigquery/docs/reference/mcp). + +## Troubleshooting + +### OAuth Fails + +- Verify the **Authorized redirect URI** in Google Cloud matches the OAuth Callback URL shown in DataHub exactly +- Ensure the Client ID and Client Secret are entered correctly +- Check that the [OAuth consent screen](https://console.cloud.google.com/auth/audience) is configured for your Google Cloud project + +### Permission Errors + +- Ensure the user has the [required IAM roles](https://docs.cloud.google.com/bigquery/docs/use-bigquery-mcp#required_roles) on the project +- Verify the BigQuery MCP server is enabled in the project + +### Query Failures + +- Verify the user has access to the datasets and tables they're querying +- Note that `execute_sql` queries are [limited to 3 minutes](https://docs.cloud.google.com/bigquery/docs/use-bigquery-mcp#limitations) by default — longer queries are canceled automatically +- BigQuery MCP does not support querying Google Drive external tables diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/databricks.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/databricks.md new file mode 100644 index 00000000..08598eb8 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/databricks.md @@ -0,0 +1,122 @@ +--- +title: Databricks Plugin +slug: /features/feature-guides/ask-datahub-plugins/databricks +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub-plugins/databricks.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Databricks Plugin + + + +> **Note**: Ask DataHub Plugins is currently in **Private Beta**. To enable this feature, please reach out to your DataHub customer support representative. + +The **Databricks Plugin** connects Ask DataHub to your Databricks workspace via the [Databricks MCP server](https://docs.databricks.com/aws/en/generative-ai/mcp/connect-external-services), enabling conversational analytics directly from the chat. Like the Snowflake plugin, DataHub acts as the **semantic layer** — finding the right data — while Databricks serves as the **data layer** — executing queries and returning results. + +## Why Connect Databricks? + +With the Databricks plugin enabled, Ask DataHub can: + +- **Conversational analytics** — ask questions in plain language and get answers backed by real data. DataHub finds the right tables using metadata, then Databricks executes the SQL query. +- **Data exploration** — understand dataset structure, preview sample data, and inspect column values across your Unity Catalog. +- **Data debugging** — investigate data issues detected by DataHub assertions by querying the actual data at the source. + +Because the plugin uses **OAuth**, each user authenticates with their own Databricks account. Users only see data they are authorized to access. + +**Example prompts:** + +- _"Query the top 10 customers by lifetime value"_ +- _"How many rows were added to the events table today?"_ +- _"Show me 5 sample rows from the orders table"_ +- _"Check for null values in the email column of the users table"_ + +## Prerequisites + +- A Databricks workspace with [Managed MCP Servers](https://docs.databricks.com/aws/en/generative-ai/mcp/connect-external-services) enabled +- Admin access to create OAuth app connections in Databricks (**Settings > App connections**) +- DataHub Cloud with Ask DataHub Plugins enabled +- Platform admin access in DataHub to configure the plugin + +## Admin Setup + +The Databricks plugin uses **User OAuth** authentication — each user authenticates with their own Databricks account. Setup involves creating an OAuth app connection in Databricks, then configuring the plugin in DataHub. + +### Step 1: Create an OAuth App Connection in Databricks + +1. In Databricks, navigate to **Settings > App connections** +2. Click **Add connection** and fill in the details: + +| Field | Value | +| ---------------------------- | -------------------------------------------------------- | +| **Application Name** | `DataHub` (or any name you'll recognize) | +| **Redirect URLs** | `https:///integrations/oauth/callback` | +| **Access scopes** | Check **SQL** | +| **Generate a client secret** | Checked | + +:::caution Redirect URL +The **Redirect URL** must match the OAuth Callback URL shown in the DataHub plugin creation form exactly (e.g. `https://your-org.acryl.io/integrations/oauth/callback`). You can copy this URL from the DataHub form when creating the plugin in Step 3. +::: + +3. Click **Add** + +

+ +

+ +### Step 2: Collect Credentials + +After creating the connection, a dialog will show your **Client ID** and **Client Secret**. Copy both immediately — the client secret won't be shown again. + +

+ +

+ +### Step 3: Create Plugin in DataHub + +1. Navigate to **Settings > AI > Plugins** in DataHub +2. Click **+ Create** and select **Custom MCP** +3. Fill in the plugin details: + +| Field | Value | +| ----------------------- | --------------------------------------------------- | +| **Name** | `Databricks` | +| **Description** | A description for the plugin | +| **MCP Server URL** | `https:///api/2.0/mcp/sql` | +| **Authentication Type** | `User OAuth (Each user authenticates)` | + +4. DataHub will automatically discover the OAuth configuration from Databricks, including the authorization URL, token URL, and available scopes. Provide the following: + +| Field | Value | +| ------------------ | ------------------------ | +| **Client ID** | From Step 2 | +| **Client Secret** | From Step 2 | +| **Default Scopes** | `sql` (minimum required) | + +

+ +

+ +:::info Access Scopes +The `sql` scope is the minimum required for Ask DataHub to execute queries. If you selected **All APIs** when creating the Databricks app connection, additional scopes will be available — select only what you need. +::: + +5. Optionally add **Instructions for the AI Assistant**, ensure **Enable for Ask DataHub** is on, and click **Create** + +## User Setup + +Navigate to **Settings > My AI Settings**, find the **Databricks** plugin, and click **Connect**. You'll be redirected to Databricks to authenticate, then back to DataHub. See the [overview](./overview.md#user-setup-enabling-plugins) for more details. + +## Troubleshooting + +### OAuth Fails + +- Verify the **Redirect URL** in Databricks matches the OAuth Callback URL shown in DataHub exactly +- Ensure the Client ID and Client Secret are entered correctly +- If your workspace has [IP access restrictions](https://docs.databricks.com/aws/en/security/network/front-end/ip-access-list), ensure DataHub's outbound IPs are allowlisted + +### Query Failures + +- Verify the user has access to the tables they're querying in Unity Catalog +- Ensure the `sql` scope was selected when creating the app connection in Databricks +- Check that a SQL warehouse is available and running in the workspace diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/dbt.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/dbt.md new file mode 100644 index 00000000..bd34d3de --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/dbt.md @@ -0,0 +1,148 @@ +--- +title: dbt Cloud Plugin +slug: /features/feature-guides/ask-datahub-plugins/dbt +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub-plugins/dbt.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# dbt Cloud Plugin + + + +> **Note**: Ask DataHub Plugins is currently in **Private Beta**. To enable this feature, please reach out to your DataHub customer support representative. + +The **dbt Cloud Plugin** connects Ask DataHub to your dbt Cloud environment via the [dbt remote MCP server](https://docs.getdbt.com/docs/dbt-ai/setup-remote-mcp), giving the AI assistant visibility into job runs, model definitions, test results, and — with the right configuration — the ability to execute SQL queries on behalf of users. + +## Why Connect dbt Cloud? + +With the dbt Cloud plugin enabled, Ask DataHub can: + +- **Check job run status** — see whether recent dbt runs succeeded or failed +- **Inspect model definitions** — view the SQL and configuration behind dbt models +- **Review test results** — understand which tests passed or failed and why +- **Execute SQL queries** — run queries against your warehouse through dbt (with additional configuration) + +**Example prompts:** + +- _"Did the latest dbt run for the revenue model succeed?"_ +- _"Show me the SQL definition of the orders model"_ +- _"Which dbt tests failed in the last 24 hours?"_ +- _"Query the customers table to check for null emails"_ + +## Prerequisites + +- A dbt Cloud account with [AI features enabled](https://docs.getdbt.com/docs/cloud/enable-dbt-copilot) and the remote MCP server active +- A dbt Cloud [personal access token or service token](https://docs.getdbt.com/docs/dbt-cloud-apis/authentication) with **Semantic Layer** and **Developer** permissions +- Your dbt Cloud **production environment ID** (and optionally a **development environment ID** for SQL execution) +- DataHub Cloud with Ask DataHub Plugins enabled +- Platform admin access in DataHub to configure the plugin + +## Admin Setup + +### Step 1: Enable the dbt Remote MCP Server + +Follow the [dbt remote MCP setup guide](https://docs.getdbt.com/docs/dbt-ai/setup-remote-mcp) to enable the remote MCP server for your dbt Cloud project. + +### Step 2: Collect Configuration Details + +You'll need the following from dbt Cloud: + +**MCP Server URL**: Construct the URL using your dbt Cloud host: + +``` +https:///api/ai/v1/mcp/ +``` + +For multi-cell accounts, the host URL is in the format `ACCOUNT_PREFIX.us1.dbt.com`. See [dbt's regions documentation](https://docs.getdbt.com/docs/cloud/about-cloud/access-regions-ip-addresses) for details. + +**Production Environment ID**: Navigate to **Orchestration > Environments** in dbt Cloud. Copy the environment ID from the URL. + +

+ +

+ +**API Token**: Generate a [personal access token](https://docs.getdbt.com/docs/dbt-cloud-apis/user-tokens) or [service token](https://docs.getdbt.com/docs/dbt-cloud-apis/service-tokens). The token needs **Semantic Layer** and **Developer** permissions. + +### Step 3: Create Plugin in DataHub + +1. Navigate to **Settings > AI > Plugins** in DataHub +2. Click **+ Create** and select **dbt Cloud MCP** +3. Fill in the plugin details: + +| Field | Value | +| ----------------------------- | -------------------------------------------- | +| **Name** | `dbt Cloud` | +| **Description** | A description for the plugin | +| **MCP Server URL** | `https:///api/ai/v1/mcp/` | +| **Production Environment ID** | Your production environment ID from Step 2 | + +

+ +

+ +4. Choose an authentication type: + +| Option | How it works | Best for | +| ------------------ | ------------------------------------------------------------------ | ------------------------------------------- | +| **Shared API Key** | Admin enters a single service token used for all users | Centralized access with one service account | +| **User API Key** | Each user provides their own personal access token when connecting | Per-user audit trails and access control | + +### Step 4: Enable SQL Execution (Optional) + +To let Ask DataHub execute SQL queries through dbt, provide the following additional fields: + +| Field | Description | +| ------------------------------ | ------------------------------------------------------------------------------------------ | +| **Development Environment ID** | Your dbt Cloud development environment ID (found on the Orchestration > Environments page) | +| **User ID** | Your dbt Cloud user ID (found in the URL at **Settings > Users** in dbt Cloud) | + +:::caution +If using a **Shared API Key**, the user ID is shared across all users — meaning queries will execute under a single identity. With **User API Key** auth, each user provides their own user ID when connecting. +::: + +You can find your dbt Cloud user ID in the URL when navigating to **Settings > Users** and selecting your profile: + +

+ +

+ +Each user also needs [development credentials](https://docs.getdbt.com/docs/cloud/connect-data-platform/connect-your-database) configured in dbt Cloud (warehouse connection under their profile) for SQL execution to work. + +5. Optionally add **Instructions for the AI Assistant**, ensure **Enable for Ask DataHub** is on, and click **Create** + +## User Setup + +Navigate to **Settings > My AI Settings** and find the **dbt Cloud** plugin. If using **User API Key** auth, click **Connect** and enter your dbt Cloud API token and user ID. If using **Shared API Key** auth, simply toggle the plugin **on**. See the [overview](./overview.md#user-setup-enabling-plugins) for more details. + +

+ +

+ +## Usage Tips + +- Combine dbt Cloud with DataHub lineage for end-to-end debugging — trace a dashboard issue back through DataHub lineage to a failed dbt model +- Ask about specific environments (production vs. staging) to get targeted results +- Use dbt test results alongside DataHub assertions for a complete data quality picture + +:::info dbt Copilot Credits +Only the `text_to_sql` tool consumes dbt Copilot credits. Other MCP tools (metadata discovery, semantic layer queries) do not. See [dbt's documentation](https://docs.getdbt.com/docs/dbt-ai/setup-remote-mcp) for details. +::: + +## Troubleshooting + +### Connection Errors + +- Verify your dbt Cloud API token is valid and has **Semantic Layer** and **Developer** permissions +- Ensure the remote MCP server is enabled in your dbt Cloud project + +### Wrong Project or Environment + +- Verify the **Production Environment ID** is set correctly +- Double-check the environment ID matches what's shown in the dbt Cloud URL + +### SQL Queries Not Working + +- Ensure [development credentials](https://docs.getdbt.com/docs/cloud/connect-data-platform/connect-your-database) are configured in dbt Cloud for the user +- Verify the **Development Environment ID** and **User ID** are set +- Check that the API token has permissions to execute queries diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/github.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/github.md new file mode 100644 index 00000000..11748f5c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/github.md @@ -0,0 +1,166 @@ +--- +title: GitHub Plugin +slug: /features/feature-guides/ask-datahub-plugins/github +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub-plugins/github.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# GitHub Plugin + + + +> **Note**: Ask DataHub Plugins is currently in **Private Beta**. To enable this feature, please reach out to your DataHub customer support representative. + +The **GitHub Plugin** connects Ask DataHub to GitHub via the [GitHub MCP server](https://docs.github.com/en/copilot/how-tos/provide-context/use-mcp/use-the-github-mcp-server), enabling the AI assistant to browse repositories, review code, raise pull requests, and more — all from within Ask DataHub. + +## Why Connect GitHub? + +With the GitHub plugin enabled, Ask DataHub can: + +- **Root-cause data issues** — when DataHub assertions detect a problem, trace it back to the source by reviewing recent commits and pull requests that touched the relevant transformation logic or pipeline code. +- **Data development** — browse dbt models, inspect pipeline definitions, and raise pull requests to fix issues — all without leaving the chat. Go from detecting a data quality problem to shipping a fix in one conversation. +- **Code exploration** — find and inspect transformation logic, configuration files, and pipeline definitions across your repositories to understand how data is produced and transformed. + +**Example prompts:** + +- _"Show me recent changes to the ETL pipeline in the data-platform repo"_ +- _"What PRs were merged this week that touch the revenue model?"_ +- _"Find the dbt model definition for customer_churn"_ +- _"Raise a PR to fix the null-handling bug in the orders transformation"_ + +## Prerequisites + +- A GitHub account (personal or organization) +- Admin access to create OAuth Apps in GitHub +- DataHub Cloud with Ask DataHub Plugins enabled +- Platform admin access in DataHub to configure the plugin + +## Admin Setup + +The GitHub plugin uses **User OAuth** authentication — each user authenticates with their own GitHub account. You'll need to create a GitHub OAuth App and configure it in DataHub. + +You can create the OAuth App under your **organization** (recommended) or your **personal account**: + +- **Organization app**: Immediately available to all org members, not scoped to just one user's repositories +- **Personal app**: Scoped to your personal repositories by default + +### Option A: Organization OAuth App (Recommended) + +#### Step 1: Create OAuth App in GitHub + +1. Open GitHub and navigate to your organization's OAuth Apps page: + `https://github.com/organizations//settings/applications` + +

+ +

+ +2. Click **New OAuth App** and fill in the application details: + +| Field | Value | +| ------------------------------ | ---------------------------------------------------------- | +| **Application name** | `DataHub AI Plugin` (or any name your team will recognize) | +| **Homepage URL** | `https://` | +| **Authorization callback URL** | `https:///integrations/oauth/callback` | + +:::caution Callback URL +The **Authorization callback URL** must match the OAuth Callback URL shown in DataHub exactly (e.g. `https://your-org.acryl.io/integrations/oauth/callback`). You can copy this URL from the DataHub plugin creation form. +::: + +3. Click **Register application** + +

+ +

+ +#### Step 2: Collect Credentials + +1. Copy the **Client ID** from the app page +2. Click **Generate a new client secret** and copy it immediately (it won't be shown again) + +

+ +

+ +#### Step 3: Create Plugin in DataHub + +1. Navigate to **Settings > AI > Plugins** in DataHub +2. Click **+ Create** and select **GitHub MCP** +3. Fill in the plugin details: + +| Field | Value | +| ------------------ | ----------------------------- | +| **Name** | `GitHub` | +| **Description** | A description for the plugin | +| **Client ID** | The client ID from Step 2 | +| **Client Secret** | The client secret from Step 2 | +| **Default Scopes** | `repo` (required — see below) | + +4. Copy the **OAuth Callback URL** shown in the DataHub form and verify it matches the **Authorization callback URL** you entered in GitHub in Step 1 + +

+ +

+ +:::warning Required Scope +You must include `repo` in the scopes so Ask DataHub can read repository code, commits, and pull requests. Without this scope, the plugin will have very limited functionality. +::: + +5. Optionally add **Instructions for the AI Assistant**, ensure **Enable for Ask DataHub** is on, and click **Create** + +:::tip Recommended: Add Custom Instructions +The GitHub OAuth App grants access to all repositories the user can see — which can be a lot. We strongly recommend using the **Instructions for the AI Assistant** field to guide the AI toward the repositories and organizations most relevant to your data team. For example: + +_"Focus on the `acme/data-platform` and `acme/etl-pipelines` repositories. These contain our core data transformation and pipeline code. When looking for dbt models, check `acme/dbt-models`."_ + +This helps the AI find relevant code faster and avoids searching across unrelated repos. +::: + +### Option B: Personal Account OAuth App + +The steps are the same as above, except navigate to **GitHub > Your Profile > Settings > Developer Settings > OAuth Apps** instead of organization settings. When creating the app, open both GitHub and DataHub side by side — you'll need to copy the OAuth Callback URL from DataHub into GitHub, and the Client ID/Secret from GitHub into DataHub. + +### Recommended Scopes + +| Scope | Required | Access | +| ---------- | -------- | ----------------------------------------------- | +| `repo` | Yes | Full access to repositories (code, PRs, issues) | +| `read:org` | No | Read organization data (teams, members) | +| `user` | No | Read user profile information | + +## User Setup + +Navigate to **Settings > My AI Settings**, find the **GitHub** plugin, and click **Connect**. A popup will open asking you to authorize the OAuth App on GitHub. After authorization, you'll be redirected back to DataHub. See the [overview](./overview.md#user-setup-enabling-plugins) for more details. + +

+ +

+ +## FAQ + +### Can I use a personal OAuth App instead of an organization one? + +Yes. A personal OAuth App works the same way — the scopes, authorization flow, and plugin configuration are identical. The difference is in management and access: + +- **Organization apps** are owned by the org and visible to all org admins. Any org member who authorizes the app can access org repositories. This is the recommended approach because it's easier for your team to manage and doesn't depend on a single person's account. +- **Personal apps** are tied to your personal GitHub account. They still work — when other users authorize the app, they'll be able to access any repository they personally have access to (including org repos). However, if your account is deactivated, the app stops working. + +We recommend organization-level apps for production use. Personal apps are fine for testing or individual use. + +## Troubleshooting + +### OAuth Popup Blocked + +If the authorization popup doesn't appear, check your browser's popup blocker settings and allow popups from your DataHub instance URL. + +### Authorization Fails + +- Verify the **Authorization callback URL** in GitHub matches the OAuth Callback URL shown in DataHub exactly +- Ensure the Client ID and Client Secret are entered correctly in DataHub +- For organization apps, ensure the OAuth App is not restricted by organization policies + +### Limited Repository Access + +- For organization OAuth Apps, users may need to [grant access to specific organizations](https://docs.github.com/en/apps/oauth-apps/using-oauth-apps/authorizing-oauth-apps) during the authorization flow +- Verify the scopes include `repo` for full repository access diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/glean.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/glean.md new file mode 100644 index 00000000..0dad31ee --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/glean.md @@ -0,0 +1,107 @@ +--- +title: Glean Plugin +slug: /features/feature-guides/ask-datahub-plugins/glean +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub-plugins/glean.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Glean Plugin + + + +> **Note**: Ask DataHub Plugins is currently in **Private Beta**. To enable this feature, please reach out to your DataHub customer support representative. + +The **Glean Plugin** connects Ask DataHub to your organization's [Glean](https://www.glean.com/) instance, bringing enterprise knowledge search into your data conversations. + +## Why Connect Glean? + +With the Glean plugin enabled, Ask DataHub can: + +- **Search organizational knowledge** — find relevant documentation, wikis, Slack threads, and other content indexed by Glean +- **Supplement metadata with context** — answer questions that require both data catalog knowledge and organizational documentation +- **Bridge the gap** between your data ecosystem and your team's tribal knowledge stored across tools + +**Example prompts:** + +- _"Find our internal documentation about the data retention policy"_ +- _"Search for any runbooks related to the revenue pipeline"_ +- _"What does our wiki say about the customer segmentation methodology?"_ +- _"Find Slack discussions about the recent data migration"_ + +## Prerequisites + +- A Glean account with the MCP server enabled (see [Glean's MCP documentation](https://developers.glean.com/guides/mcp)) +- DataHub Cloud with Ask DataHub Plugins enabled +- Platform admin access in DataHub to configure the plugin + +## Admin Setup + +The Glean plugin is the simplest to set up. DataHub supports **OAuth discovery and automatic client registration** for Glean — meaning DataHub will automatically discover the OAuth endpoints, available scopes, and register a client on your behalf. No manual OAuth configuration is required. + +### Step 1: Get the Glean MCP Server URL + +Obtain the MCP server URL from your Glean instance. You can find this in your Glean settings under **MCP Configurator** (accessible from your profile settings), or refer to [Glean's MCP setup guide](https://developers.glean.com/guides/mcp). + +### Step 2: Create Plugin in DataHub + +1. Navigate to **Settings > AI > Plugins** in DataHub +2. Click **+ Create** and select **Custom MCP** +3. Fill in the plugin details: + +| Field | Value | +| ----------------------- | -------------------------------------- | +| **Name** | `Glean` | +| **Description** | A description for the plugin | +| **MCP Server URL** | Your Glean MCP server URL | +| **Authentication Type** | `User OAuth (Each user authenticates)` | + +4. DataHub will automatically discover the OAuth configuration from Glean and register a client — no manual OAuth setup is required. This includes: + + - Authorization and token endpoints + - Available scopes + - Client registration + +5. Review the discovered scopes. All scopes are selected by default. **Ensure the `mcp` scope is selected** — this is required for the plugin to function. + +

+ +

+ +:::warning Required Scope +The **`mcp`** scope must be selected for the Glean plugin to work. Without it, the MCP server will return a `401 Unauthorized` error when Ask DataHub tries to use the plugin. If you see this error after setup, edit the plugin and verify that the `mcp` scope is enabled. +::: + +6. Optionally add **Instructions for the AI Assistant**, ensure **Enable for Ask DataHub** is on, and click **Create** + +## User Setup + +Navigate to **Settings > My AI Settings**, find the **Glean** plugin, and click **Connect**. You'll be redirected to Glean to authenticate, then back to DataHub. See the [overview](./overview.md#user-setup-enabling-plugins) for more details. + +

+ +

+ +## Usage Tips + +- Glean plugin complements DataHub's [Context Documents](../context/context-documents.md) — use Context Documents for curated, governable knowledge and Glean for broader organizational search +- The AI will automatically decide when to use Glean vs. DataHub's built-in search based on the question +- Be specific in your prompts about what kind of documentation you're looking for + +## Troubleshooting + +### 401 Unauthorized Error + +If you see `OAuth succeeded but MCP server returned 401: Client error '401 Unauthorized'`, the `mcp` scope is not selected for the plugin. Edit the plugin in **Settings > AI > Plugins**, ensure the `mcp` scope is enabled, and save. + +### OAuth Discovery Fails + +- Verify the Glean MCP server URL is correct and accessible +- Ensure your Glean instance supports OAuth discovery and dynamic client registration +- Contact your Glean administrator to verify MCP server access is enabled + +### No Results + +- Verify that Glean has indexed the content sources you expect +- Try broader search terms +- Check that the authenticated user has access to the relevant content in Glean diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/overview.md new file mode 100644 index 00000000..732efb67 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/overview.md @@ -0,0 +1,174 @@ +--- +title: Ask DataHub Plugins +slug: /features/feature-guides/ask-datahub-plugins/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub-plugins/overview.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Ask DataHub Plugins + + + +> **Note**: Ask DataHub Plugins is currently in **Private Beta**. To enable this feature, please reach out to your DataHub customer support representative. + +**Ask DataHub Plugins** let you connect external tools — like Snowflake, Databricks, BigQuery, dbt Cloud, GitHub, Glean, and more — directly into [Ask DataHub](../ask-datahub.md) via the [Model Context Protocol (MCP)](../mcp.md). + +With Plugins, can connect any MCP-compatible server that supports [Streamable HTTP transport](https://modelcontextprotocol.io/docs/concepts/transports). Once connected, Ask DataHub can use these tools alongside your metadata catalog to answer richer questions and perform cross-tool workflows. + +## Why Use Plugins? + +Plugins unlock powerful capabilities that go beyond what metadata alone can provide: + +- **Conversational analytics**: Find the right data in DataHub, then query it directly on Snowflake, Databricks, or BigQuery — all in a single conversation. DataHub acts as a semantic layer, guiding the AI to the right tables before executing queries. +- **Cross-tool debugging**: Root-cause a data issue by checking dbt run history, querying tables on Snowflake, and reviewing recent code changes on GitHub — without leaving the chat. +- **Governance workflows**: Verify data lineage in DataHub, check transformation logic in dbt, and review access controls — bringing together context from multiple systems. + +## Available Plugins + +DataHub provides built-in plugin templates for popular tools. For connecting to other MCP Servers — including internal services, custom APIs, or any tool with an MCP server — use the **Custom MCP** option. + +| Plugin | Description | Guide | +| -------------- | --------------------------------------------------- | --------------------------------------------- | +| **Snowflake** | Query and explore your Snowflake data | [Setup Guide](./snowflake.md) | +| **Databricks** | Query and explore your Databricks data | [Setup Guide](./databricks.md) | +| **BigQuery** | Query and explore your BigQuery data | [Setup Guide](./bigquery.md) | +| **dbt Cloud** | Check job runs, model definitions, and test results | [Setup Guide](./dbt.md) | +| **GitHub** | Browse repositories, raise PRs, and manage issues | [Setup Guide](./github.md) | +| **Glean** | Search your organization's knowledge base | [Setup Guide](./glean.md) | +| **Custom MCP** | Connect any MCP-compatible server | [See below](#configuring-a-custom-mcp-plugin) | + +

+ +

+ +## How It Works + +Plugins follow a two-step workflow: + +1. **Admin configures** the plugin by connecting to an MCP server (via **Settings > AI > Plugins**) +2. **Users enable** plugins they want to use from their personal settings (via **Settings > My AI Settings**) or directly from the chat interface + +Once a user enables a plugin, its tools become available during Ask DataHub conversations. The AI assistant will automatically use these tools when relevant to a question. + +## Admin Setup: Configuring Plugins + +:::info Required Permissions +You must have the **Manage Platform Settings** privilege to configure plugins. +::: + +### Creating a Plugin + +1. Navigate to **Settings > AI > Plugins** +2. Click **+ Create** in the top-right corner +3. Select a built-in plugin template (Snowflake, dbt Cloud, GitHub) or choose **Custom MCP** for other tools + +

+ +

+ +### Configuring a Custom MCP Plugin + +Use **Custom MCP** to connect any MCP-compatible server — whether it's a third-party tool like Notion or Glean, an internal service your team has built, or any other API exposed via MCP. The server must support [Streamable HTTP transport](https://modelcontextprotocol.io/docs/concepts/transports). + +When creating a Custom MCP plugin, provide: + +| Field | Description | +| ------------------------------------- | ------------------------------------------------------------------------------- | +| **Name** | A display name for the plugin (visible to all users) | +| **Description** | A brief description of what this plugin provides | +| **MCP Server URL** | The URL of the remote MCP server | +| **Authentication Type** | How users authenticate with the MCP server ([see below](#authentication-types)) | +| **Instructions for the AI Assistant** | Custom instructions fed to Ask DataHub when using this plugin (optional) | + +

+ +

+ +### Authentication Types + +Plugins support three authentication mechanisms: + +#### None (Public API) + +No authentication required. Use this for publicly accessible MCP servers that don't require credentials. + +#### Shared API Key (System-wide) + +A single API key shared across all users. The admin provides the key during setup, and it is used for every user's requests. Best for tools where a single service account is appropriate. + +

+ +

+ +#### User API Key (Each User Provides Their Own) + +Each user provides their own API key when enabling the plugin. This ensures per-user access control and audit trails. Users will be prompted to enter their key when they connect to the plugin. + +

+ +

+ +#### User OAuth (Each User Authenticates) + +Each user authenticates via OAuth when enabling the plugin. The admin configures the OAuth provider (client ID, client secret, authorization URL, token URL, scopes), and users go through a standard OAuth login flow to connect. + +

+ +

+ +### Testing the Connection + +After configuring a plugin, click **Test Connection** to verify that the MCP server is reachable and the credentials are valid before saving. + +### Custom Instructions + +The **Instructions for the AI Assistant** field lets you provide guidance to Ask DataHub on how to use the plugin. For example: + +- _"Use this plugin to query Snowflake when users ask about revenue or customer metrics."_ +- _"Always check dbt test results before reporting on data quality."_ + +These instructions are injected into the AI context whenever the plugin is active. + +## User Setup: Enabling Plugins + +Once an admin has configured a plugin, any user can enable it for their own Ask DataHub experience. + +### From Personal Settings + +1. Navigate to **Settings > My AI Settings** +2. You'll see a list of all available plugins configured by your admin +3. Toggle a plugin **on** to enable it, or click **Connect** to authenticate (for API Key or OAuth plugins) + +

+ +

+ +### From the Chat Interface + +You can also manage plugins directly from the Ask DataHub chat: + +1. Click the **plugins icon** next to the chat input +2. A dropdown shows all available plugins with their connection status +3. Toggle plugins on/off, or click **Manage AI plugins** to configure authentication + +

+ +

+ +For plugins that require authentication, you'll be prompted to provide your credentials: + +

+ +

+ +## Using Plugins in Conversations + +Once enabled, plugin tools are seamlessly available during Ask DataHub conversations. Just ask your question and the AI assistant will use the appropriate tools automatically. + +**Example workflows:** + +- _"What are the most queried tables in Snowflake this month?"_ — DataHub search + Snowflake query +- _"Did the latest dbt run succeed for the revenue model?"_ — dbt Cloud job history +- _"Show me recent changes to the ETL pipeline code"_ — GitHub commit browsing +- _"Find our internal documentation about the data retention policy"_ — Glean knowledge search diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/snowflake.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/snowflake.md new file mode 100644 index 00000000..2d48f21a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub-plugins/snowflake.md @@ -0,0 +1,199 @@ +--- +title: Snowflake Plugin +slug: /features/feature-guides/ask-datahub-plugins/snowflake +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub-plugins/snowflake.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Snowflake Plugin + + + +> **Note**: Ask DataHub Plugins is currently in **Private Beta**. To enable this feature, please reach out to your DataHub customer support representative. + +The **Snowflake Plugin** connects Ask DataHub to a [Snowflake-managed MCP server](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-agents-mcp), enabling conversational analytics directly from the chat. DataHub acts as the **semantic layer** — helping the AI find the right data — while Snowflake serves as the **data layer** — letting the AI actually query and analyze it. + +## Why Connect Snowflake? + +With the Snowflake plugin enabled, Ask DataHub can: + +- **Conversational analytics** — ask questions in plain language and get answers backed by real data. DataHub finds the right tables using metadata (descriptions, ownership, lineage), then Snowflake executes the query. No additional metadata or semantic modeling required. +- **Data exploration** — understand dataset structure, preview sample data, and inspect column values to evaluate data quality or fitness for a use case. +- **Data debugging** — investigate data issues detected by DataHub assertions by inspecting the actual data: check for nulls, duplicates, unexpected values, or freshness problems at the source. + +Because the plugin uses **OAuth**, each user authenticates with their own Snowflake credentials. Users only see data they are authorized to access — no need to manage separate policies or grants in DataHub. + +**Example prompts:** + +- _"Query the top 10 customers by revenue this quarter"_ +- _"How many null values are in the email column of the customers table?"_ +- _"Compare last month's sales to this month across regions"_ +- _"Show me 5 sample rows from the orders table"_ + +## Prerequisites + +- A Snowflake account with `ACCOUNTADMIN` (or equivalent privileges to create MCP servers, security integrations, and grant roles) +- DataHub Cloud with Ask DataHub Plugins enabled +- Platform admin access in DataHub to configure the plugin + +## Admin Setup + +The Snowflake plugin uses **User OAuth** authentication — each user authenticates with their own Snowflake account. Setup involves creating an MCP server and OAuth security integration in Snowflake, then configuring the plugin in DataHub. + +### Step 1: Create an MCP Server in Snowflake + +Create an MCP server in Snowflake with the tools you want to expose. We recommend starting with `SYSTEM_EXECUTE_SQL`, which lets Ask DataHub run SQL queries directly against your warehouse: + +| Tool Type | Purpose | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SYSTEM_EXECUTE_SQL` | **Recommended.** Execute SQL queries directly against the warehouse. This is the minimum required tool for the plugin to be useful. | +| `CORTEX_ANALYST_MESSAGE` | Natural language analytics over [Cortex Analyst semantic views](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-analyst) — a curated BI/analytics layer on top of raw SQL. | +| `CORTEX_AGENT_RUN` | Invoke [Cortex Agents](https://docs.snowflake.com/en/user-guide/snowflake-cortex/cortex-agents-overview) as sub-agents from within Ask DataHub. | + +Snowflake supports [additional tool types](https://docs.snowflake.com/en/sql-reference/sql/create-mcp-server) as well. You can include multiple tools in a single MCP server. + +**Example:** Create a server with SQL execution and a Cortex Analyst semantic view: + +```sql +USE ROLE ACCOUNTADMIN; +USE DATABASE YOUR_DATABASE; +USE SCHEMA YOUR_SCHEMA; + +CREATE OR REPLACE MCP SERVER YOUR_MCP_SERVER + FROM SPECIFICATION $$ + tools: + - name: "execute_sql" + type: "SYSTEM_EXECUTE_SQL" + description: "Execute SQL queries against the data warehouse" + title: "SQL Executor" + - name: "revenue_analytics" + type: "CORTEX_ANALYST_MESSAGE" + identifier: "YOUR_DATABASE.YOUR_SCHEMA.REVENUE_SEMANTIC_VIEW" + description: "Analytics over revenue and sales data" + title: "Revenue Analytics" + $$; +``` + +:::note One Database per MCP Server +Snowflake MCP servers are scoped to a single database. If you need Ask DataHub to query across multiple databases, create a separate MCP server (and plugin) for each one. +::: + +### Step 2: Grant Access + +Grant usage on the MCP server to the roles that should be able to use it: + +```sql +GRANT USAGE ON DATABASE YOUR_DATABASE TO ROLE YOUR_ROLE; +GRANT USAGE ON SCHEMA YOUR_DATABASE.YOUR_SCHEMA TO ROLE YOUR_ROLE; +GRANT USAGE ON MCP SERVER YOUR_DATABASE.YOUR_SCHEMA.YOUR_MCP_SERVER TO ROLE YOUR_ROLE; +``` + +### Step 3: Create an OAuth Security Integration + +Create a security integration to use Snowflake as an OAuth provider. Enable refresh tokens and set the redirect URI to your DataHub instance's OAuth callback: + +```sql +CREATE OR REPLACE SECURITY INTEGRATION YOUR_INTEGRATION_NAME + TYPE = OAUTH + ENABLED = TRUE + OAUTH_CLIENT = CUSTOM + OAUTH_CLIENT_TYPE = 'CONFIDENTIAL' + OAUTH_REDIRECT_URI = 'https:///integrations/oauth/callback' + OAUTH_ISSUE_REFRESH_TOKENS = TRUE + OAUTH_REFRESH_TOKEN_VALIDITY = 7776000; +``` + +:::caution Callback URL +The `OAUTH_REDIRECT_URI` must match the OAuth Callback URL shown in the DataHub plugin creation form exactly (e.g. `https://your-org.acryl.io/integrations/oauth/callback`). You can copy this URL from the form when creating the plugin in Step 5. +::: + +:::caution Refresh Tokens +Make sure `OAUTH_ISSUE_REFRESH_TOKENS` is set to `TRUE`. Without refresh tokens, users will need to re-authenticate frequently. +::: + +### Step 4: Collect Authentication Details + +Run the following queries to gather the information you'll need for DataHub: + +**OAuth Client Credentials:** + +```sql +SELECT SYSTEM$SHOW_OAUTH_CLIENT_SECRETS('YOUR_INTEGRATION_NAME'); +``` + +This returns a JSON object containing `OAUTH_CLIENT_ID` and `OAUTH_CLIENT_SECRET`. + +**MCP Server URL:** + +The URL follows this format: + +``` +https:///api/v2/databases//schemas//mcp-servers/ +``` + +:::note Account URL Format +Use hyphens (`-`) instead of dots (`.`) in the account locator portion of the URL. For example, if your account locator is `abc12345.us-east-1`, your URL would use `abc12345-us-east-1.snowflakecomputing.com`. +::: + +At this point, you should have the values needed for the DataHub configuration: + +| Value | Source | +| ------------------ | --------------------------------------------------------------------------------------- | +| **MCP Server URL** | Constructed from your account URL, database, schema, and server name (see format above) | +| **Client ID** | From `SYSTEM$SHOW_OAUTH_CLIENT_SECRETS` output | +| **Client Secret** | From `SYSTEM$SHOW_OAUTH_CLIENT_SECRETS` output | + +### Step 5: Create Plugin in DataHub + +1. Navigate to **Settings > AI > Plugins** in DataHub +2. Click **+ Create** and select **Snowflake MCP** +3. Fill in the plugin details: + +| Field | Value | +| ------------------ | ---------------------------------------- | +| **Name** | `Snowflake` | +| **Description** | A description for the plugin | +| **MCP Server URL** | The URL from Step 4 | +| **Client ID** | From Step 4 | +| **Client Secret** | From Step 4 | +| **Default Scopes** | `refresh_token session:role:` | + +

+ +

+ +:::warning Choosing the Right Role +The `session:role:` scope determines which Snowflake role the user assumes when connected. Replace `` with the role you granted MCP server access to in Step 2 (e.g. `session:role:DATA_ANALYST`). If omitted, the user's default role is used — which may not have access to the MCP server. Each user **must** have the specified role granted to them in Snowflake, and the role must **not** be in the security integration's blocked roles list (see [Troubleshooting](#users-cant-log-in--oauth-fails)). +::: + +4. Optionally add **Instructions for the AI Assistant**, ensure **Enable for Ask DataHub** is on, and click **Create** + +## User Setup + +Navigate to **Settings > My AI Settings**, find the **Snowflake** plugin, and click **Connect**. You'll be redirected to Snowflake to authenticate, then back to DataHub. See the [overview](./overview.md#user-setup-enabling-plugins) for more details. + +

+ +

+ +## Troubleshooting + +### Users can't log in / OAuth fails + +The OAuth security integration may be blocking the user's role. Ensure the role you want to assume is **not** in the blocked roles list: + +```sql +ALTER SECURITY INTEGRATION "YOUR_INTEGRATION_NAME" + SET BLOCKED_ROLES_LIST = (); +``` + +### Query failures + +- Verify the user's Snowflake role has `SELECT` access to the target tables +- Ensure the user has `USAGE` granted on the MCP server +- Check that the warehouse is running and not suspended + +### Token expiration + +If users are being asked to re-authenticate frequently, verify that refresh tokens are enabled on the security integration (`OAUTH_ISSUE_REFRESH_TOKENS = TRUE`) and that `OAUTH_REFRESH_TOKEN_VALIDITY` is set to an appropriate duration. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub.md new file mode 100644 index 00000000..23824141 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ask-datahub.md @@ -0,0 +1,149 @@ +--- +title: Ask DataHub +slug: /features/feature-guides/ask-datahub +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ask-datahub.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Ask DataHub + + + +**Ask DataHub** is DataHub's conversational AI assistant that brings intelligent, context-aware help directly to where you work. Using Ask DataHub, you can ask questions about your organization's data and get instant, accurate answers grounded in both your **metadata graph** and your **organizational knowledge**—like runbooks, policies, and FAQs stored in Context Graph. + +

+ +

+ +## What Can Ask DataHub Do? + +Ask DataHub empowers your organization to navigate and understand your entire data ecosystem with ease—combining technical metadata with your team's tribal knowledge for smarter, more reliable answers. + +### Find Trustworthy Data + +Discover high-quality, reliable data assets based on usage patterns, documentation quality, ownership information, and data quality metrics. Ask DataHub helps you identify the most authoritative sources for your analysis. + +### Perform Impact Analysis + +Quickly assess how changes to a data asset will ripple through your organization. Ask DataHub can trace lineage and identify all downstream dependencies, helping you make informed decisions before making changes. + +### Dive Deeper + +Understand the who, what, when, where, and why about your data ecosystem: + +- **Ownership**: Find out who owns which data specific assets +- **Popularity**: Understand which data are most popular and who uses it +- **History**: Learn about past usage patterns and organizational knowledge + +### Capture Tribal Knowledge + +Ask DataHub can reference your organization's **Context Documents**, **Glossary Terms**, **Domains**, and more to provide consistent, reliable answers grounded in your team's shared knowledge: + +- **Best practices**: Follow documented runbooks and quality standards +- **Runbooks, guides, & FAQs**: Learn the right process for requesting data access +- **Policies**: Understand compliance requirements and data handling guidelines + +:::tip Improve Ask DataHub Responses +Document your most-asked questions, policies, and definitions in [Context Documents](context/context-documents.md) to help Ask DataHub provide more accurate, consistent answers across your organization. +::: + +### Assess Data Quality + +Quickly understand the health and reliability of your data assets: + +- View assertion results and data quality scores +- Understand freshness and validation status +- Identify potential issues before using data +- Get context on historical quality trends + +### Write SQL Queries + +Generate first-draft SQL queries to answer specific analytical questions, accelerating your data exploration and analysis workflows. + +## Where You Can Use Ask DataHub + +## Getting Started + +### DataHub UI + +Ask DataHub is currently available **Public Beta** within DataHub in multiple places: + +- **Chat** tab in the left-hand navigation bar, where you can start new conversations with DataHub +- **Data Sources** configuration flow, where DataHub can help you troubleshoot ingestion sources +- **Asset Profile Sidebar** where you can start a chat about a specific DataHub asset. +- **Chrome Extension** sidebar under the **Ask Tab**. + +### Slack + +Ask DataHub is available in Slack by mentioning @DataHub in any channel. This brings DataHub's intelligence directly into your team conversations. + +Learn more: [Ask DataHub in Slack](../../managed-datahub/slack/saas-slack-app.md#ask-datahub-in-slack) + +### Microsoft Teams + +Similarly, you can use Ask DataHub in Microsoft Teams by mentioning @DataHub in channels and chats. + +Learn more: [Ask DataHub in Teams](../../managed-datahub/teams/saas-teams-app.md#ask-datahub-in-slack) + +## Customize Ask DataHub + +As of v0.3.15, you can customize how Ask DataHub responds to queries by configuring custom instructions. These are injected into AI context to tailor the AI assistant's behavior to match your organization's specific needs, terminology, and guidelines. + +### Configuring Custom Instructions + +To configure custom instructions: + +1. Navigate to **Settings > AI** in your DataHub instance +2. Locate the **Ask DataHub** section +3. Enter your custom instructions in the **Instructions** field + +That's it! + +

+ +

+ +Custom base prompts can be used to: + +- Define response style and tone +- Define organization specific terminology or terms +- Guide the model on how to navigate Glossary Terms, Tags, Domains, and properties +- Guide the assistant toward specific recommendations (e.g. helping differentiate production over staging assets) + +Once you've changed the custom instructions, it may take up 5 minutes to reflect in Ask DataHub. + +:::tip Context Documents + Custom Instructions +For organization-specific definitions, policies, and procedures, consider documenting them in [Context Documents](context/context-documents.md) instead of (or in addition to) custom instructions. Context Documents are governable, versionable, and can be linked to specific assets—making them easier to maintain as your organization evolves. +::: + +### Selecting a View + +As of v0.3.17, you can select a **View** directly from the chat interface to narrow the scope of assets that Ask DataHub searches over for a given conversation. This is useful when you want to focus on a specific subset of your data ecosystem — for example, only datasets owned by your team, a particular domain, or a curated data product collection. + +To select a view, click the view select dropdown next to the chat input. You can choose from your personal views or any public views shared across your organization. + +

+ +

+ +When a view is active, Ask DataHub will restrict its search to the datasets, dashboards, data products, domains, documents, and other assets included in that view — helping you get to the most relevant answers faster. If no view is explicitly selected, the organization's default view will be used (if one is set). + +:::note Slack & Microsoft Teams +When using Ask DataHub in Slack or Microsoft Teams, the organization's default view is always applied automatically. +::: + +## How It Works + +Ask DataHub leverages your complete metadata graph **and organizational knowledge** to provide intelligent, context-aware responses. The AI assistant considers: + +- Asset names, descriptions, and documentation +- Lineage relationships (upstream and downstream) +- Ownership and domain information +- Usage patterns and popularity metrics +- Data quality and assertion results +- Tags, glossary terms, and classifications +- Schema information and sample values +- Context Documents (published runbooks, policies, FAQs, and definitions) + +By combining structured metadata with unstructured organizational knowledge, Ask DataHub can answer both technical questions ("What's in this table?") and contextual questions ("How should I use this table?") with consistent, reliable answers grounded in your team's shared understanding. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/analytics.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/analytics.md new file mode 100644 index 00000000..eeedb08b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/analytics.md @@ -0,0 +1,148 @@ +--- +title: Form Analytics +sidebar_label: Form Analytics +slug: /features/feature-guides/compliance-forms/analytics +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/compliance-forms/analytics.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# DataHub Compliance Form Analytics + + + +DataHub Cloud provides out-of-the-box analytics to help you monitor and track the success of your Compliance Form initiatives. This guide will walk you through the available reporting views and how to leverage them effectively. + +### Prerequisites + +To access Compliance Form Analytics, you must have the **View Compliance Forms** and/or **Manage Compliance Forms** Platform privilege. + +## Overview + +Form Analytics provides out-of-the-box reporting for Compliance Form completion, enabling Form Owners to easily monitor the progress of their initiatives. The analytics dashboard offers four distinct views to help you understand completion rates and identify areas that need attention: + +1. Overall Progress +2. Form-specific Analysis +3. Domain-based Insights +4. Assignee Performance Tracking + +Each reporting view can be filtered by Assigned Date with the following preset ranges: + +- Last 7 days +- Last 30 days +- Last 90 days +- Last 365 Days +- All Time + +For deeper analysis, you can download the raw data feeding into each dashboard view using the top-right **Download** button. + +:::note +Form Analytics are calculated every day at 12am US PT. +::: + +## Understanding Form States + +Before diving into the different views, it's important to understand how Form and Asset states are tracked: + +### Form States + +Individual Forms can be in one of three states: + +1. **Not Started**: No questions have been answered yet +2. **In Progress**: At least one question has been answered, but not all required questions are complete +3. **Completed**: All required questions are answered (and verified, if it's a Verification Form) + +### Asset States + +Assets can have multiple Forms assigned simultaneously. An Asset's overall status is determined by all its assigned Forms: + +1. **Not Started**: All assigned Forms are in "Not Started" state +2. **In Progress**: At least one Form is either: + - In "In Progress" state + - Or there's a mix of "Completed" and "Not Started" Forms +3. **Completed**: All assigned Forms are in "Completed" state + +## Analytics Views + +### Overall View + +The Overall view provides a high-level summary of Form completion across your organization, helping you quickly identify which Forms, Domains, or Assignees need attention. Use this view as your starting point to determine which drill-down view (Form-specific, Domain-based, or Assignee) will be most helpful for addressing completion gaps. + +Key features: + +- Aggregated view of all assets and their Form status +- Total number of assets in each state +- High-level compliance progress metrics +- Timeline visualization of progress + +

+ Overall Progress Dashboard +

+ +### Form-Specific Analysis + +The Form-specific view allows you to drill down into individual Forms to understand their completion status. This view is particularly useful for: + +- Tracking progress of specific compliance initiatives +- Identifying questions that users might be slow to address +- Generating form-specific reports +- Pinpointing potential bottlenecks in your compliance process + +

+ Form-specific Analysis Dashboard +

+ +### Domain-Based Insights + +The Domain view provides visibility into Form completion rates across different business domains. This perspective helps: + +- Domain managers focus on their area of responsibility +- Compare progress across different domains +- Identify if specific Forms within a domain need attention +- Enable targeted action where necessary + +

+ Domain-based Insights Dashboard +

+ +### Assignee Performance Tracking + +The Assignee view allows you to monitor how specific Users or User Groups are tracking toward their assigned tasks. Use this view to: + +- Track individual user's compliance tasks +- Monitor workload distribution +- Identify potential bottlenecks +- Assess if certain individuals or teams need additional support + +

+ Assignee Performance Dashboard +

+ +## Best Practices + +To make the most of Form Analytics: + +1. **Regular Monitoring**: Check the Overall view weekly to stay on top of completion rates +2. **Targeted Follow-ups**: Use the Domain and Assignee views to identify specific areas or teams that might need additional support +3. **Question Analysis**: Leverage the Form-specific view to identify and address common bottlenecks in your Forms +4. **Data-Driven Decisions**: Download raw data for deeper analysis and to inform future Form design +5. **Workload Management**: Use the Assignee view to ensure tasks are distributed effectively +6. **Domain Oversight**: Leverage the Domain view to ensure consistent progress across your organization + +### Related Features + +- [Create a Compliance Form](create-a-form.md) +- [Complete a Compliance Form](complete-a-form.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/complete-a-form.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/complete-a-form.md new file mode 100644 index 00000000..91d03ffb --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/complete-a-form.md @@ -0,0 +1,181 @@ +--- +title: Complete a Form +sidebar_label: Complete a Form +slug: /features/feature-guides/compliance-forms/complete-a-form +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/compliance-forms/complete-a-form.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Complete a DataHub Compliance Form + + + +Once a Compliance Form has been published (see [Create a Compliance Form](create-a-form.md)), Assignees will receive notifications in their Task Center prompting them to complete the Form for each Asset they are responsible for. + +This guide provides an example of completing a Compliance Form, covering: + +1. Accessing a Form from an Asset Page or the Task Center +2. Completing a Form for a single Asset or multiple Assets (DataHub Cloud only) +3. Understanding different Form Question completion states + +The example uses the **Governance Initiative 2024**, a Verification Form with 3 Required Questions: + +

+ Sample Compliance Form +

+ +## Access a Compliance Form + +Once you have been assigned to complete a Compliance Form, you will see a **Complete Documentation** or **Complete Verification** option on the right-hand side of an Asset Page: + +

+ Open Compliance Form from Asset Page +

+ +**DataHub Cloud** users can find all outstanding Compliance Form requests by navigating to the **Task Center**: + +

+ Open Compliance Form from Task Center +

+ +## Complete a Form for a Single Asset + +When filling out a Compliance Form for a single Asset, you'll see a list of Questions tailored to that Asset, with clear labels showing which ones are required. Here's how it works: + +- **Question Details:** Each Question specifies if it's required or optional. Required Questions must be completed to submit the Form. +- **Pre-Populated Metadata:** If metadata already exists for a Question, it will appear pre-filled. You can confirm the existing value or make updates as needed. +- **Assignee Contributions:** If another Assignee has already provided a response, their name and the time of submission will be displayed. This gives you visibility into previous input, though you can still update the response. + +:::tip +For Verification Forms, after addressing all required Questions, you'll be prompted to provide final sign-off. This ensures all responses are complete and accurate, marking the Form ready for submission. +::: + +Once you complete all required responses, the sidebar will update with the status of the Asset: + +- **Documented**: All required Questions are completed, Verification is not needed +- **Verified**: All required Questions are completed and Verified + +Here's what the **Governance Initiative 2024** Verification Form looks like for `dogs_in_movies` after responding to all Required Questions: + +

+ Asset Ready to Verify +

+ +And here's the `dogs_in_movies` sidebar after Verifying all responses: + +

+ Asset is Verified +

+ +### Navigate to the Next Asset + +To continue working through the Compliance Forms assigned to you, **use the navigation arrows located in the top-right corner**. These arrows will take you to the next Asset that is still pending Form completion or Verification. Only Assets that require action will appear in this flow, allowing you to focus on the remaining tasks without unnecessary steps. + +## Complete a Form Question for Multiple Assets + +When you want to provide the same response for a question to multiple assets, you can apply it in bulk by selecting the **By Question** option in the top-right corner. This allows you to navigate through the Form question-by-question and apply the same response to multiple assets. + +:::note +Completing Form Questions for multiple Assets is only supported for DataHub Cloud. +::: + +### Example: Apply a Response in Bulk + +Let's look at an example. Imagine we are trying to provide the same answer to a Question for all Assets in a Snowflake schema called `DEMO_DB`. Here's how we'd do it: + +1. **Filter Assets**: Filter down to all datasets in the `DEMO_DB` Snowflake schema. +2. **Set a Response**: For the selected Question, provide a response. In this case, we'll set the Deletion Date to be `2024-12-31`. +3. **Apply to All Selected Assets**: Use the bulk application feature to apply this response to all filtered Assets. + +

+ Apply Response to Multiple Assets +

+ +After setting the response, toggle through each Question, providing the necessary responses to combinations of Assets. + +### Verification for Multiple Assets + +For Verification Forms, as you complete Questions, you will see the number of assets eligible for Verification in the top-right corner. This makes it easy to track which Assets have met the requirements. + +

+ Multiple Assets ready to Verify +

+ +When you are ready to bulk Verify Assets, you will be prompted to confirm that all responses are complete and accurate before proceeding. + +

+ Final Bulk Verification +

+ +### Switch Between Completion Modes + +You can easily toggle between the **Complete By Asset** and **Complete By Question** views as needed, ensuring flexibility while completing and verifying the Compliance Forms. + +## Understanding Different Form Question Completion States + +When completing a Compliance Form, you may encounter various types of Questions, each with unique completion states based on existing metadata or prior responses from other Assignees. This section highlights examples of various completion states to help you understand how Questions can be answered, confirmed, or updated when completing a Form. + +**_1. What is the primary use case for this asset?_** + +This required Question is asking the Assignee to provide Documentation on how the Asset should be used. Note that there is no text populated in the description, meaning the Asset does not have any documentation at all. + +

+ Sample Compliance Form +

+ +**_2. When will this asset be deleted?_** + +You may notice that this question has a pre-populated value. When metadata has been populated from a source _outside_ of a Form, users will have the option to update and save the value, or, simply **Confirm** that the value is accurate. + +

+ Sample Compliance Form +

+ +**_3. Who is the Data Steward of this Asset?_** + +Here's an example where a different Form Assignee has already provided an answer through the Compliance Form 3 days ago. All Assignees will still have the option to update the response, but this allows users to see how other Form Assignees have already answered the questions. + +

+ Sample Compliance Form +

+ +## FAQ and Troubleshooting + +**Why don’t I see any Compliance Forms in the Task Center or on an Asset Page?** + +If you don’t see any Compliance Forms, check with the Form author to ensure your DataHub user account has been assigned to complete a Form for one or more Assets. Forms can be assigned to Asset Owners, specific DataHub Users, or a combination of both. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/create-a-form.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/create-a-form.md new file mode 100644 index 00000000..5c7eaa22 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/create-a-form.md @@ -0,0 +1,205 @@ +--- +title: Create a Form +sidebar_label: Create a Form +slug: /features/feature-guides/compliance-forms/create-a-form +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/compliance-forms/create-a-form.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Create a DataHub Compliance Form + + + +This guide will walk you through creating and assigning Compliance Forms, including: + +1. Creating a new Compliance Form +2. Building **Questions** for the Compliance Form +3. Assigning **Assets** for the Compliance Form +4. Selecting **Assignees** for the Compliance Form +5. Publishing a Compliance Form + +:::note +Managing Compliance Forms via the DataHub UI is only available in DataHub Cloud. If you are using DataHub Core, please refer to the [Compliance Forms API Guide](../../../api/tutorials/forms.md). +::: + +### Prerequisites + +In order to create, edit, or remove Compliance Forms, you must have the **Manage Compliance Forms** Platform privilege. + +### Step 1: Create a new Compliance Form + +From the navigation bar, head to **Govern** > **Compliance Forms**. Click **+ Create** to start building your Form. + +

+ View of all Compliance Forms +

+ +First up, provide the following details: + +1. **Name:** Select a unique and descriptive name for your Compliance Form that clearly communicates its purpose, such as **"PII Certification Q4 2024"**. + + _**Pro Tip:** This name will be displayed to Assignees when they are assigned tasks, so make it clear and detailed to ensure it conveys the intent of the Form effectively._ + +2. **Description:** Craft a concise yet informative description that explains the purpose of the Compliance Form. Include key details such as the importance of the initiative, its objectives, and the expected completion timeline. This helps Assignees understand the context and significance of their role in the process. + + _**Example:** "This Compliance Form is designed to ensure all datasets containing PII are reviewed and verified by Q4 2024. Completing this Form is critical for compliance with organizational and regulatory requirements."_ + +3. **Type:** Specify the collection type for the Form, based on your compliance requirements: + + - **Completion:** The Form is considered complete once all required questions are answered for the selected Assets. We recommend this option for basic requirement completion use cases. + + - **Verification:** The Form is considered complete only when all required questions are answered for the selected Assets **and** an Assignee has explicitly "verified" the responses. We recommend this option when final sign-off by Assignees is necessary, ensuring they acknowledge the accuracy and validity of their responses. + +4. Next, click **Add Question** to begin building the requirements for your Form. + +

+ Create a new Compliance Form +

+ +### Step 2: Build Questions for your Form + +Next, define the Questions for your Compliance Forms. These are used to collect required information about selected assets, and must be completed by an Assignee in order for the Form to be considered complete. + +There are 5 different question types to choose from: + +- **Ownership:** Request one or more owners to be assigned to selected assets. Optionally restrict responses to a specific set of valid users, groups, and ownership types. + - _E.g. Who is responsible for ensuring the accuracy of this Dataset?_ +- **Domain:** Assign a Domain to the Asset, with the option to predefine the set of allowed Domains. + - _E.g. Which Domain does this Dashboard belong to? Sales, Marketing, Finance._ +- **Documentation:** Provide Documentation about the Asset and/or Column. + - _E.g. What is the primary use case of this Dataset? What caveats should others be aware of?_ +- **Glossary Terms:** Assign one or more Glossary Term to the Asset and/or Column, with the option to predefine the set of allowed Glossary Terms. + - _E.g. What types of personally identifiable information (PII) are included in this Asset? Email, Address, SSN, etc._ +- **Structured Properties:** Apply custom properties to an Asset and/or Column. + - _E.g. What date will this Dataset be deprecated and deleted?_ + +When creating a Question, use a clear and concise Title that is easy for Assignees to understand. In the Description, include additional context or instructions to guide their responses. Both the Title and Description will be visible to Assignees when completing the Form, so make sure to provide any specific hints or details they may need to answer the Question accurately and confidently. + +

+ Create a new Compliance Form prompt +

+ +### Step 3: Assign Assets to your Compliance Form + +Now that you have defined the Questions you want Assignees to complete, it's now time to assign the in-scope Assets for this exercise. + +In the **Assign Assets** section, you can easily target the specific set of Assets that are relevant for this Form with the following steps: + +1. Add a Condition or Group of Conditions +2. Choose the appropriate filter type, such as: + - Asset Type (Dataset, Chart, etc.) + - Platform (Snowflake, dbt, etc.) + - Domain (Sales, Marketing, Finance, etc.) + - Assigned Owners + - Assigned Glossary Terms +3. Decide between **All**, **Any**, or **None** of the filters should apply +4. Preview the relevant Assets to confirm you have applied the appropriate filters + +For example, you can apply filters to focus on all **Snowflake Datasets** that are also associated with the **Finance Domain**. This allows you to break down your compliance initiatives into manageable chunks, so you don't have to go after your entire data ecosystem in one go. + +

+ Assign assets to a Compliance Form +

+ +### Step 4: Select Assignees to complete your Compliance Form + +With the Questions and assigned Assets defined, the next step is to select the Assignees—the Users and/or Groups responsible for completing the Form. + +In the **Add Recipients** section, decide who is responsible for completing the Form: + +- **Asset Owners:** Any User that is assigned to one of the in-scope Assets will be able to complete the Form. This is useful for larger initiatives when you may not know the full set of Users. +- **Specific Users and/or Groups:** Select a specific set of Users and/or Groups within DataHub. This is useful when Ownership of the Assets may be poorly-defined. + +Additionally, you can configure notification preferences: + +- **Send notifications to assignees when this form is published:** Check this option to automatically notify assignees when the Form becomes available. If enabled, assignees will receive notifications through their configured channels (email and/or Slack in DataHub Cloud) in addition to seeing the task in their DataHub Task Center (DataHub Cloud only). + +

+ Assign recipients to a Compliance Form +

+ +### Step 5: Publish your Form + +After defining the Questions, assigning Assets, and selecting the Assignees, your Form is ready to be published. + +Once published: + +- Assignees will have a task waiting in their **DataHub Task Center** (DataHub Cloud only) +- If you enabled notifications and assignees have configured email and/or Slack notifications (DataHub Cloud only), they will be notified through those channels +- As new assets become eligible for the Form over time, net-new owners will automatically be notified and assigned the Form + +To publish a Form, simply click **Publish**. + +:::caution +Once you have published a Form, you **cannot** change or add Questions. You can, however, change the set of Assets and/or Assignees for the Form. +::: + +Not ready for primetime just yet? No worries! You also have the option to **Save Draft**. + +

+ Publish a Compliance Form +

+ +## FAQ and Troubleshooting + +**Does answering a Compliance Form Question update the selected Asset?** + +Yes! Compliance Forms serve as a powerful tool for gathering and updating key attributes for your mission-critical Data Assets at scale. When a Question is answered, the response directly updates the corresponding attributes of the selected Asset. + +**How does a Compliance Form interact with existing metadata?** + +If an Asset already has existing metadata that is also referenced in a Form Question, Assignees will have the option to confirm the existing value, overwrite the value, or append additional details. + +_You can find more details and examples in the [Complete a Form](complete-a-form.md#understanding-different-form-question-completion-states) guide._ + +**What is the difference between Completion and Verification Forms?** + +Both Form types are a way to configure a set of optional and/or required Questions for DataHub users to complete. When using Verification Forms, users will be presented with a final verification step once all required questions have been completed; you can think of this as a final acknowledgment of the accuracy of information submitted. + +**Can I assign multiple Forms to a single Asset?** + +You sure can! Please keep in mind that an Asset will only be considered Documented or Verified if all required questions are completed on all assigned Forms. + +**How will DataHub Users know that a Compliance Form has been assigned to them?** + +Assignees will be notified in multiple ways: + +- A task will appear in their **DataHub Task Center** (DataHub Cloud only) +- If notifications were enabled when the Form was published and assignees have configured email and/or Slack notifications (DataHub Cloud only), they will receive notifications through those channels +- As new assets become eligible for existing Forms, net-new owners will automatically be notified and assigned the Form + +**How do I track the progress of Form completion?** + +Great question. We are working on Compliance Forms Analytics that will directly show you the progress of your initiative across the selected Assets. Stay tuned! + +### API Tutorials + +- [Compliance Form API Guide](../../../api/tutorials/forms.md) + +### Related Features + +- [DataHub Structured Properties](../../feature-guides/properties/overview.md) + +## Next Steps + +Now that you have created a DataHub Compliance Form, you're ready to [Complete a Compliance Form](complete-a-form.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/overview.md new file mode 100644 index 00000000..bfe58fce --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/compliance-forms/overview.md @@ -0,0 +1,52 @@ +--- +title: Overview +sidebar_label: Overview +slug: /features/feature-guides/compliance-forms/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/compliance-forms/overview.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# About DataHub Compliance Forms + + + +**DataHub Compliance Forms** streamline the process of documenting, annotating, and classifying your most critical Data Assets through a collaborative, crowdsourced approach. + +With Compliance Forms, you can execute large-scale compliance initiatives by assigning tasks (e.g., documentation, tagging, or classification requirements) to the appropriate stakeholders — data owners, stewards, and subject matter experts. + +## What are Compliance Forms? + +A **Compliance Form** is a flexible and centrally managed tool that enables your data governance or compliance teams to define, enforce, and monitor requirements for specific Data Assets or Columns. + +A Compliance Form consists of: + +1. **Assets:** The Data Assets or Columns for which the Form must be completed. These represent the scope of the compliance initiative. + +2. **Questions:** The set of requirements or conditions that must be completed for each asset. Questions are a vehicle to collect key attributes for your data assets. These can range from simple to complex, with questions that require differing types of answers to complete. Examples include Descriptions, Domains, Owners, Tags, Glossary Terms, and custom Structured Properties. + +3. **Assignees:** The users or groups responsible for completing the Form (e.g., asset owners, domain experts, or stewards). + +Once a Compliance Form is defined, it can be published. When a form is published, the assignees who are required to complete the requirements will be notified via the Inbox of the tasks that they must complete. In addition, analytics will begin to be gathered about the assets that are meeting or violating the requirements in the form so you can understand your initiative's progress over time. + +### Why Use Compliance Forms? + +Compliance Forms enable organizations to: + +- Standardize documentation and metadata across critical Data Assets. +- Crowdsource compliance-related tasks to domain experts who are best equipped to provide accurate information. +- Scale governance initiatives efficiently while maintaining accuracy and accountability. + +By leveraging Compliance Forms, organizations can ensure consistent metadata quality and foster collaboration between data experts and governance teams. + +

+ Sample Compliance Form +

+ +## Next Steps + +Now that you understand the basics of DataHub Compliance Forms, you're ready to [Create a Compliance Form](create-a-form.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/context/context-documents.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/context/context-documents.md new file mode 100644 index 00000000..502bad50 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/context/context-documents.md @@ -0,0 +1,106 @@ +--- +title: Context Documents +slug: /features/feature-guides/context/context-documents +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/context/context-documents.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Context Documents + + + +**Context Documents** let you capture tribal knowledge in DataHub. With Context Documents, you can create runbooks, FAQs, process guides, decision logs, and more to capture curated organizational context. + +Once created, documents can be **published** to become visible to your team and AI assistants like [Ask DataHub](../ask-datahub.md) to provide grounded, consistent answers. + +

+ +

+ +## Highlights + +Context Documents are first-class citizens in DataHub. You can: + +- **Classify by type** — Runbook, FAQ, Policy, Decision Log, and more +- **Organize with metadata** — Domains, Tags, Glossary Terms, Owners, and Structured Properties +- **Link related assets** — Connect to the Datasets, Dashboards, and Charts they describe +- **Control visibility** — Publish to share with your organization and AI agents, or keep as draft +- **Track version history** — See changes over time and restore previous versions + +## Creating a Document + +Create documents from the **Documents** section in the left navigation. + +1. Click **+ New** to create a new document. +2. Choose a **parent document** (optional) to nest it in your hierarchy. +3. Enter a title and write your content using the rich text editor. +4. Add metadata (Type, Owner, Tags, Domain) as needed. + +

+ +

+ +## Publishing a Document + +Documents can be in **Draft** or **Published** states. Draft documents are only visible to you - the owner. Published documents are visible to everyone else. + +Toggle **Published** in the document header to publish or unpublish your documents. + +

+ +

+ +:::info Context Documents are created in **Published** state by default. +::: + +## Moving a Document + +Reorganize your document hierarchy at any time. + +1. Open the document or use the context menu in the directory tree. +2. Select **Move**. +3. Choose a new parent (or move to top-level). + +## Searching for Documents + +Find documents using DataHub's primary search bar alongside your data assets, and within the **Documents** tab accessible from the left navigation bar. + +## Document History + +Track changes, view previous versions, and restore document contents by visting the document history timeline. + +

+ +

+ +Change types include: + +- Document is created +- Document title is changed +- Document contents are changed +- Document is published or unpublished +- Related assets are updated + +## Context Documents in Ask DataHub + +When you ask a question in [Ask DataHub](../ask-datahub.md), the AI searches your published documents alongside your metadata graph. If relevant context is found, Ask DataHub cites the document in its response. + +

+ +

+ +_Note: Ask DataHub is available in DataHub Cloud only._ + +## Programmatic Access + +Context Documents are accessible via: + +- **Python SDK**: Create, update, and retrieve documents programmatically. See the [Documents API tutorial](../../../api/tutorials/documents.md). +- **MCP Server**: Expose documents to AI agents and external tools. See the [DataHub MCP Server](../mcp.md). + +## What's Next + +We're actively expanding Context Documents: + +- **External document connectors**: Bring in documentation from tools like Notion and Confluence while keeping the source of truth external. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-asset-summaries.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-asset-summaries.md new file mode 100644 index 00000000..c8fb29f8 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-asset-summaries.md @@ -0,0 +1,181 @@ +--- +title: Asset Summaries +sidebar_label: Asset Summaries +slug: /features/feature-guides/custom-asset-summaries +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/custom-asset-summaries.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Asset Summaries + + + +DataHub's **Asset Summaries** enable organizations to create tailored, curated discovery experiences for their most important logical assets by customizing the Summary tab view that users see when browsing Domains, Data Products, Glossary Terms, and Glossary Term Groups. + +

+ Default custom asset summary page +

+ +:::info Supported Asset Types +Asset Summaries are currently supported for **Domains, Data Products, Glossary Terms, and Glossary Term Groups only**. These logical asset types help organize and group your physical data assets within DataHub. +::: + +## Why Use Asset Summaries? + +Asset Summaries transform how your team discovers and navigates key organizational assets by enabling you to: + +- **Improve Asset Discoverability**: Highlight the most relevant information about your domains, data products, and glossary terms right on the Summary tab +- **Create Owner-Driven Experiences**: Asset owners can curate what information is most important for their users to see +- **Reduce Navigation Time**: Surface key details, documentation, and related assets without requiring users to click through multiple tabs +- **Standardize Asset Presentation**: Ensure consistent, professional presentation of your most important organizational constructs + +Whether you're a domain owner wanting to highlight key data products, a data product manager showcasing important assets, or a data steward organizing glossary terms, Asset Summaries put the most relevant information front and center. + +## What's Included + +Asset Summaries consist of three customizable sections that you can tailor to your asset's specific needs: + +### Summary Page Header + +The header section showcases key properties about your asset for immediate visibility. By default, this includes: + +- **Ownership information** - Who owns and manages this asset +- **Creation date** - When the asset was established +- **Tags** - Asset classifications (available for assets that support tagging) +- **Glossary terms** - Business context and definitions applied to the asset +- **Domain association** - Organizational placement +- **Structured properties** - Custom metadata fields you want to highlight + +You can customize this section to show the properties most relevant to your users, including removing default properties or adding new ones. Note that property availability depends on what each asset type supports - for example, not all logical asset types currently support tags. + +### Documentation and Links Section + +This section brings documentation capabilities directly to the Summary tab for easy access: + +- **Rich text documentation** - Add, edit, and format detailed descriptions about your asset +- **Resource links** - Include links to external documentation, dashboards, tools, or other relevant resources +- **Quick access** - Users no longer need to navigate to a separate Documentation tab + +### Module Section + +The module section provides contextual information and navigation for your asset through both default and custom modules: + +#### Default Modules + +Each asset type includes relevant default modules: + +- **Assets Module** (Domains, Data Products, Glossary Terms): Shows data assets that belong to this domain, data product, or are tagged with this glossary term +- **Domains Module** (Domains only): Displays the hierarchy of child domains within this domain +- **Data Products Module** (Domains only): Lists all data products contained within this domain +- **Contents Module** (Glossary Term Groups only): Shows child glossary terms and term groups for easy hierarchy navigation +- **Related Terms Module** (Glossary Terms only): Lists all terms that share relationships with the current term + +#### Custom Modules + +You can also add the same custom modules available on the home page: + +- **Collection**: Curated lists of specific assets you want to highlight +- **Documentation**: Pin important documentation that users should see +- **Hierarchy**: Top-down view of assets organized by domains or glossary terms + +

+ Summary page module examples +

+ +## Customizing Asset Summaries + +:::note +Customization is currently only supported in DataHub Cloud. DataHub Core will support out of the box summary pages with no way to customize. +::: + +### Editing Summary Page Headers + +To customize which properties appear in your asset's header: + +1. Navigate to your asset's Summary tab +2. Click the **plus icon** in the header section to add new properties +3. Replace or remove existing properties by clicking their name + +

+ Edit header properties +

+ +### Managing Documentation and Links + +Add or update documentation and links directly on the Summary tab: + +1. Click **"Edit Description"** icon on the right side of the "About" section header +2. Use the rich text editor to format your content +3. Click **"Add Link"** icon on the right side of the "About" section header +4. Organize links by editing or removing them as needed where they are listed + +

+ Edit documentation section +

+ +### Adding and Arranging Modules + +Customize the module section to highlight the most important information: + +1. Click the "plus" button below existing modules to browse available module types +2. Drag and drop modules to reorder them +3. Use the edit or remove options in the module menu to modify existing modules + +

+ Add module menu +

+ +:::note Global Changes +Changes you make to an asset's Summary page are **visible to all users** who view that asset. Your customizations become the default experience for everyone accessing that domain, data product, or glossary term. +::: + +## Permissions and Access Control + +To customize Asset Summaries, users need the **"Manage Asset Summary"** privilege for the specific asset they want to edit. This permission can be configured through DataHub's policy editor alongside other access controls. By default, Admins, Editors, and asset owners will be granted this permission. + +### Setting Up Permissions + +1. Navigate to **Settings > Permissions > Policies** +2. Create or edit a policy to include the "Manage Asset Summary" privilege +3. Assign the policy to appropriate users or groups +4. Apply the policy to specific assets or asset types + +This granular permission system ensures that only authorized users can modify how critical organizational assets are presented to the broader team. + +## Best Practices + +When customizing your Asset Summaries: + +- **Focus on user needs**: Include properties and modules that help users understand and navigate your assets effectively +- **Keep it relevant**: Don't overcrowd the Summary page - prioritize the most important information +- **Update regularly**: Review and refresh your customizations as your assets and organization evolve +- **Consider your audience**: Tailor the presentation to match how your team typically uses these assets +- **Leverage modules strategically**: Use default modules to show relationships and custom modules to highlight specific collections or documentation + +## Next Steps + +Now that you understand Asset Summaries: + +1. **Start with high-traffic assets**: Begin by customizing the Summary pages for your most frequently accessed domains and data products +2. **Gather feedback**: Ask users what information would be most helpful to see on Summary pages +3. **Iterate based on usage**: Monitor how users interact with your customized summaries and adjust accordingly +4. **Scale thoughtfully**: Apply learnings from your initial customizations to other assets in your organization + +Asset Summaries are designed to evolve with your organization's needs, providing the flexibility to create discovery experiences that truly serve your users while maintaining the structure that makes DataHub powerful. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-home-page.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-home-page.md new file mode 100644 index 00000000..8b6ca72b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/custom-home-page.md @@ -0,0 +1,256 @@ +--- +title: Home Page +sidebar_label: Home Page +slug: /features/feature-guides/custom-home-page +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/custom-home-page.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Home Page + + + +DataHub's **Home Page** empowers organizations and individual users to create personalized, modular home page experiences that put the most relevant data assets and information front and center. + +

+ Default custom home page +

+ +## Why Use the Home Page? + +The Home Page transforms how your team interacts with DataHub by enabling you to: + +- **Reduce Time to Insight**: Surface the most critical data assets, domains, and resources directly on your home page for instant access +- **Personalize Your Workflow**: Create a home page experience tailored to your specific role, projects, and data needs +- **Improve Data Discoverability**: Highlight important collections, documentation, and quick links that help your team find what they need faster +- **Standardize Organization Views**: Administrators can create consistent default experiences while still allowing individual customization + +Whether you're a data analyst who needs quick access to specific dashboards, a data engineer focused on particular domains, or an administrator wanting to promote data governance resources, the Home Page adapts to your workflow. + +## What's Included + +The Home Page consists of **modules** that you can arrange in rows of up to 3 modules each. You can choose from: + +### Default Modules + +- **Your Assets**: Data assets that you own, giving you quick access to the assets you're responsible for +- **Your Subscriptions**: Assets you've subscribed to for updates and notifications +- **Domains**: A selection of the most-used domains in your organization for easy navigation + +### Custom Modules + +- **Quick Link**: Links to important external references like documentation sites, dashboards, or tools +- **Collection**: Curated collections of data assets that you want to highlight for easy discoverability +- **Documentation**: Pinned documentation that DataHub users should see on their home page +- **Hierarchy**: A top-down view of assets organized by selected Domains or Glossary Terms/Term Groups + +### Organization Announcements + +Important announcements from your organization appear above all modules, with the ability for users to individually dismiss them. + +

+ Module examples +

+ +### Default Setup + +When you upgrade to this version, your default home page will automatically include: + +- Your organization's existing pinned links at the top +- Row 1: Your Assets and Your Subscriptions modules +- Row 2: Domains module + +## Personalizing the Home Page + +When you begin customizing your home page, you'll "fork" from your organization's default to create your own personal version. This means your changes won't affect what other users see, and you can always return to the organization default if needed. + +:::note +Customization is currently only supported in DataHub Cloud. DataHub Core will support a default home page with no way to customize. +::: + +### Add New Modules + +Click **"Add Module"** to browse and select from available module types. Custom module can be configured with specific settings: + +

+ Add module menu +

+ +### Rearrange Modules + +Drag and drop modules to reorder them within rows or move them between rows. You can have up to 3 modules per row. + +

+ Move module example +

+ +### Edit Existing Modules + +Click the **edit icon** on any module to modify its settings, title, or content. Currently, editing is only available for custom modules. + +

+ Edit module modal +

+ +### Remove Modules + +Click the **remove icon** on any module to delete it from your home page. Don't worry - you can always add default modules back later. + +

+ Edit module menu +

+ +### Reset to Default + +If you want to return to your organization's default home page: + +1. Click the **settings icon** in the bottom right corner +2. Select **"Reset to Default Home Page"** +3. Confirm that you want to remove your personal customizations + +This will delete your personal home page template and return you to the organization default. + +

+ Option to reset to default menu +

+ +

+ Confirm reset to default modal +

+ +:::tip +Even when you have a personal home page, you can always grab modules from the default home page to add to your personal view. This lets you stay up-to-date with organizational standards while maintaining your customizations. +::: + +## Configuring the Default Home Page + +Users with the `MANAGE_HOME_PAGE_TEMPLATES` privilege (administrators by default) can customize the default home page that all users see when they first log in. + +### Editing the Organization Default + +To modify the default home page for your organization: + +1. Click the **settings icon** in the bottom right corner of your home page +2. Select **"Edit Organization Default"** + +

+ Edit org default menu +

+ +When you enter organization default edit mode, you'll see the current default template. Any changes you make here will: + +- **Save directly to the organization default**: Changes take effect immediately for all users using the default template +- **Appear for new users**: Anyone who hasn't customized their home page will see your updates +- **Not override personal templates**: Users who have created personal home pages will continue to see their customized versions + +

+ Edit org default page +

+ +### Managing Default Modules + +As an administrator editing the organization default, you can: + +- Add modules that provide value to most users in your organization +- Remove modules that aren't relevant to your team's workflow +- Rearrange modules to prioritize the most important information +- Update module configurations (like which domains to feature) + +### Best Practices for Default Templates + +When configuring your organization's default home page: + +- **Include broadly useful modules**: Choose modules that provide value to most users, like "Your Assets" and "Domains" +- **Feature organizational resources**: Use Quick Link and Documentation modules to promote important company resources +- **Keep it focused**: A cluttered default page can overwhelm new users - aim for 3-6 well-chosen modules +- **Update regularly**: Review and refresh the default template periodically to ensure it stays relevant + +:::note +Administrators can have both the ability to edit the organization default AND maintain their own personal home page customizations. The two are independent. +::: + +## Next Steps + +Now that you understand how to customize your home page experience: + +1. **Start with small changes**: Try rearranging existing modules or adding a Quick Link to see how the interface works +2. **Experiment with modules**: Each module type offers different ways to surface important information +3. **Gather feedback**: If you're an administrator, ask users what they'd find most valuable on the default home page +4. **Iterate over time**: Home page needs may change as your team and data landscape evolve + +The Home Page is designed to grow with your organization's needs, providing both the flexibility for individual workflows and the consistency that administrators need to promote important resources and best practices. + +## Relevant APIs + +### Upserting a Template + +A template is what is displayed on your home page. It includes a list of rows, each row contains a list of module urns. + +```graphql +mutation upsertPageTemplate { + upsertPageTemplate( + input: { + rows: [{ modules: ["urn:li:dataHubPageModule:your_assets"] }] + scope: GLOBAL + surfaceType: HOME_PAGE + } + ) { + urn + } +} +``` + +### Upserting a Module + +A module is each individual "widget" shown on the home page. These exist in the context of a template + +```graphql +mutation upsertPageModule { + upsertPageModule( + input: { + name: "My Custom Module" + type: RICH_TEXT + scope: PERSONAL + params: { + richTextParams: { + content: "This is the content of my documentation module" + } + } + } + ) { + urn + } +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/file-upload-download.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/file-upload-download.md new file mode 100644 index 00000000..3b7df04c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/file-upload-download.md @@ -0,0 +1,206 @@ +--- +title: File Upload and Download in Documentation +sidebar_label: File Upload and Download in Documentation +slug: /features/feature-guides/file-upload-download +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/file-upload-download.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# File Upload and Download in Documentation + + + +DataHub's File Upload and Download capability enables you to enrich your asset and column documentation with supporting files like images, diagrams, and other resources, making your data catalog more informative and easier to understand. + +

+ Dropping a file to upload +

+ +## Why Use File Uploads in Documentation? + +File uploads transform how you document and communicate about your data assets by enabling you to: + +- **Enhance Documentation Quality**: Add visual diagrams, architecture drawings, and screenshots directly into your documentation to illustrate complex concepts +- **Centralize Resources**: Keep all relevant documentation materials in one place alongside your data assets rather than scattered across multiple systems +- **Improve Understanding**: Help users grasp data structures, pipelines, and relationships faster with visual aids embedded directly in context +- **Standardize Communication**: Create consistent, professional documentation that combines text and visual elements + +Whether you're a data engineer documenting a complex data pipeline, a data analyst explaining column definitions with example screenshots, or a data steward providing reference materials, file uploads make your documentation more comprehensive and accessible. + +## What's Included + +### Supported File Types + +Currently, images are displayed inline within your documentation. Files of other types can be uploaded and downloaded, with support for additional inline previews (like PDFs and text files) coming in future releases. + +### Upload Methods + +You have two convenient ways to add files to your documentation: + +- **Drag and Drop**: Simply drag files from your file system directly into the documentation editor +- **Upload Button**: Click the upload file button in the editor toolbar to browse and select files from your file system + +

+ Option to choose a file from your system +

+ +### File Management + +When you upload a file, DataHub automatically: + +- Creates a new `dataHubFile` entity to track metadata about your file (type, size, original filename) +- Stores the context of where you uploaded the file (asset documentation, column documentation, etc.) +- Uses this context information to enforce permissions when users attempt to download files + +Files are securely stored in S3, and access is controlled based on the user's permissions for the asset where the file was uploaded. + +## Setting Up File Uploads + +File upload functionality requires configuration to connect DataHub to your S3 storage. + +### Prerequisites + +Your DataHub instance must be deployed on AWS for the AWS role authentication to work seamlessly. + +### Required Configuration + +Set the following environment variables in your DataHub GMS service: + +#### DATAHUB_BUCKET_NAME + +The name of your S3 bucket where uploaded files will be stored. + +```bash +DATAHUB_BUCKET_NAME=your-datahub-files-bucket +``` + +#### DATAHUB_ROLE_ARN + +The AWS role ARN configured with permissions to upload and download files from your S3 bucket. + +```bash +DATAHUB_ROLE_ARN=arn:aws:iam::123456789012:role/DataHubFileAccessRole +``` + +### AWS Role Configuration + +Your AWS role must be properly configured with two key requirements: + +#### S3 Permissions + +The role needs appropriate permissions to interact with your S3 bucket. At minimum, this should include: + +- `s3:PutObject` - To upload files +- `s3:GetObject` - To download files +- `s3:DeleteObject` - For garbage collection of unused files + +Configure these permissions through an IAM policy attached to the role. + +#### Trust Relationship + +Update the trust relationship for your role to allow your DataHub GMS service to assume it. The trust policy should permit the AWS service or role that your DataHub GMS is running under to assume this role. + +Without proper trust relationship configuration, DataHub will not be able to authenticate with AWS to access your S3 bucket. + +## Uploading Files + +To add files to your documentation: + +1. Navigate to the asset or column where you want to add documentation +2. Open the documentation editor +3. Either: + - Drag and drop your file directly into the editor, or + - Click the upload file button in the toolbar and select your file from the file system dialog +4. Images will appear inline immediately; other file types will show as downloadable links +5. Save your documentation changes + +Files are uploaded to S3 as soon as you add them to the editor, even before you save your documentation changes. + +

+ Image file rendered inline +

+ +## Downloading Files + +When users view documentation containing uploaded files: + +- **Images**: Display inline automatically within the documentation +- **Other Files**: Appear as download links that users can click to retrieve the file + +### Permission Checks + +When a user attempts to download a file, DataHub verifies that they have permission to view the asset or column where the file was originally uploaded. This ensures that file access respects your existing DataHub permission structure. + +If a user doesn't have permission to view the associated asset, they won't be able to download the file, even if they have a direct link to it. + +## File Metadata and Tracking + +Each uploaded file is represented as a `dataHubFile` entity in DataHub, which stores: + +- **File type**: The MIME type of the uploaded file +- **File size**: Size in bytes +- **Original filename**: The name of the file when it was uploaded +- **Upload context**: The entity (asset or column) where the file was added + +This metadata enables future features like file search, usage tracking, and cleanup of orphaned files. + +## Best Practices + +When using file uploads in your documentation: + +- **Use descriptive filenames**: Original filenames are preserved, making it easier to identify files later +- **Optimize image sizes**: Large images may take longer to load; consider resizing before upload +- **Provide context**: Add text descriptions around images to explain what users are seeing +- **Consider alternatives for large files**: For very large files, consider linking to external storage rather than uploading directly +- **Review permissions**: Ensure your S3 role and bucket permissions are configured correctly before enabling this feature for users + +## Troubleshooting + +### Files Not Uploading + +If files fail to upload, check: + +- Your `DATAHUB_BUCKET_NAME` is set correctly and the bucket exists +- Your `DATAHUB_ROLE_ARN` is valid and points to an existing role +- The trust relationship on your AWS role permits your DataHub GMS to assume it +- The role has necessary S3 permissions (`s3:PutObject`) +- Ensure that your S3 bucket has a CORS policy set to accept uploads from your host URL + +### Files Not Downloading + +If users cannot download files, verify: + +- Users have permission to view the asset where the file was uploaded +- Your AWS role has `s3:GetObject` permissions on the bucket +- The file still exists in S3 (hasn't been manually deleted) + +### Authentication Issues + +If you see AWS authentication errors: + +- Confirm your DataHub instance is deployed on AWS +- Verify the trust relationship configuration on your AWS role +- Check that your GMS service has the necessary AWS credentials or instance profile + +## Next Steps + +Now that you understand how to use file uploads in documentation: + +- **Configure your environment**: Set up the required S3 bucket and AWS role +- **Test with images**: Start by uploading images to asset documentation to see how inline display works +- **Train your team**: Show data owners and stewards how to enrich their documentation with visual aids +- **Monitor usage**: Keep an eye on your S3 bucket to understand storage needs +- **Stay tuned**: Watch for upcoming features like PDF previews and additional file type support + +File uploads are designed to make your DataHub documentation more comprehensive and user-friendly, helping your organization build a richer, more informative data catalog. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/lineage.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/lineage.md new file mode 100644 index 00000000..3bb6b1f1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/lineage.md @@ -0,0 +1,112 @@ +--- +title: About DataHub Lineage +sidebar_label: Lineage +slug: /features/feature-guides/lineage +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/lineage.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# About DataHub Lineage + + + +Data lineage is a **map that shows how data flows through your organization.** It details where your data originates, how it travels, and where it ultimately ends up. +This can happen within a single system (like data moving between Snowflake tables) or across various platforms. + +With data lineage, you can + +- Maintain data integrity +- Simplify and refine complex relationships +- Perform [lineage impact analysis](../../act-on-metadata/impact-analysis.md) +- Propagate metadata (e.g. [documentation](../../automations/docs-propagation.md)) across lineage + +## Viewing Lineage + +You can view lineage under the **Lineage** tab, in an _Explorer_ visualization or the _Impact Analysis_ tool. +You can also survey an asset's impact analysis in a snap on the entity sidebar. + +

+ +

+ +By default, the UI shows the latest version of the lineage. The time picker can be used to filter out edges within the latest version to exclude those that were last updated outside of the time window. Selecting time windows in the patch will not show you historical lineages. It will only filter the view of the latest version of the lineage. + +

+ +

+ +In this example, data flows from between different Snowflake tables orchestrated by dbt into Looker views and explores. + +### Column Level Lineage + +Column-level lineage **tracks changes and movements for each specific data column.** This approach is often contrasted with table-level lineage, which specifies lineage at the table level. Column lineage can be visualized while viewing table-level +lineage by expanding the columns of tables and hovering or clicking on columns with lineage. + +

+ +

+ +If there is column-level lineage to hidden assets or the table-level view is getting too busy, +you can visualize lineage focused on a single column by clicking on the breadcrumb on a column: + +

+ +

+ +

+ +

+ +### Data Pipeline Lineage + +DataHub also supports visualizing your data pipelines' task relationships, alongside data lineage. On a data pipeline's +entity page, go to the **Lineage** tab to open the visualization. At its center will be the data pipeline node, +represented as a box containing all of its composite tasks. You can click and drag to move each task within the box, +as well as click and drag the data pipeline box itself. + +

+ +

+ +Further, you can leverage DataHub's cross-platform lineage to view the upstreams and downstreams of each task. +The number on the expand lineage button represents how many data-dependence upstreams / downstreams the task has. +After expanding, you can keep expanding lineage further like the standard lineage explorer. + +

+ +

+ +Note that you can only expand one task's upstreams and one task's downstreams at a time, to keep the visualization simple. + +

+ +

+ +## Adding Lineage + +### Ingestion Source + +If you're using an ingestion source that supports extraction of Lineage (e.g. **Table Lineage Capability**), then lineage information can be extracted automatically. The list of which sources support automatic lineage extraction can be found +[here](../../generated/lineage/automatic-lineage-extraction.md). +For detailed instructions, refer to the [source documentation](/integrations) for the source you are using. + +### UI + +As of `v0.9.5`, DataHub supports the manual editing of lineage between entities. Data experts are free to add or remove upstream and downstream lineage edges in both the Lineage Visualization screen as well as the Lineage tab on entity pages. Use this feature to supplement automatic lineage extraction or establish important entity relationships in sources that do not support automatic extraction. Editing lineage by hand is supported for Datasets, Charts, Dashboards, and Data Jobs. +Please refer to our [UI Guides on Lineage](./ui-lineage.md) for more information. + +:::caution Recommendation on UI-based lineage + +Lineage added by hand and programmatically may conflict with one another to cause unwanted overwrites. +It is strongly recommend that lineage is edited manually in cases where lineage information is not also extracted in automated fashion, e.g. by running an ingestion source. If you are going to manually edit lineage for an entity in which lineage is +automatically ingested, see if the appropriate ingestion source supports `incremental_lineage` and if so, consider enabling +that configuration flag. Note that if you do set this flag, lineage edges that used to exist will not be removed, and so +you may need to use time-based filtering to accurately determine current lineage. + +::: + +### API + +If you are not using a Lineage-support ingestion source, you can programmatically emit lineage edges between entities via API. +Please refer to [API Guides on Lineage](../../api/tutorials/lineage.md) for more information. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/centralized-management.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/centralized-management.md new file mode 100644 index 00000000..5c301570 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/centralized-management.md @@ -0,0 +1,142 @@ +--- +title: Centralized Management +slug: /features/feature-guides/logical-models/centralized-management +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/logical-models/centralized-management.md +--- +# Centralized Management + + + +:::info +This feature is currently in private beta in DataHub Cloud. Reach out to your DataHub Cloud representative to learn more. +::: + +## How It Works + +Centralized Management allows governance of all physical children at the logical parent level. This means that metadata changes made to the logical model will be propagated to all its physical children. + + + + + + + + + + + +
Metadata at Logical LevelChange at Physical Level
Tags✅ Tags are replicated on physical children.
Glossary Terms✅ Terms are replicated on physical children.
Documentation✅ The description shown on the logical parent is copied to physical children. If the child has its own description, that will be shown instead.
Ownership✅️ Owners are replicated on physical children. If the same user is an owner for multiple ownership types, only one ownership type replicated.
Structured Properties✅ Structured properties are replicated on physical children. If the child has an existing value for the same property, it will be replaced, even if that property is multi-valued.
Domains❌ Domains are not propagated.
Data Products❌ Data products are not propagated.
Data Quality Assertions❌ Assertions are not propagated.
+ +You can hover over propagated attributes to see when and from where this information came. + +

+ + +

+ +## Getting Started + +Centralized Management requires two automations. They currently must be created manually. This can be done via the [GraphQL](https://github.com/datahub-project/datahub/blob/master/docs/api/graphql) mutations below. + +:::note +You can execute these mutations by visiting `/api/graphiql`. +::: + +```graphql +mutation UpsertLogicalModelsPropagationAutomation { + upsertActionPipeline( + urn: "urn:li:dataHubAction:logical-models" + input: { + name: "Logical Models" + category: "System" + type: "datahub_integrations.propagation.propagation_v2.propagation_v2_action.PropagationV2Action" + description: "Propagation metadata from logical parents to their children" + config: { + recipe: """ + { + "name": "logical models", + "action": { + "type": "datahub_integrations.propagation.propagation_v2.propagation_v2_action.PropagationV2Action", + "config": { + "enabled": true, + "propagation_rule": { + "metadata_propagated": { + "tags": {}, + "terms": {}, + "documentation": {}, + "ownership": {}, + "structured_properties": {} + }, + "origin_urn_resolution": { + "lookup_type": "relationship", + "relationship_type": "PhysicalInstanceOf" + }, + "target_urn_resolution": [ + { + "lookup_type": "relationship", + "relationship_type": "PhysicalInstanceOf" + } + ] + } + } + } + } + """ + executorId: "default" + } + } + ) +} +``` + +:::note Logical Model Platform +If you are using a custom platform for your logical models, make sure to update the `query` field in the next mutation accordingly. If you have multiple logical platforms, you can specify multiple as so: `platform:(platformA platformB)`. +::: + +```graphql +mutation UpsertSchemaFieldPropagationAutomation { + upsertActionPipeline( + urn: "urn:li:dataHubAction:schema-field" + input: { + name: "Schema Fields" + category: "System" + type: "datahub_integrations.propagation.propagation_v2.propagation_v2_action.PropagationV2Action" + description: "Propagate tags, terms, and documentation on SchemaMetadata / EditableSchemaMetadata to aspects directly on schema fields" + config: { + recipe: """ + { + "name": "schema fields", + "action": { + "type": "datahub_integrations.propagation.propagation_v2.propagation_v2_action.PropagationV2Action", + "config": { + "enabled": true, + "propagation_rule": { + "metadata_propagated": { + "tags": { + "omit_attribution_is_propagated": true + }, + "terms": { + "omit_attribution_is_propagated": true + }, + "documentation": { + "omit_attribution_is_propagated": true + } + }, + "origin_urn_resolution": { + "lookup_type": "entity", + "entity_type": "dataset", + "query": "platform:logical" + }, + "target_urn_resolution": "schema_field" + } + } + } + } + """ + executorId: "default" + } + } + ) +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/overview.md new file mode 100644 index 00000000..cd33966f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/logical-models/overview.md @@ -0,0 +1,163 @@ +--- +title: Logical Models +slug: /features/feature-guides/logical-models/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/logical-models/overview.md +--- +# Logical Models + +:::note Supported Entity Types +Currently we only support datasets and by extension schema fields for logical models. No other entity types are supported. +::: + +## What is a Logical Model + +A logical model represents the concept and structure of a database table, without being tied to any single physical instantiation in some source system. Like any DataHub dataset entity, a logical model describes its columns, including data types and descriptions, and can be attributed with other metadata like tags, terms, owners, and custom properties. But unlike physical datasets, logical models do not represent a table in a source system that actually exists, in which data is stored and can be queried. + +Logical models are useful for those who have multiple tables that represent the same type or shape of data, or store the same data. This is common for multi-cloud data ecosystems, in which the same table may be replicated across several cloud providers, e.g. Snowflake, Redshift, and BigQuery. It is also useful in cases where multiple replicas exist in a single system, such as gold/silver/bronze layers. Logical models should be linked to each physical representation to expose this relationship in DataHub. DataHub Cloud customers can take advantage further with [Centralized Management](./centralized-management.md), in which every physical child's metadata can be managed at the logical model level. + +## How It Looks + +:::note Feature Flag +The environment variable `LOGICAL_MODELS_ENABLED` must be set to `true` on `datahub-gms` for logical models to be viewed in the UI. +::: + +For example, suppose there exists a `Users` table Snowflake, an `AllUsers` table in BigQuery, and a `UsersAttributes` table Apache Hive. These three tables may have different names and slightly different structures, but logically represent the same data: a table of users, with certain information (columns) for each user. We create a logical table called `Users` and link it to each physical child: + +

+ +

+ +Once the relationships are [created](#creating-logical-models), they will show up in the entity sidebar for both logical parents and their physical children: + +

+ + +

+ +Columns on the logical parent and physical children can be linked as well: + +

+ + +

+ +## Creating Logical Models + +Logical models are created like any DataHub dataset. We recommend using the Python SDK. + +:::note Logical Model Platform +All DataHub datasets require a platform, representing where the dataset exists. If your logical models are stored in a system users are familiar with, we recommend creating a custom platform for that system and providing a custom icon. Otherwise, we recommend using the platform `logical`, which has a special default icon. +::: + +### Create Dataset in "logical" Platform + +```python + from datahub.sdk import DataHubClient, Dataset + client = DataHubClient.from_env() + dataset = Dataset( + platform="logical", + name=logical_model_name, + description=logical_model_description, + schema=[ + # tuples of (field name / field path, data type, description) + ( + "zipcode", + "varchar(50)", + "This is the zipcode of the address. Specified using extended form and limited to addresses in the United States", + ), + ("street", "varchar(100)", "Street corresponding to the address"), + ("date_column", "date", "Date of the last sale date for this property"), + ], + ) + client.entities.upsert(dataset) +``` + +### Create Dataset in Custom Platform + +```python + # Create custom platform with custom logo + from datahub.sdk import DataHubClient + from datahub.emitter.mcp import MetadataChangeProposalWrapper + from datahub.metadata.schema_classes import DataPlatformInfoClass, PlatformTypeClass + from datahub.metadata.urns import DataPlatformUrn + + urn = DataPlatformUrn("").urn() + aspect = DataPlatformInfoClass( + name="", + type=PlatformTypeClass.OTHERS, + datasetNameDelimiter=".", + logoUrl="" + ) + client = DataHubClient.from_env() + client._graph.emit(MetadataChangeProposalWrapper(entityUrn=urn, aspect=aspect)) + + # Create dataset in custom platform + from datahub.sdk import DataHubClient, Dataset + client = DataHubClient.from_env() + dataset = Dataset( + platform="", + ... # See above + ) + client.entities.upsert(dataset) +``` + +## Linking Logical Models + +At its core, the logical -> physical relationship is created by the [`LogicalParent`](../../../generated/metamodel/entities/dataset.md#logicalparent) aspect. To link columns, this aspect must also be created on each child schmea field entity. However, for ease of use, we recommend the OpenAPI endpoint. + +### OpenAPI + +The OpenAPI endpoint creates a logical -> physical relationship for a single logical-physical pair, as well as the column-level relationships between their columns, if specified. + +```shell +curl -X POST 'http://localhost:8080/openapi/v3/logical//relationship/physicalInstanceOf/' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{ + "": "", + "": "", + "": "" + }' +``` + +These relationships can also be removed (as of DataHub Cloud v0.3.15): + +```shell +curl -X DELETE 'http://localhost:8080/openapi/v3/logical//relationship/physicalInstanceOf' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' +``` + +### Python SDK + +The Python SDK can also query the same endpoint: + +```python + from datahub.sdk import DataHubClient + client = DataHubClient.from_env() + url = f"{client._graph.config.server}/openapi/v3/logical/{child_urn}/relationship/physicalInstanceOf/{parent_urn}" + client._graph._post_generic(url, {column.parent_name: column.child_name for column in columns}) +``` + +Or it can create a single relationship by emitting the `LogicalParent` aspect. + +```python + from datahub.sdk import DataHubClient + from datahub.emitter.mcp import MetadataChangeProposalWrapper + from datahub.metadata.schema_classes import EdgeClass, LogicalParentClass + client = DataHubClient.from_env() + + client._graph.emit(MetadataChangeProposalWrapper(entityUrn=child_urn, aspect=LogicalParentClass(parent=EdgeClass(destinationUrn=parent_urn)))) +``` + +The relationship can also be removed: + +```python + from datahub.sdk import DataHubClient + from datahub.emitter.mcp import MetadataChangeProposalWrapper + from datahub.metadata.schema_classes import EdgeClass, LogicalParentClass + client = DataHubClient.from_env() + + client._graph.emit(MetadataChangeProposalWrapper(entityUrn=child_urn, aspect=LogicalParentClass(parent=None))) +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/mcp.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/mcp.md new file mode 100644 index 00000000..40db3a2d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/mcp.md @@ -0,0 +1,362 @@ +--- +title: DataHub MCP Server +sidebar_label: MCP Server +slug: /features/feature-guides/mcp +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/mcp.md +--- +# DataHub MCP Server + +The DataHub MCP Server implements the [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction), giving AI agents direct access to your DataHub metadata. Search for data assets, traverse lineage, inspect schemas, and generate SQL — all through natural language in tools like Cursor, Windsurf, Claude Desktop, and OpenAI. + +Want to learn more about the motivation, architecture, and advanced use cases? Check out our [deep dive blog post](https://datahub.com/blog/datahub-mcp-server-block-ai-agents-use-case/). + +## Deployment Options + +- [Managed MCP Server](#managed-mcp-server-usage) - Available on DataHub Cloud v0.3.12+ +- [Self-Hosted MCP Server](#self-hosted-mcp-server-usage) - Available for DataHub Core + +## Capabilities + +**Search for Data**
+Find the right data by asking questions in plain English. Supports wildcard matching (`revenue_*`), field searches (`tag:PII`), and boolean logic (`(sales OR revenue) AND quarterly`). + +**Dive Deeper**
+Get usage stats, ownership, documentation, tags, glossary terms, and quality signals for any table, column, dashboard, & more — so agents can separate signal from noise. + +**Lineage & Impact Analysis**
+Trace data flow at table and column level, upstream or downstream, across multiple hops. Understand the origins of your data, and plan for upcoming changes. + +**Query Analysis & Authoring**
+Surface real SQL queries that reference a dataset — see join patterns, common filters, and aggregation behavior — then generate new queries grounded in actual usage. + +**Works Where You Work**
+Seamlessly integrates with Cursor, Windsurf, Claude Desktop, OpenAI, and any other MCP-compatible client. + +## Tools + +The DataHub MCP Server provides the following tools: + +`search` + +Search DataHub using structured keyword search (/q syntax) with boolean logic, filters, pagination, and optional sorting by usage metrics. + +`get_lineage` + +Retrieve upstream or downstream lineage for any entity (datasets, columns, dashboards, etc.) with filtering, query-within-lineage, pagination, and hop control. + +`get_dataset_queries` + +Fetch real SQL queries referencing a dataset or column—manual or system-generated—to understand usage patterns, joins, filters, and aggregation behavior. + +`get_entities` + +Fetch detailed metadata for one or more entities by URN; supports batch retrieval for efficient inspection of search results. + +`list_schema_fields` + +List schema fields for a dataset with keyword filtering and pagination, useful when search results truncate fields or when exploring large schemas. + +`get_lineage_paths_between` + +Retrieve the exact lineage paths between two assets or columns, including intermediate transformations and SQL query information. + +### Mutation Tools + +:::info +Mutation tools are available in [mcp-server-datahub](https://github.com/acryldata/mcp-server-datahub) v0.5.0+. They are enabled via the `TOOLS_IS_MUTATION_ENABLED=true` environment variable. +::: + +`add_tags` / `remove_tags` + +Add or remove tags from entities or schema fields (columns). Supports bulk operations on multiple entities. + +`add_terms` / `remove_terms` + +Add or remove glossary terms from entities or schema fields. Useful for applying business definitions and data classification. + +`add_owners` / `remove_owners` + +Add or remove ownership assignments from entities. Supports different ownership types (technical owner, data owner, etc.). + +`set_domains` / `remove_domains` + +Assign or remove domain membership for entities. Each entity can belong to one domain. + +`update_description` + +Update, append to, or remove descriptions for entities or schema fields. Supports markdown formatting. + +`add_structured_properties` / `remove_structured_properties` + +Manage structured properties (typed metadata fields) on entities. Supports string, number, URN, date, and rich text value types. + +### User Tools + +:::info +User tools are available in [mcp-server-datahub](https://github.com/acryldata/mcp-server-datahub) v0.5.0+. They are enabled via the `TOOLS_IS_USER_ENABLED=true` environment variable. +::: + +`get_me` + +Retrieve information about the currently authenticated user, including profile details and group memberships. + +### Document Tools + +:::info +Document tools are available in [mcp-server-datahub](https://github.com/acryldata/mcp-server-datahub) v0.5.0+. Document tools are automatically hidden if no documents exist in the catalog. +::: + +`search_documents` + +Search for documents using keyword search with filters for platforms, domains, tags, glossary terms, and owners. + +`grep_documents` + +Search within document content using regex patterns. Useful for finding specific information across multiple documents. + +`save_document` + +Save standalone documents (insights, decisions, FAQs, notes) to DataHub's knowledge base. Documents are organized under a configurable parent folder. + +## Managed MCP Server Usage + +For DataHub Cloud v0.3.12+, you can connect directly to the hosted MCP server endpoint — no local installation required. + +:::info +The managed MCP server endpoint is only available with DataHub Cloud v0.3.12+. For DataHub Core and older versions of DataHub Cloud, [self-host the MCP server](#self-hosted-mcp-server-usage) instead. +::: + +:::note Streamable HTTP Only +DataHub's managed MCP server uses the [streamable HTTP transport](https://modelcontextprotocol.io/docs/concepts/transports). Some older MCP clients (e.g. chatgpt.com) may only support the deprecated SSE transport — for those, use [mcp-remote](https://github.com/geelen/mcp-remote) to bridge the gap. +::: + +### Prerequisites + +- The URL of your DataHub Cloud instance, e.g. `https://.acryl.io` +- A [personal access token](../../authentication/personal-access-tokens.md) + +### Connecting & Authenticating + +Your managed MCP server URL is: + +``` +https://.acryl.io/integrations/ai/mcp/ +``` + +There are two ways to authenticate: + +1. **Authorization header** — pass your token as a Bearer token in the `Authorization` header: + + ``` + Authorization: Bearer + ``` + +2. **Token in URL** — append your token as a query parameter: + + ``` + https://.acryl.io/integrations/ai/mcp/?token= + ``` + + This is a convenient alternative when your MCP client doesn't support custom headers. + +
+ On-Premises DataHub Cloud + +For on-premises DataHub Cloud, replace `.acryl.io` with your DataHub FQDN, e.g. `https://datahub.example.com/integrations/ai/mcp/?token=`. + +
+ +### Configure + +
+ Claude Desktop + +1. Open your `claude_desktop_config.json` file. You can find it by navigating to Claude Desktop -> Settings -> Developer -> Edit Config. +1. Update the file to include the following content. Be sure to replace `` and `` with your own values. + +```json +{ + "mcpServers": { + "datahub-cloud": { + "command": "npx", + "args": [ + "-y", + "mcp-remote", + "https://.acryl.io/integrations/ai/mcp/?token=" + ] + } + } +} +``` + +
+ +
+ Claude Code + +Claude Code natively supports streamable HTTP, so no proxy or additional dependencies are needed. + +Run the following command, replacing `` and `` with your own values: + +```bash +claude mcp add --transport http datahub-cloud "https://.acryl.io/integrations/ai/mcp/?token=" +``` + +
+ +
+ Cursor + +1. Make sure you're using Cursor v1.1 or newer. +2. Navigate to Cursor -> Settings -> Cursor Settings -> MCP -> add a new MCP server. +3. Enter the following into the file: + +```json +{ + "mcpServers": { + "datahub-cloud": { + "url": "https://.acryl.io/integrations/ai/mcp/?token=" + } + } +} +``` + +4. Once you've saved the file, confirm that the MCP settings page shows a green dot and the DataHub tools listed. + +
+ +
+ Other + +Most AI tools support remote MCP servers. Provide the hosted MCP server URL: + +``` +https://.acryl.io/integrations/ai/mcp/?token= +``` + +Make sure authentication mode is _not_ set to "OAuth" (if applicable). + +For clients that don't yet support remote MCP servers, use `mcp-remote`: + +- Command: `npx` +- Args: `-y mcp-remote https://.acryl.io/integrations/ai/mcp/?token=` + +
+ +## Self-Hosted MCP Server Usage + +Run the [open-source MCP server](https://github.com/acryldata/mcp-server-datahub) locally. This works with any DataHub instance — both DataHub Core and DataHub Cloud. + +### Prerequisites + +1. Install [`uv`](https://github.com/astral-sh/uv): + + ```bash + # macOS and Linux + curl -LsSf https://astral.sh/uv/install.sh | sh + ``` + +2. The URL of your DataHub instance's GMS endpoint, e.g. `http://localhost:8080` or `https://.acryl.io` +3. A [personal access token](../../authentication/personal-access-tokens.md) + +### Connecting & Authenticating + +The self-hosted server authenticates via environment variables: + +- `DATAHUB_GMS_URL` — your DataHub GMS endpoint +- `DATAHUB_GMS_TOKEN` — your personal access token + +These are passed to the `mcp-server-datahub` process at startup (see configuration examples below). + +### Configure + +
+ Claude Desktop + +1. Run `which uvx` to find the full path to the `uvx` command. + +1. Open your `claude_desktop_config.json` file. You can find it by navigating to Claude Desktop -> Settings -> Developer -> Edit Config. + +1. Update the file to include the following content. Be sure to replace the placeholder values. + +```js +{ + "mcpServers": { + "datahub": { + "command": "", // e.g. /Users/hsheth/.local/bin/uvx + "args": ["mcp-server-datahub@latest"], + "env": { + "DATAHUB_GMS_URL": "", + "DATAHUB_GMS_TOKEN": "" + } + } + } +} +``` + +
+ +
+ Claude Code + +Run the following command, replacing the placeholder values: + +```bash +claude mcp add datahub \ + -e DATAHUB_GMS_URL="" \ + -e DATAHUB_GMS_TOKEN="" \ + -- uvx mcp-server-datahub@latest +``` + +
+ +
+ Cursor + +1. Navigate to Cursor -> Settings -> Cursor Settings -> MCP -> add a new MCP server. +2. Enter the following into the file: + +```json +{ + "mcpServers": { + "datahub": { + "command": "uvx", + "args": ["mcp-server-datahub@latest"], + "env": { + "DATAHUB_GMS_URL": "", + "DATAHUB_GMS_TOKEN": "" + } + } + } +} +``` + +3. Once you've saved the file, confirm that the MCP settings page shows a green dot and the DataHub tools listed. + +
+ +
+ Other + +For other AI tools, provide the following configuration: + +- Command: `uvx` +- Args: `mcp-server-datahub@latest` +- Env: + - `DATAHUB_GMS_URL`: `` + - `DATAHUB_GMS_TOKEN`: `` + +
+ +### Troubleshooting + +#### `spawn uvx ENOENT` + +The full stack trace might look like this: + +``` +2025-04-08T19:58:16.593Z [datahub] [error] spawn uvx ENOENT {"stack":"Error: spawn uvx ENOENT\n at ChildProcess._handle.onexit (node:internal/child_process:285:19)\n at onErrorNT (node:internal/child_process:483:16)\n at process.processTicksAndRejections (node:internal/process/task_queues:82:21)"} +``` + +Solution: Replace the `uvx` bit of the command with the output of `which uvx`. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/create-a-property.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/create-a-property.md new file mode 100644 index 00000000..69ef9eee --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/create-a-property.md @@ -0,0 +1,273 @@ +--- +title: Create and Add a Structured Property +sidebar_label: Create and Add a Structured Property +slug: /features/feature-guides/properties/create-a-property +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/properties/create-a-property.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Create and Add a DataHub Structured Property + + + +This guide walks you through creating a Structured Property via the DataHub UI, including: + +1. Defining a new Structured Property +2. Configuring display preferences for a Structured Property +3. Adding a Structured Property to an Asset +4. Adding a Structured Property to a Column + +:::note +To learn more about creating and assigning Structured Properties via the CLI, please see the [Create Structured Properties](/docs/api/tutorials/structured-properties.md) tutorial. +::: + +### Prerequisites + +To create, edit, or remove Structured Properties, you must have the **View Structured Properties** and **Manage Structured Properties** platform privileges. + +To add an existing Structured Property to an Asset, change its value, or remove it from an Asset, you must have the **Edit Properties** metadata privilege. + +## Define a New Structured Property + +From the navigation bar, go to **Govern** > **Structured Properties**. + +Click **+ Create** to start defining your Property. + +

+ View all Structured Properties +

+ +First, provide the following details: + +1. **Name and Description:** Clearly describe the purpose and meaning of the Structured Property so users understand its role and context. +2. **Property Type:** Choose a type that best fits the metadata you want to collect. Available types include **Text**, **Number**, **Date**, **DataHub Entity**, or **Rich Text**. Choosing any of the "List" options allows multiple entries for the Property. +3. **Allowed Values (Optional):** For **Text**, **Number**, and **DataHub Entity** types, define a set of allowed values to ensure consistent input across assets. +4. **Applies To:** Specify which DataHub asset types (e.g., Datasets, Dashboards, Pipelines) the Structured Property can be associated with, ensuring relevance and precision. + +:::caution +Once you you save a Structured Property, you **cannot** edit or remove Allowed Values. However, you can add additional Allowed Values. +::: + +For example, imagine your organization wants to standardize how data assets (e.g., Datasets, Tasks, Pipelines) are categorized during their development cycle. By creating a **Lifecycle Stage** Structured Property, you can set a pre-defined list of allowed statuses, such as **Draft**, **Review**, and **Prod**, ensuring consistency and transparency. + +

+ View all Structured Properties +

+ +## Set Display Preferences for the Structured Property + +When defining a Structured Property, you can customize how it will be visible to DataHub users. By default, Structured Properties are visible in an Asset's **Properties** tab but can be conditionally configured with the following options: + +1. **Hide Property:** + Use this option if the Structured Property contains sensitive metadata that should not be visible to DataHub users via the UI. This ensures that only users with the necessary permissions can view or interact with the property values. + +2. **Customize Visibility:** + Decide where the Structured Property appears across the DataHub UI: + + - **Asset Badge:** Display the property value as a badge on Assets to highlight key metadata. + - **Asset Sidebar:** Show the property in the Asset Sidebar for quick visibility while navigating an Asset. + +3. **Show in Search Filters:** + Enable this option to allow users to filter Assets by the values of this Structured Property. This improves discoverability and facilitates searches for Assets with specific attributes or classifications. + +4. **Show in Columns Table:** + Use this option to display the Structured Property value in the Dataset Schema view’s Columns Table. This is particularly useful for capturing field-level custom metadata and making it accessible alongside schema details. + +For the **Lifecycle Stage** example, imagine you want to allow users to filter by lifecycle status and view it at a glance during data discovery. To achieve this, you would enable **Show in Search Filters**, **Asset Badge**, and **Asset Sidebar**: + +

+ Configure Structured Property Visibility +

+ +## Add a Structured Property to an Asset + +Once a Structured Property has been defined, you can add it to the designated Asset Types. + +From an Asset's **Properties** tab, click the `+` button to see a drop-down list of all available Structured Properties. For example, you can now see **Lifecycle Stage** as an option for the `pet_profiles` Dataset: + +

+ Add a Structured Property to an Asset +

+ +Continuing with the **Lifecycle Stage** example, designate the `pet_profiles` Dataset as being in **Prod**: + +

+ Select a Structured Property Value +

+ +After clicking **Save**, the **Lifecycle Stage** for `pet_profiles` will appear in the following sections of the Asset Page: + +1. Properties Tab +2. Asset Badge +3. Asset Sidebar + +

+ View a Structured Property on an Asset +

+ +:::info +[**DataHub Compliance Forms**](../compliance-forms/overview.md) make it easy to update values for multiple Assets at once! +::: + +### Edit or Remove a Structured Property from an Asset + +After a Structured Property has been added to an Asset, users can modify its value or remove the property entirely using the **More** menu: + +

+ Editing or Removing a Structured Property on an Asset +

+ +### Search for Assets by a Structured Property Value + +For Structured Properties that have **Show in Search Filters** enabled, users can filter search results based on allowed values. + +For example, with the **Lifecycle Stage** property enabled as a filter, users can find it under the **More** dropdown in the Search interface: + +

+ Structured Property as Search Filter +

+ +From here, you can quickly narrow down Search results based on the desired stage, such as **Prod**: + +

+ Filtered Results by Structured Property Value +

+ +Notice how the **Prod** value is displayed prominently on the `pet_profiles` Asset Badge: + +

+ View Structured Property in Asset Badge +

+ +## Add a Structured Property to a Column + +Structured Properties can be applied at the column level, providing deeper context for how individual dataset fields relate to business concepts or terminology. In this example, we’ll create a Structured Property called **Business Label** to help business users understand how dataset columns align with common terminology, acronyms, or key business concepts. + +### Define the Business Label Property + +Follow these steps to define and configure the **Business Label** Structured Property: + +1. **Property Details:** + + - **Name:** Business Label + - **Description:** Provide a description to explain its purpose, such as: + _"A user-friendly name for a dataset column, helping business users understand its meaning."_ + - **Property Type:** Select **Text**, allowing any valid string to be entered. + - **Applies To:** Set this property to apply exclusively to **Columns**. + +2. **Display Preferences:** + - By default, column-level Structured Properties will be enabled for **all columns** on **all datasets** within DataHub, accessible via the Column Sidebar. + - Optionally, enable **Show in Table Columns** to make the **Business Label** visible within the Columns Table on the dataset schema. + +

+ Configuring Column Structured Property +

+ +:::caution +While the column sidebar provides convenient access to assigned properties, adding too many Structured Properties can clutter the view. Limit the number of properties shown in the sidebar to maintain clarity and usability. +::: + +Once configured, the **Business Label** Structured Property will automatically be added to all columns on dataset assets within DataHub. + +For example, after assigning the property, it will appear in two key areas of the `pet_profiles` Asset Page: + +1. **Columns Table:** The **Business Label** property and its populated values will be displayed directly within the Columns Table on the Dataset Schema, enabling users to view field-level metadata easily. +2. **Column Sidebar:** Structured Properties configured for columns, including **Business Label**, will also appear in the column’s sidebar. + +By applying column-level Structured Properties like **Business Label**, you enhance data discoverability and provide business users with valuable insights while keeping the interface user-friendly. + +

+ Column-level Structured Property in Columns Table +

+ +### Update the Business Label from the Column Sidebar + +When selecting a specific column in the UI, the **Business Label** Structured Property will be visible in the column’s sidebar. Users with appropriate permissions can view or update the value directly from this interface. + +

+ Column-level Structured Property in Sidebar +

+ +This setup ensures that column-specific metadata, such as the **Business Label**, is accessible and actionable, helping business users better understand the dataset's structure and its alignment with key business concepts. + +## FAQ and Troubleshooting + +### Why can’t I change a Structured Property’s definition? + +Once a Structured Property has been defined, only certain aspects can be modified: + +**You can change:** + +- Title and description +- Add new allowed values +- Add new supported asset types +- Update display preferences + +**You cannot change:** + +- The type of the Structured Property +- Existing allowed values and their definitions + +### Why can't I configure a Structured Property to appear as an Asset Badge? + +- Only **Text** and **Number** types with allowed values can be configured as Asset Badges. +- Only one Structured Property can be displayed as a Badge for a given Asset. + +### Why can't I filter Search Results by a Structured Property? + +- Verify that the Structured Property has been configured to appear in search filters. +- Ensure the filter is relevant by checking if there are assets associated with the Structured Property's value in your search results. Try different search terms or relax other applied filters. + +### Why can't I add a Structured Property to an Asset? + +- Confirm you have the **Edit Properties** privilege. +- Ensure the Structured Property has already been created and supports the type of Asset you're trying to modify. + +### API Tutorials + +- [Structured Properties API Guide](/docs/api/tutorials/structured-properties.md) + +### Related Features + +- [DataHub Compliance Forms](/docs/features/feature-guides/compliance-forms/overview.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/overview.md new file mode 100644 index 00000000..0934eb25 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/properties/overview.md @@ -0,0 +1,59 @@ +--- +title: Overview +sidebar_label: Overview +slug: /features/feature-guides/properties/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/properties/overview.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# About DataHub Structured Properties + + + +DataHub **Structured Properties** allow you to add custom, validated properties to any Entity type in DataHub. Using Structured Properties, you can enable data discovery and governance based on attributes unique to your organization. + +

+ +

+ +## What are Structured Properties? + +**Structured Properties** are a powerful way to customize your DataHub environment, enabling you to align metadata with your organization’s unique needs. By defining specific property types—such as Date, Integer, DataHub Asset, or Text—you can apply meaningful, context-aware attributes to your Assets. Validation rules, like restricting allowed values or enforcing specific formats, ensure consistency while giving you the flexibility to reflect your business’s terminology, workflows, and priorities. + +Structured Properties can be added to the following Asset Types: + +- Data Assets, such as Datasets, Columns, Tasks, Pipelines, Charts, Dashboards, and more. +- DataHub Entities, such as Domains, Glossary Terms & Groups, and Data Products. + +### Key Features of Structured Properties: + +1. **Typed Fields:** Properties are explicitly typed, including options like Date, Integer, URN, or Text. +2. **Allowed Values:** Enforce standards by restricting values to a specific format or a pre-defined list of acceptable inputs. +3. **Targeted Application:** Structured Properties can be tailored to specific asset types—such as Datasets, Columns, or Dashboards—ensuring they align with your organization’s data management needs and usage context. + +### Display Settings + +Structured Properties offer several configuration options to enhance metadata management: + +- **Hide Property:** For use cases where property values should not be viewable by DataHub users. +- **Show in Search Filters:** Enables users to filter for Assets based on specific property values, improving discoverability. +- **Customize Visibility:** Allows you to control where the Structured Property appears, such as in the Asset Badge, Asset Sidebar, and/or a Dataset Schema view’s Columns Table. + +## Why Use Structured Properties? + +Structured Properties are especially useful for organizations that require: + +- **Customization:** Customize how your end-users find assets within DataHub. +- **Governance and Compliance:** Collect metadata in a way that supports compliance with internal or external standards. + +

+ +

+ +By leveraging these configurations, teams can ensure their metadata adheres to organizational policies and improves the discoverability and usability of Data Assets. + +## Next Steps + +Now that you understand Structured Properties, you’re ready to [Create a Structured Property](create-a-property.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/search-access-controls.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/search-access-controls.md new file mode 100644 index 00000000..2f411581 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/search-access-controls.md @@ -0,0 +1,306 @@ +--- +title: Search Access Controls +slug: /features/feature-guides/search-access-controls +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/search-access-controls.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Search Access Controls + + + +Search Access Controls allow organizations to restrict which entities users can discover through search results. This feature uses the **View Entity** permission to filter search results based on policies, ensuring users only see metadata they are authorized to access. + +## Key Concepts + +### View Entity Permission + +The **View Entity** permission controls whether a user can discover and access an entity. When Search Access Controls are enabled: + +- Users will only see entities in search results that they have been granted access to view +- Users cannot access restricted entities via direct URL +- Access control applies consistently across search, browse, and direct access + +This unified approach ensures that access control is applied consistently regardless of how a user attempts to discover or view an entity. + +### Default Behavior + +Without Search Access Controls enabled: + +- All users can see all entities in search results +- No filtering is applied based on policies + +When Search Access Controls are enabled: + +- Search results are filtered based on the user's applicable policies +- Only entities matching at least one policy with the "View Entity" privilege are returned +- This creates a "default deny" model where explicit permission grants are required + +### Policy-Based Filtering + +Search results are automatically filtered based on: + +- Active policies that grant the "View Entity" privilege + +## How It Works + +When a user performs a search: + +1. DataHub identifies all active policies that grant the "View Entity" privilege to the user +2. For each policy, the resource filters are evaluated (domains, tags) +3. Only entities matching at least one applicable policy are included in the search results +4. The filtering happens at query time, ensuring consistent results across all search interfaces + +The feature is enabled by your DataHub Cloud administrator. Contact your admin to enable Search Access Controls for your organization. + +## Peer Group Recommendations + +When Search Access Controls are enabled, the "Most Popular" recommendations on the home page can also be filtered to prevent information leakage. + +### How Peer Groups Work + +The peer group setting controls how "Most Popular" recommendations are calculated: + +| Setting | Behavior | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Peer Group Enabled** | Recommendations show what users in your same groups have been viewing. This allows you to see popular assets among your peers while preventing visibility into what other teams are accessing. | +| **Peer Group Disabled** | Recommendations are based only on your own activity. You will only see assets you have previously viewed. | + +This prevents scenarios where users could infer the existence of sensitive data by seeing it appear in "Most Popular" recommendations, even if they cannot access it directly. + +## Practical Scenario: Two Groups, Three Domains + +This section walks through a complete example of setting up Search Access Controls for an organization with two teams that need different levels of access. + +### Business Context + +An organization wants to ensure: + +- Engineering can only discover engineering-related data +- Finance can only discover financial data +- Both teams need access to shared company metrics + +### Access Matrix + +| Domain | Engineering Team | Finance Team | +| ---------------- | ---------------- | ------------ | +| Engineering Data | Can View | Cannot View | +| Finance Data | Cannot View | Can View | +| Company Metrics | Can View | Can View | + +### Step 1: Create the Domains + +Navigate to **Settings > Domains** and create the following domains: + +

+ +

+ +1. **Engineering Data** + + - Description: Contains all engineering datasets, dashboards, and pipelines + - Assign all engineering-related assets to this domain + +2. **Finance Data** + + - Description: Contains all financial datasets and reports + - Assign all finance-related assets to this domain + +3. **Company Metrics** + - Description: Contains shared KPIs and dashboards accessible to all teams + - Assign cross-functional assets to this domain + + + + + + + +
+ + + + + +
+ +### Step 2: Create the Groups + +Navigate to **Settings > Users & Groups > Groups** and create: + +1. **Engineering Team** + - Add all engineering users as members +2. **Finance Team** + - Add all finance users as members + + + + + + +
+ + + +
+ +### Step 3: Remove Default Read Access Policies + +To ensure users can only see what they are explicitly granted access to, you must disable or remove the default read access policies: + +1. Navigate to **Settings > Permissions > Policies** +2. Locate the default policies that grant "View Entity" to "All Users" +3. Disable or delete these policies + +

+ +

+ +:::caution +Removing the default read access policies means users will not see any entities in search results until you create explicit access policies. Plan your access policies before making this change. +::: + +### Step 4: Create Engineering Access Policy + +Navigate to **Settings > Permissions > Policies** and click **Create Policy**: + +1. **Policy Name**: Engineering Team - View Access +2. **Policy Type**: Metadata Policy +3. **Privileges**: Select **View Entity** +4. **Actors**: + - Select the **Engineering Team** group +5. **Resources**: + - From the Domain list, select **Engineering Data** and **Company Metrics** +6. Click **Save** + +### Step 5: Create Finance Access Policy + +Create another policy with: + +1. **Policy Name**: Finance Team - View Access +2. **Policy Type**: Metadata Policy +3. **Privileges**: Select **View Entity** +4. **Actors**: + - Select the **Finance Team** group +5. **Resources**: + - From the Domain list, select **Finance Data** and **Company Metrics** +6. Click **Save** + + + + + + +
+ + + +
+ +After creating both policies, you can view all configured view access policies: + +

+ +

+ +### Step 6: Verify the Configuration + +Now let's verify that the access controls are working correctly by logging in as users from each group. + +**Alice (Engineering Team)** + +When Alice logs in and searches, she can only discover entities in the Engineering Data and Company Metrics domains: + + + + + + +
+ + + +
+ +**David (Finance Team)** + +When David logs in and searches, he can only discover entities in the Finance Data and Company Metrics domains: + + + + + + +
+ + + +
+ +## Important Considerations + +### Domain Assignment + +- **All entities should be assigned to domains** for this approach to work effectively +- Entities without a domain assignment will not be matched by domain-based policies +- Consider creating a catch-all policy or default domain for unassigned entities + +### Platform Administrators + +- Users with platform administrator privileges bypass Search Access Controls +- Admin users will see all entities regardless of policies +- Use admin accounts only when necessary + +### Roles + +- All roles (Admin, Editor, Reader) override view-based access policies +- Users assigned any role will be able to see all entities regardless of domain-based View Entity policies +- When configuring Search Access Controls, ensure users are not assigned any role if you want to restrict their access + +### Consistent Access Control + +- The View Entity permission applies to both search results and direct URL access +- Users without permission cannot discover entities in search or access them via direct link +- This ensures consistent access control regardless of how users attempt to view entities + +### Resource Filter Types + +Policies can filter resources by: + +- **Domain**: Most common for organizational boundaries +- **Tag**: Useful for classification-based access (e.g., PII, Confidential) or granting access to specific entities + +## FAQ + +**What happens if an entity has no domain assigned?** + +Entities without a domain will not match domain-based policies. These entities will only be visible to users with policies that: + +- Grant access to all resources (no resource filter) +- Use other filter types (tags) that match the entity + +**How do I grant access to specific entities rather than domains?** + +Use tags to identify specific entities that should be accessible. Create a tag (e.g., "Finance Approved") and apply it to the entities you want to grant access to. Then create a policy with a tag-based resource filter. This approach is more maintainable than URN-based policies since you can easily add or remove entities by updating tags. + +**Can I use tags instead of domains for access control?** + +Yes. Instead of domain filters, select "Tag" as the resource filter type. This is useful when your access boundaries align with data classification rather than organizational structure. + +**How do I troubleshoot when a user cannot see expected results?** + +1. Verify the user is a member of the correct group +2. Check that the policy is active (not disabled) +3. Confirm the entity is assigned to the correct domain/has the correct tags +4. Verify the policy includes the "View Entity" privilege +5. Check that no conflicting deny policies exist +6. Remember that platform admins see all entities regardless of policies + +**Do Search Access Controls affect the GraphQL API?** + +Yes. The same filtering applies to programmatic access via the GraphQL API. Users will only receive entities they have permission to view. + +**Can I create a policy that denies access instead of granting it?** + +DataHub policies are grant-based. To deny access, you must remove the grant. Note that you also need to disable or remove the default read access policies that grant "View Entity" to all users (see Step 3 above). Once the default policies are removed and Search Access Controls are enabled, users have no access until explicitly granted. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/service-accounts.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/service-accounts.md new file mode 100644 index 00000000..e8a282a1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/service-accounts.md @@ -0,0 +1,359 @@ +--- +title: Service Accounts +slug: /features/feature-guides/service-accounts +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/service-accounts.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Service Accounts + + + +Available starting in DataHub Cloud v0.3.17, DataHub Core v1.4.0. + +## Introduction + +Service Accounts provide a secure way to enable programmatic access to DataHub APIs without using personal user credentials. They are designed for automated workflows, CI/CD pipelines, data ingestion processes, and any other use case where a non-human identity needs to interact with DataHub. + +Key benefits of using service accounts: + +1. **Separation of Concerns** - Decouple automation credentials from personal user accounts +2. **Auditable Access** - Track API usage by specific automation workflows +3. **Granular Permissions** - Assign specific roles and policies to each service account +4. **Token Management** - Generate, rotate, and revoke tokens independently for each service account +5. **Security** - Revoke access instantly without affecting human users + +## Prerequisites and Permissions + +To manage service accounts, a user must have the **Manage Service Accounts** platform privilege. This privilege allows users to: + +- Create new service accounts +- View and list existing service accounts +- Delete service accounts +- Generate access tokens for service accounts + +By default, users with the **Admin** role have this privilege. You can grant this privilege to other users or groups through [Policies](../../authorization/policies.md). + +## Using Service Accounts + +### Accessing Service Accounts + +Service accounts can be managed from the **Settings** page in DataHub: + +1. Navigate to **Settings** (gear icon in the top navigation) +2. Click **Users & Groups** in the left sidebar +3. Select the **Service Accounts** tab + + +

+ +

+ +### Creating a Service Account + +To create a new service account: + +1. Click the **Create Service Account** button +2. Enter a **Name** for the service account (e.g., "Ingestion Pipeline", "CI/CD Bot") +3. Optionally, add a **Description** to explain the service account's purpose +4. Click **Create** + + +

+ +

+ +The service account will be assigned a unique identifier (URN) automatically. This URN follows the format: + +``` +urn:li:corpuser:service_ +``` + +### Generating Access Tokens + +After creating a service account, you need to generate an access token for it to authenticate with DataHub APIs: + +1. Navigate to **Settings** > **Access Tokens** +2. Click **Generate new token** +3. Select **Service Account** as the token type +4. Choose the service account from the dropdown +5. Provide a **Name** for the token (e.g., "production-ingestion-token") +6. Select a **Duration** for the token validity +7. Click **Generate** + + +

+ +

+ +:::caution Important +Copy and store the generated token securely. It will only be displayed once and cannot be retrieved later. +::: + +### Assigning Roles to Service Accounts + +Service accounts can be assigned DataHub roles to control their permissions. To assign a role: + +1. Navigate to **Settings** > **Users & Groups** > **Service Accounts** +2. Find the service account in the list +3. Use the **Role** dropdown to select a role (e.g., Admin, Editor, Reader) +4. Confirm the role assignment + + +

+ +

+ +Alternatively, you can add service accounts to [Policies](../../authorization/policies.md) for more granular permission control. + +### Deleting a Service Account + +To delete a service account: + +1. Navigate to **Settings** > **Users & Groups** > **Service Accounts** +2. Find the service account you want to delete +3. Click the 3-dot more menu on the right side +4. Click the **Delete** button (trash icon) + +:::warning +Deleting a service account will immediately invalidate all access tokens associated with it. Any automated workflows using those tokens will stop working. +::: + +### Revoking Access Tokens + +To revoke a specific token without deleting the entire service account: + +1. Navigate to **Settings** > **Access Tokens** +2. Find the token you want to revoke +3. Click **Revoke** +4. Confirm the revocation + +## Using Service Account Tokens + +Once you have generated an access token for a service account, you can use it to authenticate with DataHub APIs. + +### With the DataHub CLI + +```bash +# Set the token as an environment variable +export DATAHUB_GMS_TOKEN="" + +# Or pass it directly +datahub get --urn "urn:li:dataset:..." +``` + +### With the Python SDK + +```python +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Create an emitter with service account token +emitter = DatahubRestEmitter( + gms_server="http://localhost:8080", + token="" +) + +# Use the emitter for API calls +emitter.emit(...) +``` + +### With GraphQL API + +```bash +curl -X POST 'https://your-datahub-instance/api/graphql' \ + -H 'Authorization: Bearer ' \ + -H 'Content-Type: application/json' \ + -d '{"query": "{ me { corpUser { urn } } }"}' +``` + +### With REST API + +```bash +curl 'https://your-datahub-instance/openapi/v3/entity/dataset/...' \ + -H 'Authorization: Bearer ' +``` + +## GraphQL API Reference + +Service accounts can also be managed programmatically using the GraphQL API. + +### Create a Service Account + +```graphql +mutation createServiceAccount($input: CreateServiceAccountInput!) { + createServiceAccount(input: $input) { + urn + type + name + displayName + description + createdBy + createdAt + } +} +``` + +Variables: + +```json +{ + "input": { + "displayName": "My Ingestion Pipeline", + "description": "Service account for automated data ingestion" + } +} +``` + +### List Service Accounts + +```graphql +query listServiceAccounts($input: ListServiceAccountsInput!) { + listServiceAccounts(input: $input) { + start + count + total + serviceAccounts { + urn + name + displayName + description + createdBy + createdAt + updatedAt + } + } +} +``` + +Variables: + +```json +{ + "input": { + "start": 0, + "count": 20, + "query": "" + } +} +``` + +### Get a Service Account + +```graphql +query getServiceAccount($urn: String!) { + getServiceAccount(urn: $urn) { + urn + name + displayName + description + createdBy + createdAt + updatedAt + } +} +``` + +### Delete a Service Account + +```graphql +mutation deleteServiceAccount($urn: String!) { + deleteServiceAccount(urn: $urn) +} +``` + +### Create Access Token for Service Account + +```graphql +mutation createAccessToken($input: CreateAccessTokenInput!) { + createAccessToken(input: $input) { + accessToken + metadata { + id + actorUrn + ownerUrn + name + description + } + } +} +``` + +Variables: + +```json +{ + "input": { + "type": "SERVICE_ACCOUNT", + "actorUrn": "urn:li:corpuser:service_", + "name": "my-token-name", + "duration": "ONE_YEAR" + } +} +``` + +Valid duration options: `ONE_HOUR`, `ONE_DAY`, `ONE_MONTH`, `THREE_MONTHS`, `SIX_MONTHS`, `ONE_YEAR`, `NO_EXPIRY` + +## Best Practices + +### Naming Conventions + +Use descriptive names that indicate the purpose of each service account: + +- `ingestion-snowflake-prod` - For Snowflake production ingestion +- `cicd-github-actions` - For GitHub Actions CI/CD pipelines +- `airflow-metadata-sync` - For Airflow metadata synchronization + +### Token Rotation + +Regularly rotate service account tokens to maintain security: + +1. Generate a new token for the service account +2. Update your automation to use the new token +3. Revoke the old token + +### Least Privilege Access + +Assign the minimum permissions required for each service account: + +- For ingestion pipelines: Assign a custom policy that only allows creating/updating specific entity types +- For read-only integrations: Use a Reader role or custom read-only policy +- For admin automation: Use the Admin role only when absolutely necessary + +## FAQ and Troubleshooting + +### Why can't I see the Service Accounts tab? + +The Service Accounts tab is only visible to users with the **Manage Service Accounts** privilege. Contact your DataHub administrator to request access. + +### My service account token stopped working + +Check the following: + +1. **Token expiration**: The token may have expired. Generate a new token. +2. **Service account deleted**: The service account may have been deleted. Create a new one. +3. **Token revoked**: Someone may have revoked the token. Generate a new one. +4. **Permission changes**: The service account's permissions may have been modified. + +### Can I see which tokens exist for a service account? + +Yes, navigate to **Settings** > **Access Tokens** and filter by the service account name to see all tokens associated with it. Note that YOU must filter for the owner of the service account to see it. Whoever created the service account will be the token owner. + +### What happens to a service account's tokens when I delete it? + +All tokens associated with a service account are invalidated when the service account is deleted. + +### Can I use service accounts in policies? + +Yes, service accounts can be added to policies just like regular users. Use the service account's URN (e.g., `urn:li:corpuser:service_`) when configuring policy actors. + +### What's the difference between a personal access token and a service account token? + +| Feature | Personal Access Token | Service Account Token | +| ----------------- | ----------------------------- | ---------------------------------------- | +| Associated with | A human user | A service account | +| Permissions | Inherits user's permissions | Based on service account's role/policies | +| Revocation impact | Only affects the token holder | Only affects the specific automation | +| Use case | Individual user API access | Automated workflows and integrations | diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ui-lineage.md b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ui-lineage.md new file mode 100644 index 00000000..607d5b2b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/features/feature-guides/ui-lineage.md @@ -0,0 +1,67 @@ +--- +title: Managing Data Lineage via UI +slug: /features/feature-guides/ui-lineage +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/features/feature-guides/ui-lineage.md +--- +# Managing Data Lineage via UI + +## Viewing Data Lineage + +The UI shows the latest version of the data lineage. The time picker can be used to filter out edges within the latest version to exclude those that were last updated outside of the time window. Selecting time windows in the patch will not show you historical data lineages. It will only filter the view of the latest version of the lineage. + +## Editing from Lineage Graph View + +Ensure that you have `Edit lineage` privilege on both upstream and downstream entities before you try to add upstream or downstream lineage. + +The first place that you can edit data lineage for entities is from the Lineage Visualization screen. Click on the "Lineage" button on the top right of an entity's profile to get to this view. + +

+ +

+ +Once you find the entity that you want to edit the lineage of, click on the three-dot menu dropdown to select whether you want to edit lineage in the upstream direction or the downstream direction. + +

+ +

+ +If you want to edit upstream lineage for entities downstream of the center node or downstream lineage for entities upstream of the center node, you can simply re-center to focus on the node you want to edit. Once focused on the desired node, you can edit lineage in either direction. + +

+ +

+ +### Adding Lineage Edges + +Once you click "Edit Upstream" or "Edit Downstream," a modal will open that allows you to manage data lineage for the selected entity in the chosen direction. In order to add a lineage edge to a new entity, search for it by name in the provided search bar and select it. Once you're satisfied with everything you've added, click "Save Changes." If you change your mind, you can always cancel or exit without saving the changes you've made. + +

+ +

+ +### Removing Lineage Edges + +You can remove lineage edges from the same modal used to add lineage edges. Find the edge(s) that you want to remove, and click the "X" on the right side of it. And just like adding, you need to click "Save Changes" to save and if you exit without saving, your changes won't be applied. + +

+ +

+ +### Reviewing Changes + +Any time lineage is edited manually, we keep track of who made the change and when they made it. You can see this information in the modal where you add and remove edges. If an edge was added manually, a user avatar will be in line with the edge that was added. You can hover over this avatar in order to see who added it and when. + +

+ +

+ +## Editing from Lineage Tab + +The other place that you can edit lineage for entities is from the Lineage Tab on an entity's profile. Click on the "Lineage" tab in an entity's profile and then find the "Edit" dropdown that allows you to edit upstream or downstream lineage for the given entity. + +

+ +

+ +Using the modal from this view will work the same as described above for editing from the Lineage Visualization screen. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/abs.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/abs.md new file mode 100644 index 00000000..0377a5a7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/abs.md @@ -0,0 +1,1059 @@ +--- +sidebar_position: 1 +title: ABS Data Lake +slug: /generated/ingestion/sources/abs +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# ABS Data Lake + +## Overview + +Azure Blob Storage is a storage and lakehouse platform. Learn more in the [official Azure Blob Storage documentation](https://learn.microsoft.com/azure/storage/blobs/storage-blobs-introduction). + +The DataHub integration for Azure Blob Storage covers file/lakehouse metadata entities such as datasets, paths, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------- | ----------------------------------------------------------------------------------------- | ---------------- | +| `"abs"` | [Data Platform](/docs/generated/metamodel/entities/dataplatform/) | | +| abs blob / Folder containing abs blobs | [Dataset](/docs/generated/metamodel/entities/dataset/) | | +| abs container | [Container](/docs/generated/metamodel/entities/container/) | Subtype `Folder` | + + +## Module `abs` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Extract ABS containers and folders. Supported for types - Folder, ABS container. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Extract Tags | ✅ | Can extract ABS object/container tags if enabled. | + +### Overview + +The `abs` module ingests metadata from Abs into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This connector supports both local files and those stored on Azure Blob Storage (which must be identified using the +prefix `http(s)://.blob.core.windows.net/` or `azure://`). + +#### Supported file types + +Supported file types are as follows: + +- CSV (`*.csv`) +- TSV (`*.tsv`) +- JSONL (`*.jsonl`) +- JSON (`*.json`) +- Parquet (`*.parquet`) +- Apache Avro (`*.avro`) + +Schemas for Parquet and Avro files are extracted as provided. + +Schemas for schemaless formats (CSV, TSV, JSONL, JSON) are inferred. For CSV, TSV and JSONL files, we consider the first +100 rows by default, which can be controlled via the `max_rows` recipe parameter (see [below](#config-details)) +JSON file schemas are inferred on the basis of the entire file (given the difficulty in extracting only the first few +objects of the file), which may impact performance. +We are working on using iterator-based JSON parsers to avoid reading in the entire JSON object. + +Profiling is not available in the current release. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[abs]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: abs + config: + path_specs: + - include: "https://storageaccountname.blob.core.windows.net/covid19-lake/covid_knowledge_graph/csv/nodes/*.*" + + azure_config: + account_name: "*****" + sas_token: "*****" + container_name: "covid_knowledge_graph" + env: "PROD" + +# sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
path_specs ✅
array
| List of PathSpec. See [below](#path-spec) the details about PathSpec | +|
path_specs.PathSpec
PathSpec
| | +|
path_specs.PathSpec.include ❓
string
| Path to table. Name variable `{table}` is used to mark the folder with dataset. In absence of `{table}`, file level dataset will be created. Check below examples for more details. | +|
path_specs.PathSpec.allow_double_stars
boolean
| Allow double stars in the include path. This can affect performance significantly if enabled
Default: False
| +|
path_specs.PathSpec.autodetect_partitions
boolean
| Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`
Default: True
| +|
path_specs.PathSpec.default_extension
One of string, null
| For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.
Default: None
| +|
path_specs.PathSpec.enable_compression
boolean
| Enable or disable processing compressed files. Currently .gz and .bz files are supported.
Default: True
| +|
path_specs.PathSpec.include_hidden_folders
boolean
| Include hidden folders in the traversal (folders starting with . or _
Default: False
| +|
path_specs.PathSpec.sample_files
boolean
| Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled
Default: True
| +|
path_specs.PathSpec.table_name
One of string, null
| Display name of the dataset.Combination of named variables from include path and strings
Default: None
| +|
path_specs.PathSpec.traversal_method
Enum
| One of: "ALL", "MIN_MAX", "MAX" | +|
path_specs.PathSpec.exclude
One of array, null
| list of paths in glob pattern which will be excluded while scanning for the datasets
Default: []
| +|
path_specs.PathSpec.exclude.string
string
| | +|
path_specs.PathSpec.file_types
array
| Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.
Default: ['csv', 'tsv', 'json', 'parquet', 'avro']
| +|
path_specs.PathSpec.file_types.string
string
| | +|
path_specs.PathSpec.tables_filter_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
path_specs.PathSpec.tables_filter_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
add_partition_columns_to_schema
boolean
| Whether to add partition fields to the schema.
Default: False
| +|
max_rows
integer
| Maximum number of rows to use when inferring schemas for TSV and CSV files.
Default: 100
| +|
number_of_files_to_sample
integer
| Number of files to list to sample for schema inference. This will be ignored if sample_files is set to False in the pathspec.
Default: 100
| +|
platform
string
| The platform that this source connects to (either 'abs' or 'file'). If not specified, the platform will be inferred from the path_specs.
Default:
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
spark_config
object
| Spark configuration properties to set on the SparkSession. Put config property names into quotes. For example: '"spark.executor.memory": "2g"'
Default: {}
| +|
spark_driver_memory
string
| Max amount of memory to grant Spark.
Default: 4g
| +|
use_abs_blob_properties
One of boolean, null
| Whether to create tags in datahub from the abs blob properties
Default: None
| +|
use_abs_blob_tags
One of boolean, null
| Whether to create tags in datahub from the abs blob tags
Default: None
| +|
use_abs_container_properties
One of boolean, null
| Whether to create tags in datahub from the abs container properties
Default: None
| +|
verify_ssl
One of boolean, string
| Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use.
Default: True
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
azure_config
One of AzureConnectionConfig, null
| Azure configuration
Default: None
| +|
azure_config.account_name ❓
string
| Name of the Azure storage account. See [Microsoft official documentation on how to create a storage account.](https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account) | +|
azure_config.container_name ❓
string
| Azure storage account container name. | +|
azure_config.account_key
One of string(password), null
| Azure storage account access key that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**
Default: None
| +|
azure_config.base_path
string
| Base folder in hierarchical namespaces to start from.
Default: /
| +|
azure_config.client_id
One of string, null
| Azure client (Application) ID required when a `client_secret` is used as a credential.
Default: None
| +|
azure_config.client_secret
One of string(password), null
| Azure client secret that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**
Default: None
| +|
azure_config.sas_token
One of string(password), null
| Azure storage account Shared Access Signature (SAS) token that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**
Default: None
| +|
azure_config.tenant_id
One of string, null
| Azure tenant (Directory) ID required when a `client_secret` is used as a credential.
Default: None
| +|
profile_patterns
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_patterns.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_patterns.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
profile_patterns.allow.string
string
| | +|
profile_patterns.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
profile_patterns.deny.string
string
| | +|
profiling
DataLakeProfilerConfig
| | +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: True
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: True
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: True
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only or include column-level profiling as well.
Default: False
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AzureConnectionConfig": { + "additionalProperties": false, + "description": "Common Azure credentials config.\n\nhttps://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-directory-file-acl-python", + "properties": { + "base_path": { + "default": "/", + "description": "Base folder in hierarchical namespaces to start from.", + "title": "Base Path", + "type": "string" + }, + "container_name": { + "description": "Azure storage account container name.", + "title": "Container Name", + "type": "string" + }, + "account_name": { + "description": "Name of the Azure storage account. See [Microsoft official documentation on how to create a storage account.](https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account)", + "title": "Account Name", + "type": "string" + }, + "account_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure storage account access key that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", + "title": "Account Key" + }, + "sas_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure storage account Shared Access Signature (SAS) token that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", + "title": "Sas Token" + }, + "client_secret": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure client secret that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", + "title": "Client Secret" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure client (Application) ID required when a `client_secret` is used as a credential.", + "title": "Client Id" + }, + "tenant_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure tenant (Directory) ID required when a `client_secret` is used as a credential.", + "title": "Tenant Id" + } + }, + "required": [ + "container_name", + "account_name" + ], + "title": "AzureConnectionConfig", + "type": "object" + }, + "DataLakeProfilerConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": true, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": true, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": true, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + } + }, + "title": "DataLakeProfilerConfig", + "type": "object" + }, + "FolderTraversalMethod": { + "enum": [ + "ALL", + "MIN_MAX", + "MAX" + ], + "title": "FolderTraversalMethod", + "type": "string" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "PathSpec": { + "additionalProperties": false, + "properties": { + "include": { + "description": "Path to table. Name variable `{table}` is used to mark the folder with dataset. In absence of `{table}`, file level dataset will be created. Check below examples for more details.", + "title": "Include", + "type": "string" + }, + "exclude": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": [], + "description": "list of paths in glob pattern which will be excluded while scanning for the datasets", + "title": "Exclude" + }, + "file_types": { + "default": [ + "csv", + "tsv", + "json", + "parquet", + "avro" + ], + "description": "Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.", + "items": { + "type": "string" + }, + "title": "File Types", + "type": "array" + }, + "default_extension": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.", + "title": "Default Extension" + }, + "table_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Display name of the dataset.Combination of named variables from include path and strings", + "title": "Table Name" + }, + "enable_compression": { + "default": true, + "description": "Enable or disable processing compressed files. Currently .gz and .bz files are supported.", + "title": "Enable Compression", + "type": "boolean" + }, + "sample_files": { + "default": true, + "description": "Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled", + "title": "Sample Files", + "type": "boolean" + }, + "allow_double_stars": { + "default": false, + "description": "Allow double stars in the include path. This can affect performance significantly if enabled", + "title": "Allow Double Stars", + "type": "boolean" + }, + "autodetect_partitions": { + "default": true, + "description": "Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`", + "title": "Autodetect Partitions", + "type": "boolean" + }, + "traversal_method": { + "$ref": "#/$defs/FolderTraversalMethod", + "default": "MAX", + "description": "Method to traverse the folder. ALL: Traverse all the folders, MIN_MAX: Traverse the folders by finding min and max value, MAX: Traverse the folder with max value" + }, + "include_hidden_folders": { + "default": false, + "description": "Include hidden folders in the traversal (folders starting with . or _", + "title": "Include Hidden Folders", + "type": "boolean" + }, + "tables_filter_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "The tables_filter_pattern configuration field uses regular expressions to filter the tables part of the Pathspec for ingestion, allowing fine-grained control over which tables are included or excluded based on specified patterns. The default setting allows all tables." + } + }, + "required": [ + "include" + ], + "title": "PathSpec", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "path_specs": { + "description": "List of PathSpec. See [below](#path-spec) the details about PathSpec", + "items": { + "$ref": "#/$defs/PathSpec" + }, + "title": "Path Specs", + "type": "array" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "platform": { + "default": "", + "description": "The platform that this source connects to (either 'abs' or 'file'). If not specified, the platform will be inferred from the path_specs.", + "title": "Platform", + "type": "string" + }, + "azure_config": { + "anyOf": [ + { + "$ref": "#/$defs/AzureConnectionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure configuration" + }, + "use_abs_container_properties": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether to create tags in datahub from the abs container properties", + "title": "Use Abs Container Properties" + }, + "use_abs_blob_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether to create tags in datahub from the abs blob tags", + "title": "Use Abs Blob Tags" + }, + "use_abs_blob_properties": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Whether to create tags in datahub from the abs blob properties", + "title": "Use Abs Blob Properties" + }, + "profile_patterns": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for tables to profile " + }, + "profiling": { + "$ref": "#/$defs/DataLakeProfilerConfig", + "default": { + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "profile_table_level_only": false, + "max_number_of_fields_to_profile": null, + "include_field_null_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": true, + "include_field_distinct_value_frequencies": true, + "include_field_histogram": true, + "include_field_sample_values": true + }, + "description": "Data profiling configuration" + }, + "spark_driver_memory": { + "default": "4g", + "description": "Max amount of memory to grant Spark.", + "title": "Spark Driver Memory", + "type": "string" + }, + "spark_config": { + "additionalProperties": true, + "default": {}, + "description": "Spark configuration properties to set on the SparkSession. Put config property names into quotes. For example: '\"spark.executor.memory\": \"2g\"'", + "title": "Spark Config", + "type": "object" + }, + "max_rows": { + "default": 100, + "description": "Maximum number of rows to use when inferring schemas for TSV and CSV files.", + "title": "Max Rows", + "type": "integer" + }, + "add_partition_columns_to_schema": { + "default": false, + "description": "Whether to add partition fields to the schema.", + "title": "Add Partition Columns To Schema", + "type": "boolean" + }, + "verify_ssl": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": true, + "description": "Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use.", + "title": "Verify Ssl" + }, + "number_of_files_to_sample": { + "default": 100, + "description": "Number of files to list to sample for schema inference. This will be ignored if sample_files is set to False in the pathspec.", + "title": "Number Of Files To Sample", + "type": "integer" + } + }, + "required": [ + "path_specs" + ], + "title": "DataLakeSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +This connector ingests Azure Blob Storage (abbreviated to abs) datasets into DataHub. It allows mapping an individual +file or a folder of files to a dataset in DataHub. +To specify the group of files that form a dataset, use `path_specs` configuration in ingestion recipe. Refer +section [Path Specs](/docs/generated/ingestion/sources/s3/#path-specs) for more details. + +#### Path Specs + +Path Specs (`path_specs`) is a list of Path Spec (`path_spec`) objects, where each individual `path_spec` represents one or more datasets. The include path (`path_spec.include`) represents a formatted path to the dataset. This path must end with `*.*` or `*.[ext]` to represent the leaf level. If `*.[ext]` is provided, then only files with the specified extension type will be scanned. "`.[ext]`" can be any of the [supported file types](#supported-file-types). Refer to [example 1](#example-1---individual-file-as-dataset) below for more details. + +All folder levels need to be specified in the include path. You can use `/*/` to represent a folder level and avoid specifying the exact folder name. To map a folder as a dataset, use the `{table}` placeholder to represent the folder level for which the dataset is to be created. For a partitioned dataset, you can use the placeholder `{partition_key[i]}` to represent the name of the `i`th partition and `{partition[i]}` to represent the value of the `i`th partition. During ingestion, `i` will be used to match the partition_key to the partition. Refer to [examples 2 and 3](#example-2---folder-of-files-as-dataset-without-partitions) below for more details. + +Exclude paths (`path_spec.exclude`) can be used to ignore paths that are not relevant to the current `path_spec`. This path cannot have named variables (`{}`). The exclude path can have `**` to represent multiple folder levels. Refer to [example 4](#example-4---folder-of-files-as-dataset-with-partitions-and-exclude-filter) below for more details. + +Refer to [example 5](#example-5---advanced---either-individual-file-or-folder-of-files-as-dataset) if your container has a more complex dataset representation. + +**Additional points to note** + +- Folder names should not contain {, }, \*, / in their names. +- Named variable {folder} is reserved for internal working. please do not use in named variables. + +#### Path Specs - Examples + +##### Example 1 - Individual file as Dataset + +Container structure: + +``` +test-container +├── employees.csv +├── departments.json +└── food_items.csv +``` + +Path specs config to ingest `employees.csv` and `food_items.csv` as datasets: + +``` +path_specs: + - include: https://storageaccountname.blob.core.windows.net/test-container/*.csv + +``` + +This will automatically ignore `departments.json` file. To include it, use `*.*` instead of `*.csv`. + +##### Example 2 - Folder of files as Dataset (without Partitions) + +Container structure: + +``` +test-container +└── offers + ├── 1.avro + └── 2.avro + +``` + +Path specs config to ingest folder `offers` as dataset: + +``` +path_specs: + - include: https://storageaccountname.blob.core.windows.net/test-container/{table}/*.avro +``` + +`{table}` represents folder for which dataset will be created. + +##### Example 3 - Folder of files as Dataset (with Partitions) + +Container structure: + +``` +test-container +├── orders +│ └── year=2022 +│ └── month=2 +│ ├── 1.parquet +│ └── 2.parquet +└── returns + └── year=2021 + └── month=2 + └── 1.parquet + +``` + +Path specs config to ingest folders `orders` and `returns` as datasets: + +``` +path_specs: + - include: https://storageaccountname.blob.core.windows.net/test-container/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet +``` + +One can also use `include: https://storageaccountname.blob.core.windows.net/test-container/{table}/*/*/*.parquet` here however above format is preferred as it allows declaring partitions explicitly. + +##### Example 4 - Folder of files as Dataset (with Partitions), and Exclude Filter + +Container structure: + +``` +test-container +├── orders +│ └── year=2022 +│ └── month=2 +│ ├── 1.parquet +│ └── 2.parquet +└── tmp_orders + └── year=2021 + └── month=2 + └── 1.parquet + + +``` + +Path specs config to ingest folder `orders` as dataset but not folder `tmp_orders`: + +``` +path_specs: + - include: https://storageaccountname.blob.core.windows.net/test-container/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet + exclude: + - **/tmp_orders/** +``` + +##### Example 5 - Advanced - Either Individual file OR Folder of files as Dataset + +Container structure: + +``` +test-container +├── customers +│ ├── part1.json +│ ├── part2.json +│ ├── part3.json +│ └── part4.json +├── employees.csv +├── food_items.csv +├── tmp_10101000.csv +└── orders + └── year=2022 + └── month=2 + ├── 1.parquet + ├── 2.parquet + └── 3.parquet + +``` + +Path specs config: + +``` +path_specs: + - include: https://storageaccountname.blob.core.windows.net/test-container/*.csv + exclude: + - **/tmp_10101000.csv + - include: https://storageaccountname.blob.core.windows.net/test-container/{table}/*.json + - include: https://storageaccountname.blob.core.windows.net/test-container/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet +``` + +Above config has 3 path_specs and will ingest following datasets + +- `employees.csv` - Single File as Dataset +- `food_items.csv` - Single File as Dataset +- `customers` - Folder as Dataset +- `orders` - Folder as Dataset + and will ignore file `tmp_10101000.csv` + +**Valid path_specs.include** + +```python +https://storageaccountname.blob.core.windows.net/my-container/foo/tests/bar.avro # single file table +https://storageaccountname.blob.core.windows.net/my-container/foo/tests/*.* # mulitple file level tables +https://storageaccountname.blob.core.windows.net/my-container/foo/tests/{table}/*.avro #table without partition +https://storageaccountname.blob.core.windows.net/my-container/foo/tests/{table}/*/*.avro #table where partitions are not specified +https://storageaccountname.blob.core.windows.net/my-container/foo/tests/{table}/*.* # table where no partitions as well as data type specified +https://storageaccountname.blob.core.windows.net/my-container/{dept}/tests/{table}/*.avro # specifying keywords to be used in display name +https://storageaccountname.blob.core.windows.net/my-container/{dept}/tests/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.avro # specify partition key and value format +https://storageaccountname.blob.core.windows.net/my-container/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.avro # specify partition value only format +https://storageaccountname.blob.core.windows.net/my-container/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # for all extensions +https://storageaccountname.blob.core.windows.net/my-container/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # table is present at 2 levels down in container +https://storageaccountname.blob.core.windows.net/my-container/*/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # table is present at 3 levels down in container +``` + +**Valid path_specs.exclude** + +- \*\*/tests/\*\* +- https://storageaccountname.blob.core.windows.net/my-container/hr/** +- \*_/tests/_.csv +- https://storageaccountname.blob.core.windows.net/my-container/foo/*/my_table/** + +If you would like to write a more complicated function for resolving file names, then a {transformer} would be a good fit. + +:::caution + +Specify as long fixed prefix ( with out /\*/ ) as possible in `path_specs.include`. This will reduce the scanning time and cost, specifically on AWS S3 + +::: + +:::caution + +Running profiling against many tables or over many rows can run up significant costs. +While we've done our best to limit the expensiveness of the queries the profiler runs, you +should be prudent about the set of tables profiling is enabled on or the frequency +of the profiling runs. + +::: + +:::caution + +If you are ingesting datasets from AWS S3, we recommend running the ingestion on a server in the same region to avoid high egress costs. + +::: + +#### Compatibility + +Profiles are computed with PyDeequ, which relies on PySpark. Therefore, for computing profiles, we currently require Spark 3.0.3 with Hadoop 3.2 to be installed and the `SPARK_HOME` and `SPARK_VERSION` environment variables to be set. The Spark+Hadoop binary can be downloaded [here](https://www.apache.org/dyn/closer.lua/spark/spark-3.0.3/spark-3.0.3-bin-hadoop3.2.tgz). + +For an example guide on setting up PyDeequ on AWS, see [this guide](https://aws.amazon.com/blogs/big-data/testing-data-quality-at-scale-with-pydeequ/). + +:::caution + +From Spark 3.2.0+, Avro reader fails on column names that don't start with a letter and contains other character than letters, number, and underscore. [https://github.com/apache/spark/blob/72c62b6596d21e975c5597f8fff84b1a9d070a02/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroFileFormat.scala#L158] +Avro files that contain such columns won't be profiled. +::: + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.abs.source.ABSSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/abs/source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for ABS Data Lake, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/athena.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/athena.md new file mode 100644 index 00000000..8522b8f0 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/athena.md @@ -0,0 +1,1185 @@ +--- +sidebar_position: 3 +title: Athena +slug: /generated/ingestion/sources/athena +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Athena + +## Overview + +Athena is a data platform used to store and query analytical or operational data. Learn more in the [official Athena documentation](https://aws.amazon.com/athena/). + +The DataHub integration for Athena covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `athena` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Supported for S3 tables. Supported for types - View, Table. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. Profiling uses sql queries on whole table which can be expensive operation. Supported for types - Table. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Supported for S3 tables. Supported for types - View, Table. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `athena` module ingests metadata from Athena into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Create a policy with the following permissions and attach it to the AWS role or credentials used in your ingestion recipe. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "athena:GetTableMetadata", + "athena:StartQueryExecution", + "athena:GetQueryResults", + "athena:GetDatabase", + "athena:ListDataCatalogs", + "athena:GetDataCatalog", + "athena:ListQueryExecutions", + "athena:GetWorkGroup", + "athena:StopQueryExecution", + "athena:GetQueryResultsStream", + "athena:ListDatabases", + "athena:GetQueryExecution", + "athena:ListTableMetadata", + "athena:BatchGetQueryExecution", + "glue:GetTables", + "glue:GetDatabases", + "glue:GetTable", + "glue:GetDatabase", + "glue:SearchTables", + "glue:GetTableVersions", + "glue:GetTableVersion", + "glue:GetPartition", + "glue:GetPartitions", + "s3:GetObject", + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:athena:${region-id}:${account-id}:datacatalog/*", + "arn:aws:athena:${region-id}:${account-id}:workgroup/*", + "arn:aws:glue:${region-id}:${account-id}:tableVersion/*/*/*", + "arn:aws:glue:${region-id}:${account-id}:table/*/*", + "arn:aws:glue:${region-id}:${account-id}:catalog", + "arn:aws:glue:${region-id}:${account-id}:database/*", + "arn:aws:s3:::${datasets-bucket}", + "arn:aws:s3:::${datasets-bucket}/*" + ] + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucketMultipartUploads", + "s3:AbortMultipartUpload", + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::${athena-query-result-bucket}/*", + "arn:aws:s3:::${athena-query-result-bucket}" + ] + } + ] +} +``` + +Replace `${var}` with values appropriate for your Athena setup. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[athena]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: athena + config: + + # AWS Keys (Optional - Required only if local aws credentials are not set) + username: my_aws_access_key_id + password: my_aws_secret_access_key + + # Coordinates + aws_region: my_aws_region + work_group: primary + + # Options + query_result_location: "s3://my_staging_athena_results_bucket/results/" + +sink: +# sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
aws_region ✅
string
| Aws region where your Athena database is located | +|
query_result_location ✅
string
| S3 path to the [query result bucket](https://docs.aws.amazon.com/athena/latest/ug/querying.html#query-results-specify-location) which should be used by AWS Athena to store results of thequeries executed by DataHub. | +|
work_group ✅
string
| The name of your Amazon Athena Workgroups | +|
aws_role_arn
One of string, null
| AWS Role arn for Pyathena to assume in its connection
Default: None
| +|
aws_role_assumption_duration
integer
| Duration to assume the AWS Role for. Maximum of 43200 (12 hours)
Default: 3600
| +|
catalog_name
string
| Athena Catalog Name
Default: awsdatacatalog
| +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| The athena database to ingest from. If not set it will be autodetected
Default: None
| +|
emit_schema_fieldpaths_as_v1
boolean
| Convert simple field paths to DataHub field path v1 format. Simple column paths are those that do not contain any nested fields.
Default: False
| +|
extract_partitions
boolean
| Extract partitions for tables. Partition extraction needs to run a query (`select * from table$partitions`) on the table. Disable this if you don't want to grant select permission.
Default: True
| +|
extract_partitions_using_create_statements
boolean
| Extract partitions using the `SHOW CREATE TABLE` statement instead of querying the table's partitions directly. This needs to be enabled to extract Iceberg partitions. If extraction fails it falls back to the default partition extraction. This is experimental.
Default: False
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. | +|
password
One of string(password), null
| Same detection scheme as username
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
s3_staging_dir
One of string, null
| [deprecated in favor of `query_result_location`] S3 query location
Default: None
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| Username credential. If not specified, detected with boto3 rules. See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
AthenaProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Enable partition profiling. This will profile the latest partition of the table.
Default: False
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AthenaProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": false, + "description": "Enable partition profiling. This will profile the latest partition of the table.", + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "AthenaProfilingConfig", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schemas to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/AthenaProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": false, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username credential. If not specified, detected with boto3 rules. See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Same detection scheme as username", + "title": "Password" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The athena database to ingest from. If not set it will be autodetected", + "title": "Database" + }, + "aws_region": { + "description": "Aws region where your Athena database is located", + "title": "Aws Region", + "type": "string" + }, + "aws_role_arn": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS Role arn for Pyathena to assume in its connection", + "title": "Aws Role Arn" + }, + "aws_role_assumption_duration": { + "default": 3600, + "description": "Duration to assume the AWS Role for. Maximum of 43200 (12 hours)", + "title": "Aws Role Assumption Duration", + "type": "integer" + }, + "s3_staging_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "deprecated": true, + "description": "[deprecated in favor of `query_result_location`] S3 query location", + "title": "S3 Staging Dir" + }, + "work_group": { + "description": "The name of your Amazon Athena Workgroups", + "title": "Work Group", + "type": "string" + }, + "catalog_name": { + "default": "awsdatacatalog", + "description": "Athena Catalog Name", + "title": "Catalog Name", + "type": "string" + }, + "query_result_location": { + "description": "S3 path to the [query result bucket](https://docs.aws.amazon.com/athena/latest/ug/querying.html#query-results-specify-location) which should be used by AWS Athena to store results of thequeries executed by DataHub.", + "title": "Query Result Location", + "type": "string" + }, + "extract_partitions": { + "default": true, + "description": "Extract partitions for tables. Partition extraction needs to run a query (`select * from table$partitions`) on the table. Disable this if you don't want to grant select permission.", + "title": "Extract Partitions", + "type": "boolean" + }, + "extract_partitions_using_create_statements": { + "default": false, + "description": "Extract partitions using the `SHOW CREATE TABLE` statement instead of querying the table's partitions directly. This needs to be enabled to extract Iceberg partitions. If extraction fails it falls back to the default partition extraction. This is experimental.", + "title": "Extract Partitions Using Create Statements", + "type": "boolean" + }, + "emit_schema_fieldpaths_as_v1": { + "default": false, + "description": "Convert simple field paths to DataHub field path v1 format. Simple column paths are those that do not contain any nested fields.", + "title": "Emit Schema Fieldpaths As V1", + "type": "boolean" + } + }, + "required": [ + "aws_region", + "work_group", + "query_result_location" + ], + "title": "AthenaConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.athena.AthenaSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Athena, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-ad.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-ad.md new file mode 100644 index 00000000..6e6852a7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-ad.md @@ -0,0 +1,492 @@ +--- +sidebar_position: 4 +title: Azure AD +slug: /generated/ingestion/sources/azure-ad +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure AD + +## Overview + +Microsoft Entra ID is an identity and access management platform. Learn more in the [official Microsoft Entra ID documentation](https://learn.microsoft.com/entra/identity/). + +The DataHub integration for Microsoft Entra ID covers identity entities such as users, groups, and memberships. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------- | ------------------- | ---------------------------------------------------------------- | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | + + +## Module `azure-ad` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | + +### Overview + +The `azure-ad` module ingests metadata from Azure Ad into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +#### Required Azure AD Application Permissions + +Create a DataHub application in the Azure AD portal and grant these **Application** permissions: + +- `Group.Read.All` +- `GroupMember.Read.All` +- `User.Read.All` + +You can add permissions in the **API permissions** tab of your application configuration. + +

+ +

+ +You can verify required endpoint values from the **Endpoints** action in the application overview. + +

+ +

+ +#### SSO Caveat + +Users ingested from this connector will only be able to log in to DataHub if Okta OIDC SSO is configured in your DataHub deployment. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[azure-ad]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "azure-ad" + config: + client_id: "00000000-0000-0000-0000-000000000000" + tenant_id: "00000000-0000-0000-0000-000000000000" + client_secret: "xxxxx" + redirect: "https://login.microsoftonline.com/common/oauth2/nativeclient" + authority: "https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000" + token_url: "https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/token" + graph_url: "https://graph.microsoft.com/v1.0" + ingest_users: True + ingest_groups: True + groups_pattern: + allow: + - ".*" + users_pattern: + allow: + - ".*" + +sink: + # sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
authority ✅
string
| The authority (https://docs.microsoft.com/en-us/azure/active-directory/develop/msal-client-application-configuration) is a URL that indicates a directory that MSAL can request tokens from. | +|
client_id ✅
string
| Application ID. Found in your app registration on Azure AD Portal | +|
client_secret ✅
string(password)
| Client secret. Found in your app registration on Azure AD Portal | +|
tenant_id ✅
string
| Directory ID. Found in your app registration on Azure AD Portal | +|
token_url ✅
string
| The token URL that acquires a token from Azure AD for authorizing requests. This source will only work with v1.0 endpoint. | +|
azure_ad_response_to_groupname_attr
string
| Which Azure AD Group Response attribute to use as input to DataHub group name mapping.
Default: displayName
| +|
azure_ad_response_to_groupname_regex
string
| A regex used to parse the DataHub group name from the attribute specified in `azure_ad_response_to_groupname_attr`.
Default: (.*)
| +|
azure_ad_response_to_username_attr
string
| Which Azure AD User Response attribute to use as input to DataHub username mapping.
Default: userPrincipalName
| +|
azure_ad_response_to_username_regex
string
| A regex used to parse the DataHub username from the attribute specified in `azure_ad_response_to_username_attr`.
Default: (.*)
| +|
graph_url
string
| [Microsoft Graph API endpoint](https://docs.microsoft.com/en-us/graph/use-the-api)
Default: https://graph.microsoft.com/v1.0
| +|
ingest_group_membership
boolean
| Whether group membership should be ingested into DataHub. ingest_groups must be True if this is True.
Default: True
| +|
ingest_groups
boolean
| Whether groups should be ingested into DataHub.
Default: True
| +|
ingest_groups_users
boolean
| This option is useful only when `ingest_users` is set to False and `ingest_group_membership` to True. As effect, only the users which belongs to the selected groups will be ingested.
Default: True
| +|
ingest_users
boolean
| Whether users should be ingested into DataHub.
Default: True
| +|
mask_group_id
boolean
| Whether workunit ID's for groups should be masked to avoid leaking sensitive information.
Default: True
| +|
mask_user_id
boolean
| Whether workunit ID's for users should be masked to avoid leaking sensitive information.
Default: True
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
redirect
string
| Redirect URI. Found in your app registration on Azure AD Portal.
Default: https://login.microsoftonline.com/common/oauth2/na...
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
groups_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
groups_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
users_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
users_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Azure AD Stateful Ingestion Config.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Config to create a token and connect to Azure AD instance", + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure AD Stateful Ingestion Config." + }, + "client_id": { + "description": "Application ID. Found in your app registration on Azure AD Portal", + "title": "Client Id", + "type": "string" + }, + "tenant_id": { + "description": "Directory ID. Found in your app registration on Azure AD Portal", + "title": "Tenant Id", + "type": "string" + }, + "client_secret": { + "description": "Client secret. Found in your app registration on Azure AD Portal", + "format": "password", + "title": "Client Secret", + "type": "string", + "writeOnly": true + }, + "authority": { + "description": "The authority (https://docs.microsoft.com/en-us/azure/active-directory/develop/msal-client-application-configuration) is a URL that indicates a directory that MSAL can request tokens from.", + "title": "Authority", + "type": "string" + }, + "token_url": { + "description": "The token URL that acquires a token from Azure AD for authorizing requests. This source will only work with v1.0 endpoint.", + "title": "Token Url", + "type": "string" + }, + "redirect": { + "default": "https://login.microsoftonline.com/common/oauth2/nativeclient", + "description": "Redirect URI. Found in your app registration on Azure AD Portal.", + "title": "Redirect", + "type": "string" + }, + "graph_url": { + "default": "https://graph.microsoft.com/v1.0", + "description": "[Microsoft Graph API endpoint](https://docs.microsoft.com/en-us/graph/use-the-api)", + "title": "Graph Url", + "type": "string" + }, + "azure_ad_response_to_username_attr": { + "default": "userPrincipalName", + "description": "Which Azure AD User Response attribute to use as input to DataHub username mapping.", + "title": "Azure Ad Response To Username Attr", + "type": "string" + }, + "azure_ad_response_to_username_regex": { + "default": "(.*)", + "description": "A regex used to parse the DataHub username from the attribute specified in `azure_ad_response_to_username_attr`.", + "title": "Azure Ad Response To Username Regex", + "type": "string" + }, + "azure_ad_response_to_groupname_attr": { + "default": "displayName", + "description": "Which Azure AD Group Response attribute to use as input to DataHub group name mapping.", + "title": "Azure Ad Response To Groupname Attr", + "type": "string" + }, + "azure_ad_response_to_groupname_regex": { + "default": "(.*)", + "description": "A regex used to parse the DataHub group name from the attribute specified in `azure_ad_response_to_groupname_attr`.", + "title": "Azure Ad Response To Groupname Regex", + "type": "string" + }, + "ingest_users": { + "default": true, + "description": "Whether users should be ingested into DataHub.", + "title": "Ingest Users", + "type": "boolean" + }, + "ingest_groups": { + "default": true, + "description": "Whether groups should be ingested into DataHub.", + "title": "Ingest Groups", + "type": "boolean" + }, + "ingest_group_membership": { + "default": true, + "description": "Whether group membership should be ingested into DataHub. ingest_groups must be True if this is True.", + "title": "Ingest Group Membership", + "type": "boolean" + }, + "ingest_groups_users": { + "default": true, + "description": "This option is useful only when `ingest_users` is set to False and `ingest_group_membership` to True. As effect, only the users which belongs to the selected groups will be ingested.", + "title": "Ingest Groups Users", + "type": "boolean" + }, + "users_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for users to filter in ingestion." + }, + "groups_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for groups to include in ingestion." + }, + "mask_group_id": { + "default": true, + "description": "Whether workunit ID's for groups should be masked to avoid leaking sensitive information.", + "title": "Mask Group Id", + "type": "boolean" + }, + "mask_user_id": { + "default": true, + "description": "Whether workunit ID's for users should be masked to avoid leaking sensitive information.", + "title": "Mask User Id", + "type": "boolean" + } + }, + "required": [ + "client_id", + "tenant_id", + "client_secret", + "authority", + "token_url" + ], + "title": "AzureADConfig", + "type": "object" +} +``` + + + + + +As a prerequisite, you should [create a DataHub Application](https://docs.microsoft.com/en-us/graph/toolkit/get-started/add-aad-app-registration) within the Azure AD Portal with the permissions +to read your organization's Users and Groups. The following permissions are required, with the `Application` permission type: + +- `Group.Read.All` +- `GroupMember.Read.All` +- `User.Read.All` + +You can add a permission by navigating to the permissions tab in your DataHub application on the Azure AD portal. + +

+ +

+ +You can view the necessary endpoints to configure by clicking on the Endpoints button in the Overview tab. + +

+ +

+ +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Extracting DataHub Users + +##### Usernames + +Usernames serve as unique identifiers for users on DataHub. This connector extracts usernames using the +"userPrincipalName" field of an [Azure AD User Response](https://docs.microsoft.com/en-us/graph/api/user-list?view=graph-rest-1.0&tabs=http#response-1), +which is the unique identifier for your Azure AD users. + +If this is not how you wish to map to DataHub usernames, you can provide a custom mapping using the configurations options detailed below. Namely, `azure_ad_response_to_username_attr` +and `azure_ad_response_to_username_regex`. + +##### Responses + +This connector also extracts basic user response information from Azure. The following fields of the Azure User Response are extracted +and mapped to the DataHub `CorpUserInfo` aspect: + +- display name +- first name +- last name +- email +- title +- country + +#### Extracting DataHub Groups + +##### Group Names + +Group names serve as unique identifiers for groups on DataHub. This connector extracts group names using the "name" attribute of an Azure Group Response. +By default, a URL-encoded version of the full group name is used as the unique identifier (CorpGroupKey) and the raw "name" attribute is mapped +as the display name that will appear in DataHub's UI. + +If this is not how you wish to map to DataHub group names, you can provide a custom mapping using the configurations options detailed below. Namely, `azure_ad_response_to_groupname_attr` +and `azure_ad_response_to_groupname_regex`. + +##### Responses + +This connector also extracts basic group information from Azure. The following fields of the [Azure AD Group Response](https://docs.microsoft.com/en-us/graph/api/group-list?view=graph-rest-1.0&tabs=http#response-1) are extracted and mapped to the +DataHub `CorpGroupInfo` aspect: + +- name +- description + +#### Extracting Group Membership + +This connector additional extracts the edges between Users and Groups that are stored in [Azure AD](https://docs.microsoft.com/en-us/graph/api/group-list-members?view=graph-rest-1.0&tabs=http#response-1). It maps them to the `GroupMembership` aspect +associated with DataHub users (CorpUsers). + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.identity.azure_ad.AzureADSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/identity/azure_ad.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Azure AD, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-data-factory.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-data-factory.md new file mode 100644 index 00000000..2d272a7a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/azure-data-factory.md @@ -0,0 +1,369 @@ +--- +sidebar_position: 5 +title: Azure Data Factory +slug: /generated/ingestion/sources/azure-data-factory +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Azure Data Factory + +## Overview + +Azure Data Factory is a streaming or integration platform. Learn more in the [official Azure Data Factory documentation](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app). + +The DataHub integration for Azure Data Factory covers streaming/integration entities such as topics, connectors, pipelines, or jobs. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| ADF Concept | DataHub Entity | +| ------------ | ------------------- | +| Data Factory | Container | +| Pipeline | DataFlow | +| Activity | DataJob | +| Dataset | Dataset | +| Pipeline Run | DataProcessInstance | + + +## Module `azure-data-factory` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Data Factory. | +| Column-level Lineage | ✅ | Extracts column-level lineage from Copy activities. Supported for types - Copy Activity. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Extracts lineage from Copy and Data Flow activities. Supported for types - Copy Activity, Data Flow Activity. | + +### Overview + +The `azure-data-factory` module ingests metadata from Azure Data Factory into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +#### Required Permissions + +The connector only performs **read operations**. Grant one of the following: + +**Option 1: Built-in Reader Role** (recommended) + +Assign the [Reader](https://learn.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#reader) role at subscription, resource group, or Data Factory level. + +**Option 2: Custom Role with Minimal Permissions** + +Download [`datahub-adf-reader-role.json`](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/datahub-adf-reader-role.json), update the `{subscription-id}`, then: + +```bash +# Create custom role +az role definition create --role-definition datahub-adf-reader-role.json + +# Assign to service principal +az role assignment create \ + --assignee \ + --role "DataHub ADF Reader" \ + --scope /subscriptions/{subscription-id} +``` + +For detailed instructions, see [Azure custom roles](https://learn.microsoft.com/en-us/azure/role-based-access-control/custom-roles). + +#### Setup + +1. Configure authentication for the connector runtime. +2. Grant read permissions on the target Data Factory resources. +3. Provide a subscription scope and optional pattern filters in the ingestion recipe. + +This section intentionally complements (and does not duplicate) the generated **Starter Recipe** section. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[azure-data-factory]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +# Example recipe for Azure Data Factory source +# See README.md for full configuration options + +source: + type: azure-data-factory + config: + # Required: Azure subscription containing Data Factories + subscription_id: ${AZURE_SUBSCRIPTION_ID} + + # Optional: Filter to specific resource group + # resource_group: my-resource-group + + # Authentication (using service principal) + credential: + authentication_method: service_principal + client_id: ${AZURE_CLIENT_ID} + client_secret: ${AZURE_CLIENT_SECRET} + tenant_id: ${AZURE_TENANT_ID} + + # Optional: Filter factories by name pattern + factory_pattern: + allow: + - ".*" # Allow all factories by default + deny: [] + + # Optional: Filter pipelines by name pattern + pipeline_pattern: + allow: + - ".*" # Allow all pipelines by default + deny: [] + + # Feature flags + include_lineage: true + include_column_lineage: false # Advanced: requires Data Flow parsing + include_execution_history: false # Set to true for pipeline run history + execution_history_days: 7 # Only used when include_execution_history is true + + # Optional: Map linked services to platform instances for accurate lineage + # platform_instance_map: + # "my-snowflake-connection": "prod_snowflake" + + # Optional: Platform instance for this ADF connector + # platform_instance: "main-adf" + + # Environment + env: PROD + + # Optional: Stateful ingestion for stale entity removal + # stateful_ingestion: + # enabled: true + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" + + +``` + +### Config Details +Configuration schema is not auto-generated for this module. Refer to the source code coordinates and module guidance below. + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +:::note Not Azure Fabric +This connector is for **Azure Data Factory** (classic), not Azure Fabric's Data Factory. Azure Fabric support is planned for a future release. +::: + +#### Authentication Methods + +The connector supports multiple authentication methods: + +| Method | Best For | Configuration | +| -------------------------- | ------------------------------------------------ | --------------------------------------------------- | +| **Service Principal** | Production environments | `authentication_method: service_principal` | +| **Managed Identity** | Azure-hosted deployments (VMs, AKS, App Service) | `authentication_method: managed_identity` | +| **Azure CLI** | Local development | `authentication_method: cli` (run `az login` first) | +| **DefaultAzureCredential** | Flexible environments | `authentication_method: default` | + +For service principal setup, see [Register an application with Microsoft Entra ID](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app). + +#### Lineage Extraction + +##### Which Activities Produce Lineage? + +The connector extracts **table-level lineage** from these ADF activity types: + +| Activity Type | Lineage Behavior | +| ------------------- | ---------------------------------------------------------- | +| **Copy Activity** | Creates lineage from input dataset(s) to output dataset | +| **Data Flow** | Extracts sources, sinks, and transformation script | +| **Lookup Activity** | Creates input lineage from the lookup dataset | +| **ExecutePipeline** | Creates pipeline-to-pipeline lineage to the child pipeline | + +Lineage is enabled by default (`include_lineage: true`). + +##### How Lineage Resolution Works + +For lineage to connect properly to datasets ingested from other sources (e.g., Snowflake, BigQuery), the connector needs to know which DataHub platform each ADF linked service corresponds to. + +**Step 1: Automatic Platform Mapping** + +The connector automatically maps ADF linked service types to DataHub platforms. For example, a `Snowflake` linked service maps to the `snowflake` platform. + +
+View all supported linked service mappings + +| ADF Linked Service Type | DataHub Platform | +| ------------------------ | ---------------- | +| AzureBlobStorage | `abs` | +| AzureBlobFS | `abs` | +| AzureDataLakeStore | `abs` | +| AzureFileStorage | `abs` | +| AzureSqlDatabase | `mssql` | +| AzureSqlDW | `mssql` | +| AzureSynapseAnalytics | `mssql` | +| AzureSqlMI | `mssql` | +| SqlServer | `mssql` | +| AzureDatabricks | `databricks` | +| AzureDatabricksDeltaLake | `databricks` | +| AmazonS3 | `s3` | +| AmazonS3Compatible | `s3` | +| AmazonRedshift | `redshift` | +| GoogleCloudStorage | `gcs` | +| GoogleBigQuery | `bigquery` | +| Snowflake | `snowflake` | +| PostgreSql | `postgres` | +| AzurePostgreSql | `postgres` | +| MySql | `mysql` | +| AzureMySql | `mysql` | +| Oracle | `oracle` | +| OracleServiceCloud | `oracle` | +| Db2 | `db2` | +| Teradata | `teradata` | +| Vertica | `vertica` | +| Hive | `hive` | +| Spark | `spark` | +| Hdfs | `hdfs` | +| Salesforce | `salesforce` | +| SalesforceServiceCloud | `salesforce` | +| SalesforceMarketingCloud | `salesforce` | + +Unsupported linked service types log a warning and skip lineage for that dataset. + +
+ +**Step 2: Platform Instance Mapping (for cross-recipe lineage)** + +If you're ingesting the same data sources with other DataHub connectors (e.g., Snowflake, BigQuery), you need to ensure the `platform_instance` values match. Use `platform_instance_map` to map your ADF linked service names to the platform instance used in your other recipes: + +```yaml +# ADF Recipe +source: + type: azure-data-factory + config: + subscription_id: ${AZURE_SUBSCRIPTION_ID} + platform_instance_map: + # Key: Your ADF linked service name (exact match required) + # Value: The platform_instance from your other source recipe + "snowflake-prod-connection": "prod_warehouse" + "bigquery-analytics": "analytics_project" +``` + +```yaml +# Corresponding Snowflake Recipe (platform_instance must match) +source: + type: snowflake + config: + platform_instance: "prod_warehouse" # Must match the value in platform_instance_map + # ... other config +``` + +Without matching `platform_instance` values, lineage will create separate dataset entities instead of connecting to your existing ingested datasets. + +##### Data Flow Transformation Scripts + +For Data Flow activities, the connector extracts the transformation script and stores it in the `dataTransformLogic` aspect, visible in the DataHub UI under activity details. + +##### Column-Level Lineage + +The connector extracts **column-level lineage** from Copy activities, enabled by default (`include_column_lineage: true`). + +**Supported Mapping Formats** + +| Format | Description | ADF Configuration | +| --------------------- | --------------------------------------------------------------------------- | ------------------------------------------------------- | +| **Dictionary Format** | Legacy format with direct source-to-sink column mapping | `translator.columnMappings: {"src_col": "sink_col"}` | +| **List Format** | Current format with structured source/sink objects | `translator.mappings: [{source: {name}, sink: {name}}]` | +| **Auto-mapping** | Inferred 1:1 mappings when no explicit mappings and source schema available | TabularTranslator with no columnMappings or mappings | + +**Limitations** + +- **Copy Activity Only**: Column lineage is currently extracted only from Copy activities. Other activity types (Data Flow, Lookup, etc.) produce table-level lineage only. +- **Schema Availability**: Auto-mapping inference requires source dataset schema information (defined in ADF dataset's `schema` or `structure` property). If schema is unavailable, only explicit mappings are extracted. + +#### Execution History + +Pipeline runs are extracted as `DataProcessInstance` entities by default: + +```yaml +source: + type: azure-data-factory + config: + include_execution_history: true # default + execution_history_days: 7 # 1-90 days +``` + +This provides run status, duration, timestamps, trigger info, parameters, and activity-level details. + +#### Advanced: Multi-Environment Setup + +##### When to Use `platform_instance` + +Use the ADF connector's `platform_instance` config to distinguish **separate ADF deployments** when ingesting from multiple subscriptions or tenants: + +| Scenario | Risk | Solution | +| ---------------------- | ------------------------------ | ------------ | +| Single subscription | None | Not needed | +| Multiple subscriptions | Low | Recommended | +| Multiple tenants | **High** - name collision risk | **Required** | + +```yaml +# Multi-tenant example +source: + type: azure-data-factory + config: + subscription_id: "tenant-a-sub" + platform_instance: "tenant-a" # Prevents URN collisions +``` + +:::warning +Factory names are unique within Azure, but different tenants could have identically-named factories. Use `platform_instance` to prevent entity overwrites. +::: + +##### URN Format + +Pipeline URNs follow this format: + +``` +urn:li:dataFlow:(azure-data-factory,{factory_name}.{pipeline_name},{env}) +``` + +With `platform_instance`: + +``` +urn:li:dataFlow:(azure-data-factory,{platform_instance}.{factory_name}.{pipeline_name},{env}) +``` + +For Azure naming rules, see [Azure Data Factory naming rules](https://learn.microsoft.com/en-us/azure/data-factory/naming-rules). + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.azure_data_factory.adf_source.AzureDataFactorySource` + +:::tip Questions? + +If you've got any questions on configuring ingestion for Azure Data Factory, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/bigquery.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/bigquery.md new file mode 100644 index 00000000..c352339c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/bigquery.md @@ -0,0 +1,2150 @@ +--- +sidebar_position: 6 +title: BigQuery +slug: /generated/ingestion/sources/bigquery +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# BigQuery + +## Overview + +BigQuery is a data platform used to store and query analytical or operational data. Learn more in the [official BigQuery documentation](https://cloud.google.com/bigquery). + +The DataHub integration for BigQuery covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `bigquery` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Project, Dataset. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Optionally enabled via configuration. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Dataset Usage | ✅ | Enabled by default, can be disabled via configuration `include_usage_statistics`. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| Partition Support | ✅ | Enabled by default, partition keys and clustering keys are supported. | +| [Platform Instance](../../../platform-instances.md) | ❌ | Platform instance is pre-set to the BigQuery project id. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Optionally enabled via configuration. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `bigquery` module ingests metadata from Bigquery into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Familiarize yourself with BigQuery ingestion architecture: + +

+ +

+ +Two key concepts: + +- **Extractor Project**: Project containing the service account used to run metadata extraction queries +- **BigQuery Projects**: Projects from which DataHub collects metadata (tables, lineage, usage, profiling). By default includes the extractor project; configure `project_ids` to specify projects explicitly + +#### Create a datahub profile in GCP + +1. Create a custom role for DataHub following [BigQuery docs](https://cloud.google.com/iam/docs/creating-custom-roles#creating_a_custom_role) +2. Grant permissions to this role on the extractor project and all target projects (see below) + +##### Basic Requirements (needed for metadata ingestion) + +**1. Grant the following permissions on the Extractor Project:** + +| permission                       | Description                                                                                                                         | Capability                                                               | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `bigquery.jobs.create`           | Run jobs (e.g. queries) within the project. _This only needs for the extractor project where the service account belongs_           |                                                                                                               | +| `bigquery.jobs.list`             | Manage the queries that the service account has sent. _This only needs for the extractor project where the service account belongs_ |                                                                                                               | +| `bigquery.readsessions.create`   | Create a session for streaming large results. _This only needs for the extractor project where the service account belongs_         |                                                                                                               | +| `bigquery.readsessions.getData` | Get data from the read session. _This only needs for the extractor project where the service account belongs_                       | + +**2. Grant the following permissions on all target projects for metadata extraction:** + +:::info + +These permissions must be granted on **every project** you want to extract metadata from. + +::: +| Permission | Description | Capability | Default GCP Role Which Contains This Permission | +|----------------------------------|-----------------------------------------------------------------------------------------------------------------|-------------------------------------|---------------------------------------------------------------------------| +| `bigquery.datasets.get` | Retrieve metadata about a dataset. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.datasets.getIamPolicy` | Read a dataset's IAM permissions. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.tables.list` | List BigQuery tables. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.tables.get` | Retrieve metadata for a table. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.routines.get` | Get Routines. Needs to retrieve metadata for a table from system table. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.routines.list` | List Routines. Needs to retrieve metadata for a table from system table. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `resourcemanager.projects.get` | Get project metadata. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `resourcemanager.projects.list` | Search projects. Needed if not setting `project_ids`. | Table Metadata Extraction | [roles/bigquery.metadataViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) | +| `bigquery.jobs.listAll` | List all jobs (queries) submitted by any user. Needs for Lineage extraction. | Lineage Extraction/Usage Extraction | [roles/bigquery.resourceViewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer) | +| `logging.logEntries.list` | Fetch log entries for lineage/usage data. Not required if `use_exported_bigquery_audit_metadata` is enabled. | Lineage Extraction/Usage Extraction | [roles/logging.privateLogViewer](https://cloud.google.com/logging/docs/access-control#logging.privateLogViewer) | +| `logging.privateLogEntries.list` | Fetch log entries for lineage/usage data. Not required if `use_exported_bigquery_audit_metadata` is enabled. | Lineage Extraction/Usage Extraction | [roles/logging.privateLogViewer](https://cloud.google.com/logging/docs/access-control#logging.privateLogViewer) | +| `bigquery.tables.getData` | Access table data to extract storage size, last updated at, partition information, data profiles etc. **Required when profiling is enabled or when `use_tables_list_query_v2` is enabled.** This permission is needed to query BigQuery's `__TABLES__` pseudo-table. | Profiling/Enhanced Table Metadata | | +| `datacatalog.policyTags.get` | _Optional_ Get policy tags for columns with associated policy tags. This permission is required only if `extract_policy_tags_from_catalog` is enabled. | Policy Tag Extraction | [roles/datacatalog.viewer](https://cloud.google.com/data-catalog/docs/access-control#permissions-and-roles) | + +:::warning Important: bigquery.tables.getData Permission + +The `bigquery.tables.getData` permission is **required** in the following scenarios: + +- When **profiling is enabled** (`profiling.enabled: true`) +- When **`use_tables_list_query_v2` is enabled** (for enhanced table metadata extraction) + +Without this permission, you'll encounter errors when the connector tries to access BigQuery's `__TABLES__` pseudo-table for detailed table information including partition data, row counts, and storage metrics. + +::: + +#### Create a service account in the Extractor Project + +1. Create a service account following [BigQuery docs](https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-console) and assign the custom role created above +2. Download a service account JSON keyfile + Example credential file: + +```json +{ + "type": "service_account", + "project_id": "project-id-1234567", + "private_key_id": "d0121d0000882411234e11166c6aaa23ed5d74e0", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-----", + "client_email": "test@suppproject-id-1234567.iam.gserviceaccount.com", + "client_id": "113545814931671546333", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%suppproject-id-1234567.iam.gserviceaccount.com" +} +``` + +3. To provide credentials to the source, you can either: + + Set an environment variable: + + ```sh + $ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/keyfile.json" + ``` + + _or_ + + Set credential config in your source based on the credential json file. For example: + + ```yml + credential: + project_id: project-id-1234567 + private_key_id: "d0121d0000882411234e11166c6aaa23ed5d74e0" + private_key: "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-----\n" + client_email: "test@suppproject-id-1234567.iam.gserviceaccount.com" + client_id: "123456678890" + ``` + +##### Profiling Requirements + +**For external tables backed by Google Drive:** + +Grant "Viewer" access to the service account's email (`client_email` from credentials JSON) on the Google Drive documents: + +1. Find the source document: BigQuery Console → Table → Details → "Source" field +2. Share the document: Open document → Share → Add service account email with "Viewer" access + +![Google Drive Sharing Dialog](https://github.com/datahub-project/static-assets/raw/main/imgs/integrations/bigquery/google_drive_share.png) + + +### Install the Plugin +```shell +pip install 'acryl-datahub[bigquery]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: bigquery + config: + dataset_pattern: + allow: + - finance_bq_dataset + table_pattern: + deny: + # The exact name of the table is revenue_table_name + # The reason we have this `.*` at the beginning is because the current implmenetation of table_pattern is testing + # project_id.dataset_name.table_name + # We will improve this in the future + - .*revenue_table_name + include_table_lineage: true + include_usage_statistics: true + profiling: + enabled: true + profile_table_level_only: true + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
column_limit
integer
| Maximum number of columns to process in a table. This is a low level config property which should be touched with care. This restriction is needed because excessively wide tables can result in failure to ingest the schema.
Default: 300
| +|
convert_column_urns_to_lowercase
boolean
| When enabled, converts column URNs to lowercase to ensure cross-platform compatibility.
Default: False
| +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
debug_include_full_payloads
boolean
| Include full payload into events. It is only for debugging and internal use.
Default: False
| +|
enable_legacy_sharded_table_support
boolean
| Use the legacy sharded table urn suffix added.
Default: True
| +|
enable_stateful_lineage_ingestion
boolean
| Enable stateful lineage ingestion. This will store lineage window timestamps after successful lineage ingestion. and will not run lineage ingestion for same timestamps in subsequent run. NOTE: This only works with use_queries_v2=False (legacy extraction path). For queries v2, use enable_stateful_time_window instead.
Default: True
| +|
enable_stateful_profiling
boolean
| Enable stateful profiling. This will store profiling timestamps per dataset after successful profiling. and will not run profiling again in subsequent run if table has not been updated.
Default: True
| +|
enable_stateful_time_window
boolean
| Enable stateful time window tracking. This will store the time window after successful extraction and adjust the time window in subsequent runs to avoid reprocessing. NOTE: This is ONLY applicable when using queries v2 (use_queries_v2=True). This replaces enable_stateful_lineage_ingestion and enable_stateful_usage_ingestion for the queries v2 extraction path, since queries v2 extracts lineage, usage, operations, and queries together from a single audit log and uses a unified time window.
Default: False
| +|
enable_stateful_usage_ingestion
boolean
| Enable stateful lineage ingestion. This will store usage window timestamps after successful usage ingestion. and will not run usage ingestion for same timestamps in subsequent run. NOTE: This only works with use_queries_v2=False (legacy extraction path). For queries v2, use enable_stateful_time_window instead.
Default: True
| +|
end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
exclude_empty_projects
boolean
| Option to exclude empty projects from being ingested.
Default: False
| +|
extra_client_options
object
| Additional options to pass to google.cloud.logging_v2.client.Client.
Default: {}
| +|
extract_column_lineage
boolean
| If enabled, generate column level lineage. Requires lineage_use_sql_parser to be enabled.
Default: False
| +|
extract_lineage_from_catalog
boolean
| This flag enables the data lineage extraction from Data Lineage API exposed by Google Data Catalog. NOTE: This extractor can't build views lineage. It's recommended to enable the view's DDL parsing. Read the docs to have more information about: https://cloud.google.com/data-catalog/docs/concepts/about-data-lineage
Default: False
| +|
extract_policy_tags_from_catalog
boolean
| This flag enables the extraction of policy tags from the Google Data Catalog API. When enabled, the extractor will fetch policy tags associated with BigQuery table columns. For more information about policy tags and column-level security, refer to the documentation: https://cloud.google.com/bigquery/docs/column-level-security-intro
Default: False
| +|
include_column_lineage_with_gcs
boolean
| When enabled, column-level lineage will be extracted from the gcs.
Default: True
| +|
include_data_platform_instance
boolean
| Whether to create a DataPlatformInstance aspect, equal to the BigQuery project id. If enabled, will cause redundancy in the browse path for BigQuery entities in the UI, because the project id is represented as the top-level container.
Default: False
| +|
include_external_url
boolean
| Whether to populate BigQuery Console url to Datasets/Tables
Default: True
| +|
include_queries
boolean
| If enabled, generate query entities associated with lineage edges. Only applicable if `use_queries_v2` is enabled.
Default: True
| +|
include_query_usage_statistics
boolean
| If enabled, generate query popularity statistics. Only applicable if `use_queries_v2` is enabled.
Default: True
| +|
include_schema_metadata
boolean
| Whether to ingest the BigQuery schema, i.e. projects, schemas, tables, and views.
Default: True
| +|
include_table_constraints
boolean
| Whether to ingest table constraints. If you know you don't use table constraints, you can disable it to save one extra query per dataset. In general it should be enabled
Default: True
| +|
include_table_lineage
One of boolean, null
| Option to enable/disable lineage generation. Is enabled by default.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_table_snapshots
One of boolean, null
| Whether table snapshots should be ingested.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_usage_statistics
boolean
| Generate usage statistic
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
lineage_sql_parser_use_raw_names
boolean
| This parameter ignores the lowercase pattern stipulated in the SQLParser. NOTE: Ignored if lineage_use_sql_parser is False.
Default: False
| +|
lineage_use_sql_parser
boolean
| Use sql parser to resolve view/table lineage.
Default: True
| +|
log_page_size
integer
| The number of log item will be queried per page for lineage collection
Default: 1000
| +|
match_fully_qualified_names
boolean
| [deprecated] Whether `dataset_pattern` is matched against fully qualified dataset name `.`.
Default: True
| +|
max_query_duration
string(duration)
| Correction to pad start_time and end_time with. For handling the case where the read happens within our time range but the query completion event is delayed and happens after the configured end time.
Default: PT15M
| +|
max_threads_dataset_parallelism
integer
| Number of worker threads to use to parallelize BigQuery Dataset Metadata Extraction. Set to 1 to disable.
Default: 20
| +|
number_of_datasets_process_in_batch_if_profiling_enabled
integer
| Number of partitioned table queried in batch when getting metadata. This is a low level config property which should be touched with care. This restriction is needed because we query partitions system view which throws error if we try to touch too many tables.
Default: 1000
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. | +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
project_on_behalf
One of string, null
| [Advanced] The BigQuery project in which queries are executed. Will be passed when creating a job. If not passed, falls back to the project associated with the service account.
Default: None
| +|
rate_limit
boolean
| Should we rate limit requests made to API.
Default: False
| +|
requests_per_min
integer
| Used to control number of API calls made per min. Only used when `rate_limit` is set to `True`.
Default: 60
| +|
scheme
string
|
Default: bigquery
| +|
sharded_table_pattern
string
| The regex pattern to match sharded tables and group as one table. This is a very low level config parameter, only change if you know what you are doing,
Default: ((.+\D)[_$]?)?(\d\d\d\d(?:0[1-9]|1[0-2])(?:0[1-9]|...
| +|
start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
temp_table_dataset_prefix
string
| If you are creating temp tables in a dataset with a particular prefix you can use this config to set the prefix for the dataset. This is to support workflows from before bigquery's introduction of temp tables. By default we use `_` because of datasets that begin with an underscore are hidden by default https://cloud.google.com/bigquery/docs/datasets#dataset-naming.
Default: _
| +|
upstream_lineage_in_report
boolean
| Useful for debugging lineage information. Set to True to see the raw lineage created internally. Only works with legacy approach (`use_queries_v2: False`).
Default: False
| +|
use_date_sharded_audit_log_tables
boolean
| Whether to read date sharded tables or time partitioned tables when extracting usage from exported audit logs.
Default: False
| +|
use_exported_bigquery_audit_metadata
boolean
| When configured, use BigQueryAuditMetadata in bigquery_audit_metadata_datasets to compute lineage information.
Default: False
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
use_queries_v2
boolean
| If enabled, uses the new queries extractor to extract queries from bigquery.
Default: True
| +|
use_tables_list_query_v2
boolean
| List tables using an improved query that extracts partitions and last modified timestamps more accurately. Requires the ability to read table data. Automatically enabled when profiling is enabled.
Default: False
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
bigquery_audit_metadata_datasets
One of array, null
| A list of datasets that contain a table named cloudaudit_googleapis_com_data_access which contain BigQuery audit logs, specifically, those containing BigQueryAuditMetadata. It is recommended that the project of the dataset is also specified, for example, projectA.datasetB.
Default: None
| +|
bigquery_audit_metadata_datasets.string
string
| | +|
capture_dataset_label_as_tag
One of boolean, AllowDenyPattern
| Capture BigQuery dataset labels as DataHub tag
Default: False
| +|
capture_dataset_label_as_tag.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
capture_dataset_label_as_tag.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
capture_dataset_label_as_tag.allow.string
string
| | +|
capture_dataset_label_as_tag.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
capture_dataset_label_as_tag.deny.string
string
| | +|
capture_table_label_as_tag
One of boolean, AllowDenyPattern
| Capture BigQuery table labels as DataHub tag
Default: False
| +|
capture_table_label_as_tag.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
capture_table_label_as_tag.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
capture_table_label_as_tag.allow.string
string
| | +|
capture_table_label_as_tag.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
capture_table_label_as_tag.deny.string
string
| | +|
capture_view_label_as_tag
One of boolean, AllowDenyPattern
| Capture BigQuery view labels as DataHub tag
Default: False
| +|
capture_view_label_as_tag.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
capture_view_label_as_tag.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
capture_view_label_as_tag.allow.string
string
| | +|
capture_view_label_as_tag.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
capture_view_label_as_tag.deny.string
string
| | +|
credential
One of GCPCredential, null
| BigQuery credential informations
Default: None
| +|
credential.client_email ❓
string
| Client email | +|
credential.client_id ❓
string
| Client Id | +|
credential.private_key ❓
string(password)
| Private key in a form of '-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n' | +|
credential.private_key_id ❓
string
| Private key id | +|
credential.auth_provider_x509_cert_url
string
| Auth provider x509 certificate url
Default: https://www.googleapis.com/oauth2/v1/certs
| +|
credential.auth_uri
string
| Authentication uri
Default: https://accounts.google.com/o/oauth2/auth
| +|
credential.client_x509_cert_url
One of string, null
| If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email
Default: None
| +|
credential.project_id
One of string, null
| Project id to set the credentials
Default: None
| +|
credential.token_uri
string
| Token uri
Default: https://oauth2.googleapis.com/token
| +|
credential.type
string
| Authentication type
Default: service_account
| +|
dataset_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
dataset_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
gcs_lineage_config
GcsLineageProviderConfig
| Any source that produces gcs lineage from/to Datasets should inherit this class. | +|
gcs_lineage_config.ignore_non_path_spec_path
boolean
| Ignore paths that are not match in path_specs. It only applies if path_specs are specified.
Default: False
| +|
gcs_lineage_config.strip_urls
boolean
| Strip filename from gcs url. It only applies if path_specs are not specified.
Default: True
| +|
gcs_lineage_config.path_specs
array
| List of PathSpec. See below the details about PathSpec
Default: []
| +|
gcs_lineage_config.path_specs.PathSpec
PathSpec
| | +|
gcs_lineage_config.path_specs.PathSpec.include ❓
string
| Path to table. Name variable `{table}` is used to mark the folder with dataset. In absence of `{table}`, file level dataset will be created. Check below examples for more details. | +|
gcs_lineage_config.path_specs.PathSpec.allow_double_stars
boolean
| Allow double stars in the include path. This can affect performance significantly if enabled
Default: False
| +|
gcs_lineage_config.path_specs.PathSpec.autodetect_partitions
boolean
| Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`
Default: True
| +|
gcs_lineage_config.path_specs.PathSpec.default_extension
One of string, null
| For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.
Default: None
| +|
gcs_lineage_config.path_specs.PathSpec.enable_compression
boolean
| Enable or disable processing compressed files. Currently .gz and .bz files are supported.
Default: True
| +|
gcs_lineage_config.path_specs.PathSpec.include_hidden_folders
boolean
| Include hidden folders in the traversal (folders starting with . or _
Default: False
| +|
gcs_lineage_config.path_specs.PathSpec.sample_files
boolean
| Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled
Default: True
| +|
gcs_lineage_config.path_specs.PathSpec.table_name
One of string, null
| Display name of the dataset.Combination of named variables from include path and strings
Default: None
| +|
gcs_lineage_config.path_specs.PathSpec.traversal_method
Enum
| One of: "ALL", "MIN_MAX", "MAX" | +|
gcs_lineage_config.path_specs.PathSpec.exclude
One of array, null
| list of paths in glob pattern which will be excluded while scanning for the datasets
Default: []
| +|
gcs_lineage_config.path_specs.PathSpec.exclude.string
string
| | +|
gcs_lineage_config.path_specs.PathSpec.file_types
array
| Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.
Default: ['csv', 'tsv', 'json', 'parquet', 'avro']
| +|
gcs_lineage_config.path_specs.PathSpec.file_types.string
string
| | +|
gcs_lineage_config.path_specs.PathSpec.tables_filter_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
gcs_lineage_config.path_specs.PathSpec.tables_filter_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
project_id_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
project_id_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
project_ids
array
| Ingests specified project_ids. Use this property if you want to specify what projects to ingest or don't want to give project resourcemanager.projects.list to your service account. Overrides `project_id_pattern`. | +|
project_ids.string
string
| | +|
project_labels
array
| Ingests projects with the specified labels. Set value in the format of `key:value`. Use this property to define which projects to ingest basedon project-level labels. If project_ids or project_id is set, this configuration has no effect. The ingestion process filters projects by label first, and then applies the project_id_pattern. | +|
project_labels.string
string
| | +|
pushdown_allow_usernames
array
| List of user email patterns using SQL LIKE syntax (e.g., 'analyst_%@company.com') which WILL be considered for lineage/usage/queries extraction. Uses case-insensitive LIKE for server-side filtering (e.g., 'analyst_%' matches 'Analyst_John@company.com'). Only applicable if `use_queries_v2` is enabled. If not specified, all users not in deny list are included.
Default: []
| +|
pushdown_allow_usernames.string
string
| | +|
pushdown_deny_usernames
array
| List of user email patterns using SQL LIKE syntax (e.g., 'bot_%', '%@%.iam.gserviceaccount.com') which will NOT be considered for lineage/usage/queries extraction. Uses case-insensitive LIKE for server-side filtering (e.g., 'bot_%' matches 'Bot_User@example.com'). This is primarily useful for improving performance by filtering out users with extremely high query volumes. Only applicable if `use_queries_v2` is enabled.
Default: []
| +|
pushdown_deny_usernames.string
string
| | +|
region_qualifiers
array
| BigQuery regions to be scanned for bigquery jobs when using `use_queries_v2`. See [this](https://cloud.google.com/bigquery/docs/information-schema-jobs#scope_and_syntax) for details.
Default: ['region-us', 'region-eu']
| +|
region_qualifiers.string
string
| | +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_snapshot_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_snapshot_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
usage
BigQueryUsageConfig
| | +|
usage.apply_view_usage_to_tables
boolean
| Whether to apply view's usage to its base tables. If set to False, uses sql parser and applies usage to views / tables mentioned in the query. If set to True, usage is applied to base tables only.
Default: False
| +|
usage.bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
usage.end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
usage.format_sql_queries
boolean
| Whether to format sql queries
Default: False
| +|
usage.include_operational_stats
boolean
| Whether to display operational stats.
Default: True
| +|
usage.include_read_operational_stats
boolean
| Whether to report read operational stats. Experimental.
Default: False
| +|
usage.include_top_n_queries
boolean
| Whether to ingest the top_n_queries.
Default: True
| +|
usage.max_query_duration
string(duration)
| Correction to pad start_time and end_time with. For handling the case where the read happens within our time range but the query completion event is delayed and happens after the configured end time.
Default: PT15M
| +|
usage.start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
usage.top_n_queries
integer
| Number of top queries to save to each table.
Default: 10
| +|
usage.user_email_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
usage.user_email_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "BigQueryUsageConfig": { + "additionalProperties": false, + "properties": { + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "top_n_queries": { + "default": 10, + "description": "Number of top queries to save to each table.", + "exclusiveMinimum": 0, + "title": "Top N Queries", + "type": "integer" + }, + "user_email_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for user emails to filter in usage." + }, + "include_operational_stats": { + "default": true, + "description": "Whether to display operational stats.", + "title": "Include Operational Stats", + "type": "boolean" + }, + "include_read_operational_stats": { + "default": false, + "description": "Whether to report read operational stats. Experimental.", + "title": "Include Read Operational Stats", + "type": "boolean" + }, + "format_sql_queries": { + "default": false, + "description": "Whether to format sql queries", + "title": "Format Sql Queries", + "type": "boolean" + }, + "include_top_n_queries": { + "default": true, + "description": "Whether to ingest the top_n_queries.", + "title": "Include Top N Queries", + "type": "boolean" + }, + "max_query_duration": { + "default": "PT15M", + "description": "Correction to pad start_time and end_time with. For handling the case where the read happens within our time range but the query completion event is delayed and happens after the configured end time.", + "format": "duration", + "title": "Max Query Duration", + "type": "string" + }, + "apply_view_usage_to_tables": { + "default": false, + "description": "Whether to apply view's usage to its base tables. If set to False, uses sql parser and applies usage to views / tables mentioned in the query. If set to True, usage is applied to base tables only.", + "title": "Apply View Usage To Tables", + "type": "boolean" + } + }, + "title": "BigQueryUsageConfig", + "type": "object" + }, + "BucketDuration": { + "enum": [ + "DAY", + "HOUR" + ], + "title": "BucketDuration", + "type": "string" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "FolderTraversalMethod": { + "enum": [ + "ALL", + "MIN_MAX", + "MAX" + ], + "title": "FolderTraversalMethod", + "type": "string" + }, + "GCPCredential": { + "additionalProperties": false, + "properties": { + "project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Project id to set the credentials", + "title": "Project Id" + }, + "private_key_id": { + "description": "Private key id", + "title": "Private Key Id", + "type": "string" + }, + "private_key": { + "description": "Private key in a form of '-----BEGIN PRIVATE KEY-----\\nprivate-key\\n-----END PRIVATE KEY-----\\n'", + "format": "password", + "title": "Private Key", + "type": "string", + "writeOnly": true + }, + "client_email": { + "description": "Client email", + "title": "Client Email", + "type": "string" + }, + "client_id": { + "description": "Client Id", + "title": "Client Id", + "type": "string" + }, + "auth_uri": { + "default": "https://accounts.google.com/o/oauth2/auth", + "description": "Authentication uri", + "title": "Auth Uri", + "type": "string" + }, + "token_uri": { + "default": "https://oauth2.googleapis.com/token", + "description": "Token uri", + "title": "Token Uri", + "type": "string" + }, + "auth_provider_x509_cert_url": { + "default": "https://www.googleapis.com/oauth2/v1/certs", + "description": "Auth provider x509 certificate url", + "title": "Auth Provider X509 Cert Url", + "type": "string" + }, + "type": { + "default": "service_account", + "description": "Authentication type", + "title": "Type", + "type": "string" + }, + "client_x509_cert_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email", + "title": "Client X509 Cert Url" + } + }, + "required": [ + "private_key_id", + "private_key", + "client_email", + "client_id" + ], + "title": "GCPCredential", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "GcsLineageProviderConfig": { + "additionalProperties": false, + "description": "Any source that produces gcs lineage from/to Datasets should inherit this class.", + "properties": { + "path_specs": { + "default": [], + "description": "List of PathSpec. See below the details about PathSpec", + "items": { + "$ref": "#/$defs/PathSpec" + }, + "title": "Path Specs", + "type": "array" + }, + "strip_urls": { + "default": true, + "description": "Strip filename from gcs url. It only applies if path_specs are not specified.", + "title": "Strip Urls", + "type": "boolean" + }, + "ignore_non_path_spec_path": { + "default": false, + "description": "Ignore paths that are not match in path_specs. It only applies if path_specs are specified.", + "title": "Ignore Non Path Spec Path", + "type": "boolean" + } + }, + "title": "GcsLineageProviderConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "PathSpec": { + "additionalProperties": false, + "properties": { + "include": { + "description": "Path to table. Name variable `{table}` is used to mark the folder with dataset. In absence of `{table}`, file level dataset will be created. Check below examples for more details.", + "title": "Include", + "type": "string" + }, + "exclude": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": [], + "description": "list of paths in glob pattern which will be excluded while scanning for the datasets", + "title": "Exclude" + }, + "file_types": { + "default": [ + "csv", + "tsv", + "json", + "parquet", + "avro" + ], + "description": "Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.", + "items": { + "type": "string" + }, + "title": "File Types", + "type": "array" + }, + "default_extension": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.", + "title": "Default Extension" + }, + "table_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Display name of the dataset.Combination of named variables from include path and strings", + "title": "Table Name" + }, + "enable_compression": { + "default": true, + "description": "Enable or disable processing compressed files. Currently .gz and .bz files are supported.", + "title": "Enable Compression", + "type": "boolean" + }, + "sample_files": { + "default": true, + "description": "Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled", + "title": "Sample Files", + "type": "boolean" + }, + "allow_double_stars": { + "default": false, + "description": "Allow double stars in the include path. This can affect performance significantly if enabled", + "title": "Allow Double Stars", + "type": "boolean" + }, + "autodetect_partitions": { + "default": true, + "description": "Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`", + "title": "Autodetect Partitions", + "type": "boolean" + }, + "traversal_method": { + "$ref": "#/$defs/FolderTraversalMethod", + "default": "MAX", + "description": "Method to traverse the folder. ALL: Traverse all the folders, MIN_MAX: Traverse the folders by finding min and max value, MAX: Traverse the folder with max value" + }, + "include_hidden_folders": { + "default": false, + "description": "Include hidden folders in the traversal (folders starting with . or _", + "title": "Include Hidden Folders", + "type": "boolean" + }, + "tables_filter_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "The tables_filter_pattern configuration field uses regular expressions to filter the tables part of the Pathspec for ingestion, allowing fine-grained control over which tables are included or excluded based on specified patterns. The default setting allows all tables." + } + }, + "required": [ + "include" + ], + "title": "PathSpec", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "enable_stateful_profiling": { + "default": true, + "description": "Enable stateful profiling. This will store profiling timestamps per dataset after successful profiling. and will not run profiling again in subsequent run if table has not been updated. ", + "title": "Enable Stateful Profiling", + "type": "boolean" + }, + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "enable_stateful_time_window": { + "default": false, + "description": "Enable stateful time window tracking. This will store the time window after successful extraction and adjust the time window in subsequent runs to avoid reprocessing. NOTE: This is ONLY applicable when using queries v2 (use_queries_v2=True). This replaces enable_stateful_lineage_ingestion and enable_stateful_usage_ingestion for the queries v2 extraction path, since queries v2 extracts lineage, usage, operations, and queries together from a single audit log and uses a unified time window.", + "title": "Enable Stateful Time Window", + "type": "boolean" + }, + "enable_stateful_lineage_ingestion": { + "default": true, + "description": "Enable stateful lineage ingestion. This will store lineage window timestamps after successful lineage ingestion. and will not run lineage ingestion for same timestamps in subsequent run. NOTE: This only works with use_queries_v2=False (legacy extraction path). For queries v2, use enable_stateful_time_window instead.", + "title": "Enable Stateful Lineage Ingestion", + "type": "boolean" + }, + "enable_stateful_usage_ingestion": { + "default": true, + "description": "Enable stateful lineage ingestion. This will store usage window timestamps after successful usage ingestion. and will not run usage ingestion for same timestamps in subsequent run. NOTE: This only works with use_queries_v2=False (legacy extraction path). For queries v2, use enable_stateful_time_window instead.", + "title": "Enable Stateful Usage Ingestion", + "type": "boolean" + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "include_data_platform_instance": { + "default": false, + "description": "Whether to create a DataPlatformInstance aspect, equal to the BigQuery project id. If enabled, will cause redundancy in the browse path for BigQuery entities in the UI, because the project id is represented as the top-level container.", + "title": "Include Data Platform Instance", + "type": "boolean" + }, + "enable_legacy_sharded_table_support": { + "default": true, + "description": "Use the legacy sharded table urn suffix added.", + "title": "Enable Legacy Sharded Table Support", + "type": "boolean" + }, + "project_ids": { + "description": "Ingests specified project_ids. Use this property if you want to specify what projects to ingest or don't want to give project resourcemanager.projects.list to your service account. Overrides `project_id_pattern`.", + "items": { + "type": "string" + }, + "title": "Project Ids", + "type": "array" + }, + "project_labels": { + "description": "Ingests projects with the specified labels. Set value in the format of `key:value`. Use this property to define which projects to ingest basedon project-level labels. If project_ids or project_id is set, this configuration has no effect. The ingestion process filters projects by label first, and then applies the project_id_pattern.", + "items": { + "type": "string" + }, + "title": "Project Labels", + "type": "array" + }, + "project_id_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for project_id to filter in ingestion." + }, + "dataset_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for dataset to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'" + }, + "match_fully_qualified_names": { + "default": true, + "description": "[deprecated] Whether `dataset_pattern` is matched against fully qualified dataset name `.`.", + "title": "Match Fully Qualified Names", + "type": "boolean" + }, + "table_snapshot_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for table snapshots to filter in ingestion. Specify regex to match the entire snapshot name in database.schema.snapshot format. e.g. to match all snapshots starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "rate_limit": { + "default": false, + "description": "Should we rate limit requests made to API.", + "title": "Rate Limit", + "type": "boolean" + }, + "requests_per_min": { + "default": 60, + "description": "Used to control number of API calls made per min. Only used when `rate_limit` is set to `True`.", + "title": "Requests Per Min", + "type": "integer" + }, + "temp_table_dataset_prefix": { + "default": "_", + "description": "If you are creating temp tables in a dataset with a particular prefix you can use this config to set the prefix for the dataset. This is to support workflows from before bigquery's introduction of temp tables. By default we use `_` because of datasets that begin with an underscore are hidden by default https://cloud.google.com/bigquery/docs/datasets#dataset-naming.", + "title": "Temp Table Dataset Prefix", + "type": "string" + }, + "sharded_table_pattern": { + "default": "((.+\\D)[_$]?)?(\\d\\d\\d\\d(?:0[1-9]|1[0-2])(?:0[1-9]|[12][0-9]|3[01]))$", + "deprecated": true, + "description": "The regex pattern to match sharded tables and group as one table. This is a very low level config parameter, only change if you know what you are doing, ", + "title": "Sharded Table Pattern", + "type": "string" + }, + "credential": { + "anyOf": [ + { + "$ref": "#/$defs/GCPCredential" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BigQuery credential informations" + }, + "extra_client_options": { + "additionalProperties": true, + "default": {}, + "description": "Additional options to pass to google.cloud.logging_v2.client.Client.", + "title": "Extra Client Options", + "type": "object" + }, + "project_on_behalf": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[Advanced] The BigQuery project in which queries are executed. Will be passed when creating a job. If not passed, falls back to the project associated with the service account.", + "title": "Project On Behalf" + }, + "gcs_lineage_config": { + "$ref": "#/$defs/GcsLineageProviderConfig", + "default": { + "path_specs": [], + "strip_urls": true, + "ignore_non_path_spec_path": false + }, + "description": "Common config for gcs lineage generation" + }, + "include_schema_metadata": { + "default": true, + "description": "Whether to ingest the BigQuery schema, i.e. projects, schemas, tables, and views.", + "title": "Include Schema Metadata", + "type": "boolean" + }, + "usage": { + "$ref": "#/$defs/BigQueryUsageConfig", + "default": { + "bucket_duration": "DAY", + "end_time": "2026-03-30T15:25:38.606320Z", + "start_time": "2026-03-29T00:00:00Z", + "queries_character_limit": 24000, + "top_n_queries": 10, + "user_email_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "include_operational_stats": true, + "include_read_operational_stats": false, + "format_sql_queries": false, + "include_top_n_queries": true, + "max_query_duration": "PT15M", + "apply_view_usage_to_tables": false + }, + "description": "Usage related configs" + }, + "include_usage_statistics": { + "default": true, + "description": "Generate usage statistic", + "title": "Include Usage Statistics", + "type": "boolean" + }, + "capture_table_label_as_tag": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/AllowDenyPattern" + } + ], + "default": false, + "description": "Capture BigQuery table labels as DataHub tag", + "title": "Capture Table Label As Tag" + }, + "capture_view_label_as_tag": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/AllowDenyPattern" + } + ], + "default": false, + "description": "Capture BigQuery view labels as DataHub tag", + "title": "Capture View Label As Tag" + }, + "capture_dataset_label_as_tag": { + "anyOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/AllowDenyPattern" + } + ], + "default": false, + "description": "Capture BigQuery dataset labels as DataHub tag", + "title": "Capture Dataset Label As Tag" + }, + "include_table_constraints": { + "default": true, + "description": "Whether to ingest table constraints. If you know you don't use table constraints, you can disable it to save one extra query per dataset. In general it should be enabled", + "title": "Include Table Constraints", + "type": "boolean" + }, + "include_external_url": { + "default": true, + "description": "Whether to populate BigQuery Console url to Datasets/Tables", + "title": "Include External Url", + "type": "boolean" + }, + "include_table_snapshots": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether table snapshots should be ingested.", + "title": "Include Table Snapshots" + }, + "debug_include_full_payloads": { + "default": false, + "description": "Include full payload into events. It is only for debugging and internal use.", + "title": "Debug Include Full Payloads", + "type": "boolean" + }, + "number_of_datasets_process_in_batch_if_profiling_enabled": { + "default": 1000, + "description": "Number of partitioned table queried in batch when getting metadata. This is a low level config property which should be touched with care. This restriction is needed because we query partitions system view which throws error if we try to touch too many tables.", + "title": "Number Of Datasets Process In Batch If Profiling Enabled", + "type": "integer" + }, + "use_tables_list_query_v2": { + "default": false, + "description": "List tables using an improved query that extracts partitions and last modified timestamps more accurately. Requires the ability to read table data. Automatically enabled when profiling is enabled.", + "title": "Use Tables List Query V2", + "type": "boolean" + }, + "use_queries_v2": { + "default": true, + "description": "If enabled, uses the new queries extractor to extract queries from bigquery.", + "title": "Use Queries V2", + "type": "boolean" + }, + "include_queries": { + "default": true, + "description": "If enabled, generate query entities associated with lineage edges. Only applicable if `use_queries_v2` is enabled.", + "title": "Include Queries", + "type": "boolean" + }, + "include_query_usage_statistics": { + "default": true, + "description": "If enabled, generate query popularity statistics. Only applicable if `use_queries_v2` is enabled.", + "title": "Include Query Usage Statistics", + "type": "boolean" + }, + "column_limit": { + "default": 300, + "description": "Maximum number of columns to process in a table. This is a low level config property which should be touched with care. This restriction is needed because excessively wide tables can result in failure to ingest the schema.", + "title": "Column Limit", + "type": "integer" + }, + "lineage_use_sql_parser": { + "default": true, + "description": "Use sql parser to resolve view/table lineage.", + "title": "Lineage Use Sql Parser", + "type": "boolean" + }, + "lineage_sql_parser_use_raw_names": { + "default": false, + "description": "This parameter ignores the lowercase pattern stipulated in the SQLParser. NOTE: Ignored if lineage_use_sql_parser is False.", + "title": "Lineage Sql Parser Use Raw Names", + "type": "boolean" + }, + "extract_column_lineage": { + "default": false, + "description": "If enabled, generate column level lineage. Requires lineage_use_sql_parser to be enabled.", + "title": "Extract Column Lineage", + "type": "boolean" + }, + "extract_lineage_from_catalog": { + "default": false, + "description": "This flag enables the data lineage extraction from Data Lineage API exposed by Google Data Catalog. NOTE: This extractor can't build views lineage. It's recommended to enable the view's DDL parsing. Read the docs to have more information about: https://cloud.google.com/data-catalog/docs/concepts/about-data-lineage", + "title": "Extract Lineage From Catalog", + "type": "boolean" + }, + "extract_policy_tags_from_catalog": { + "default": false, + "description": "This flag enables the extraction of policy tags from the Google Data Catalog API. When enabled, the extractor will fetch policy tags associated with BigQuery table columns. For more information about policy tags and column-level security, refer to the documentation: https://cloud.google.com/bigquery/docs/column-level-security-intro", + "title": "Extract Policy Tags From Catalog", + "type": "boolean" + }, + "scheme": { + "default": "bigquery", + "title": "Scheme", + "type": "string" + }, + "log_page_size": { + "default": 1000, + "description": "The number of log item will be queried per page for lineage collection", + "exclusiveMinimum": 0, + "title": "Log Page Size", + "type": "integer" + }, + "include_table_lineage": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Option to enable/disable lineage generation. Is enabled by default.", + "title": "Include Table Lineage" + }, + "include_column_lineage_with_gcs": { + "default": true, + "description": "When enabled, column-level lineage will be extracted from the gcs.", + "title": "Include Column Lineage With Gcs", + "type": "boolean" + }, + "max_query_duration": { + "default": "PT15M", + "description": "Correction to pad start_time and end_time with. For handling the case where the read happens within our time range but the query completion event is delayed and happens after the configured end time.", + "format": "duration", + "title": "Max Query Duration", + "type": "string" + }, + "bigquery_audit_metadata_datasets": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of datasets that contain a table named cloudaudit_googleapis_com_data_access which contain BigQuery audit logs, specifically, those containing BigQueryAuditMetadata. It is recommended that the project of the dataset is also specified, for example, projectA.datasetB.", + "title": "Bigquery Audit Metadata Datasets" + }, + "use_exported_bigquery_audit_metadata": { + "default": false, + "description": "When configured, use BigQueryAuditMetadata in bigquery_audit_metadata_datasets to compute lineage information.", + "title": "Use Exported Bigquery Audit Metadata", + "type": "boolean" + }, + "use_date_sharded_audit_log_tables": { + "default": false, + "description": "Whether to read date sharded tables or time partitioned tables when extracting usage from exported audit logs.", + "title": "Use Date Sharded Audit Log Tables", + "type": "boolean" + }, + "upstream_lineage_in_report": { + "default": false, + "description": "Useful for debugging lineage information. Set to True to see the raw lineage created internally. Only works with legacy approach (`use_queries_v2: False`).", + "title": "Upstream Lineage In Report", + "type": "boolean" + }, + "convert_column_urns_to_lowercase": { + "default": false, + "description": "When enabled, converts column URNs to lowercase to ensure cross-platform compatibility.", + "title": "Convert Column Urns To Lowercase", + "type": "boolean" + }, + "exclude_empty_projects": { + "default": false, + "description": "Option to exclude empty projects from being ingested.", + "title": "Exclude Empty Projects", + "type": "boolean" + }, + "max_threads_dataset_parallelism": { + "default": 20, + "description": "Number of worker threads to use to parallelize BigQuery Dataset Metadata Extraction. Set to 1 to disable.", + "title": "Max Threads Dataset Parallelism", + "type": "integer" + }, + "region_qualifiers": { + "default": [ + "region-us", + "region-eu" + ], + "description": "BigQuery regions to be scanned for bigquery jobs when using `use_queries_v2`. See [this](https://cloud.google.com/bigquery/docs/information-schema-jobs#scope_and_syntax) for details.", + "items": { + "type": "string" + }, + "title": "Region Qualifiers", + "type": "array" + }, + "pushdown_deny_usernames": { + "default": [], + "description": "List of user email patterns using SQL LIKE syntax (e.g., 'bot_%', '%@%.iam.gserviceaccount.com') which will NOT be considered for lineage/usage/queries extraction. Uses case-insensitive LIKE for server-side filtering (e.g., 'bot_%' matches 'Bot_User@example.com'). This is primarily useful for improving performance by filtering out users with extremely high query volumes. Only applicable if `use_queries_v2` is enabled.", + "items": { + "type": "string" + }, + "title": "Pushdown Deny Usernames", + "type": "array" + }, + "pushdown_allow_usernames": { + "default": [], + "description": "List of user email patterns using SQL LIKE syntax (e.g., 'analyst_%@company.com') which WILL be considered for lineage/usage/queries extraction. Uses case-insensitive LIKE for server-side filtering (e.g., 'analyst_%' matches 'Analyst_John@company.com'). Only applicable if `use_queries_v2` is enabled. If not specified, all users not in deny list are included.", + "items": { + "type": "string" + }, + "title": "Pushdown Allow Usernames", + "type": "array" + } + }, + "title": "BigQueryV2Config", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Lineage and Usage Computation Details + +DataHub's BigQuery connector supports two approaches for extracting lineage and usage statistics: + +##### Modern Approach (Default): `use_queries_v2: true` + +**Recommended for most users** - Uses BigQuery's Information Schema for efficient metadata extraction. + +- **Data Source**: BigQuery Information Schema (`INFORMATION_SCHEMA.JOBS*` tables) +- **Features**: + - Advanced lineage extraction using SQL parsing + - Query entities with full query text + - Query popularity statistics and rankings + - Multi-region support via `region_qualifiers` + - Table and column-level usage statistics + - User filtering pushdown for performance (see [User Email Filtering Pushdown](#user-email-filtering-pushdown-performance-optimization) section below) +- **Requirements**: + - `bigquery.jobs.listAll` permission on target projects + - No additional Cloud Logging permissions needed + +**Configuration**: + +```yaml +source: + type: bigquery + config: + use_queries_v2: true # Default + include_queries: true # Enable query entities + include_query_usage_statistics: true # Query popularity stats + region_qualifiers: ["region-us", "region-eu"] # Multi-region support +``` + +##### User Email Filtering Pushdown (Performance Optimization) + +The `pushdown_deny_usernames` and `pushdown_allow_usernames` options push user filtering directly to BigQuery's SQL query, reducing data transfer and improving performance for large query volumes. + +**When to Use:** + +- You have large query volumes (>10k queries in your time window) +- You want to exclude high-volume service accounts or bots +- You want to reduce BigQuery data transfer costs +- You want to reduce overall DataHub ingestion time + +**Example Configuration:** + +```yaml +source: + type: bigquery + config: + use_queries_v2: true # Required for pushdown + pushdown_deny_usernames: + - "bot_%" + - "%@%.iam.gserviceaccount.com" # Exclude service accounts + pushdown_allow_usernames: + - "analyst_%@example.com" + - "data_%@example.com" +``` + +**Behavior:** + +- When patterns are configured: Filtering happens server-side with BigQuery SQL using case-insensitive `LIKE` +- When empty (default): No server-side filtering; use `usage.user_email_pattern` for client-side filtering +- Patterns use SQL LIKE syntax (`%` = any characters, `_` = single character) +- Matching is case-insensitive (e.g., `bot_%` matches `Bot_User@example.com`) +- If a user matches both allow AND deny patterns, deny takes precedence (user is excluded) + +**Prerequisites:** + +- `use_queries_v2: true` must be enabled (default) +- Patterns must be valid SQL LIKE patterns + +**Note:** These configs are independent from `usage.user_email_pattern`. The pushdown filters are applied at the SQL query level for performance, while `user_email_pattern` is applied client-side during processing. + +##### Legacy Approach: `use_queries_v2: false` + +**Use when you need specific legacy features** - Processes BigQuery audit logs for metadata extraction. + +- **Data Source**: BigQuery audit logs (two options below) +- **Features**: + - Basic table-level lineage and usage statistics + - `upstream_lineage_in_report` debugging feature + - Works with existing audit log exports + +**Two data source options**: + +##### Option 1: Google Cloud Logging API (Default) + +```yaml +source: + type: bigquery + config: + use_queries_v2: false + use_exported_bigquery_audit_metadata: false # Default +``` + +- **Requirements**: `logging.logEntries.list` and `logging.privateLogEntries.list` permissions +- **Limitations**: API rate limits, potential costs for large volumes + +##### Option 2: Pre-exported Audit Logs in BigQuery Tables + +```yaml +source: + type: bigquery + config: + use_queries_v2: false + use_exported_bigquery_audit_metadata: true + bigquery_audit_metadata_datasets: + - "my-project.audit_dataset" + - "another-project.audit_logs" +``` + +- **Requirements**: + - Pre-exported audit logs in BigQuery tables + - Tables must be named `cloudaudit_googleapis_com_data_access` + - Only protoPayloads with `type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata` are supported +- **Benefits**: No Cloud Logging API limits, better for large-scale ingestion +- **Setup**: Follow [BigQuery audit logs export guide](https://cloud.google.com/bigquery/docs/reference/auditlogs#defining_a_bigquery_log_sink_using_gcloud) +- **Note**: The `bigquery_audit_metadata_datasets` parameter accepts datasets in `$PROJECT.$DATASET` format, allowing lineage computation from multiple projects. + +#### Profiling Details + +:::note Profiling Permission Requirement + +When profiling is enabled, the `bigquery.tables.getData` permission is **required**. This is needed to access detailed table metadata including partition information. See the permissions section above for details. + +::: + +For performance reasons, we only profile the latest partition for partitioned tables and the latest shard for sharded tables. +You can set partition explicitly with `partition.partition_datetime` property if you want, though note that partition config will be applied to all partitioned tables. + +#### Caveats + +- For materialized views, lineage is dependent on logs being retained. If your GCP logging is retained for 30 days (default) and 30 days have passed since the creation of the materialized view we won't be able to get lineage for them. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.bigquery_v2.bigquery.BigqueryV2Source` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for BigQuery, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/business-glossary.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/business-glossary.md new file mode 100644 index 00000000..c61b9e8e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/business-glossary.md @@ -0,0 +1,433 @@ +--- +sidebar_position: 7 +title: Business Glossary +slug: /generated/ingestion/sources/business-glossary +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Business Glossary + +## Overview + +Business Glossary is a DataHub utility or metadata-focused integration. Learn more in the [official Business Glossary documentation](https://datahub.com/docs/). + +The DataHub integration for Business Glossary covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `datahub-business-glossary` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +Capability metadata is not explicitly declared for this module. Refer to module documentation and configuration sections below. + +### Overview + +The `datahub-business-glossary` module ingests metadata from Business Glossary into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[datahub-business-glossary]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: datahub-business-glossary + config: + # Coordinates + file: /path/to/business_glossary_yaml + enable_auto_id: true # recommended to set to true so datahub will auto-generate guids from your term names + +# sink configs if needed + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
file ✅
One of string, string(path)
| File path or URL to business glossary file to ingest. | +|
enable_auto_id
boolean
| Generate guid urns instead of a plaintext path urn with the node/term's hierarchy.
Default: False
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "additionalProperties": false, + "properties": { + "file": { + "anyOf": [ + { + "type": "string" + }, + { + "format": "path", + "type": "string" + } + ], + "description": "File path or URL to business glossary file to ingest.", + "title": "File" + }, + "enable_auto_id": { + "default": false, + "description": "Generate guid urns instead of a plaintext path urn with the node/term's hierarchy.", + "title": "Enable Auto Id", + "type": "boolean" + } + }, + "required": [ + "file" + ], + "title": "BusinessGlossarySourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Business Glossary File Format + +The business glossary source file should be a .yml file with the following top-level keys: + +**Glossary**: the top level keys of the business glossary file + +Example **Glossary**: + +```yaml +version: "1" # the version of business glossary file config the config conforms to. Currently the only version released is `1`. +source: DataHub # the source format of the terms. Currently only supports `DataHub` +owners: # owners contains two nested fields + users: # (optional) a list of user IDs + - njones + groups: # (optional) a list of group IDs + - logistics +url: "https://github.com/datahub-project/datahub/" # (optional) external url pointing to where the glossary is defined externally, if applicable +nodes: # list of child **GlossaryNode** objects. See **GlossaryNode** section below + ... +``` + +**GlossaryNode**: a container of **GlossaryNode** and **GlossaryTerm** objects + +Example **GlossaryNode**: + +```yaml +- name: "Shipping" # name of the node + id: "Shipping-Logistics" # (optional) custom identifier for the node + description: Provides terms related to the shipping domain # description of the node + owners: # (optional) owners contains 2 nested fields + users: # (optional) a list of user IDs + - njones + groups: # (optional) a list of group IDs + - logistics + nodes: # list of child **GlossaryNode** objects + ... + knowledge_links: # (optional) list of **KnowledgeCard** objects + - label: Wiki link for shipping + url: "https://en.wikipedia.org/wiki/Freight_transport" +``` + +**GlossaryTerm**: a term in your business glossary + +Example **GlossaryTerm**: + +```yaml +- name: "Full Address" # name of the term + id: "Full-Address-Details" # (optional) custom identifier for the term + description: A collection of information to give the location of a building or plot of land. # description of the term + owners: # (optional) owners contains 2 nested fields + users: # (optional) a list of user IDs + - njones + groups: # (optional) a list of group IDs + - logistics + term_source: "EXTERNAL" # one of `EXTERNAL` or `INTERNAL`. Whether the term is coming from an external glossary or one defined in your organization. + source_ref: FIBO # (optional) if external, what is the name of the source the glossary term is coming from? + source_url: "https://www.google.com" # (optional) if external, what is the url of the source definition? + inherits: # (optional) list of **GlossaryTerm** that this term inherits from + - Privacy.PII + contains: # (optional) a list of **GlossaryTerm** that this term contains + - Shipping.ZipCode + - Shipping.CountryCode + - Shipping.StreetAddress + custom_properties: # (optional) a map of key/value pairs of arbitrary custom properties + - is_used_for_compliance_tracking: "true" + knowledge_links: # (optional) a list of **KnowledgeCard** related to this term. These appear as links on the glossary node's page + - url: "https://en.wikipedia.org/wiki/Address" + label: Wiki link + domain: "urn:li:domain:Logistics" # (optional) domain name or domain urn +``` + +#### ID Management and URL Generation + +The business glossary provides two primary ways to manage term and node identifiers: + +1. **Custom IDs**: You can explicitly specify an ID for any term or node using the `id` field. This is recommended for terms that need stable, predictable identifiers: + + ```yaml + terms: + - name: "Response Time" + id: "support-response-time" # Explicit ID + description: "Target time to respond to customer inquiries" + ``` + +2. **Automatic ID Generation**: When no ID is specified, the system will generate one based on the `enable_auto_id` setting: + + - With `enable_auto_id: false` (default): + + - Node and term names are converted to URL-friendly format + - Spaces within names are replaced with hyphens + - Special characters are removed (except hyphens) + - Case is preserved + - Multiple hyphens are collapsed to single ones + - Path components (node/term hierarchy) are joined with periods + - Example: Node "Customer Support" with term "Response Time" → "Customer-Support.Response-Time" + + - With `enable_auto_id: true`: + - Generates GUID-based IDs + - Recommended for guaranteed uniqueness + - Required for terms with non-ASCII characters + +Here's how path-based ID generation works: + +```yaml +nodes: + - name: "Customer Support" # Node ID: Customer-Support + terms: + - name: "Response Time" # Term ID: Customer-Support.Response-Time + description: "Response SLA" + + - name: "First Reply" # Term ID: Customer-Support.First-Reply + description: "Initial response" + + - name: "Product Feedback" # Node ID: Product-Feedback + terms: + - name: "Response Time" # Term ID: Product-Feedback.Response-Time + description: "Feedback response" +``` + +**Important Notes**: + +- Periods (.) are used exclusively as path separators between nodes and terms +- Periods in term or node names themselves will be removed +- Each component of the path (node names, term names) is cleaned independently: + - Spaces to hyphens + - Special characters removed + - Case preserved +- The cleaned components are then joined with periods to form the full path +- Non-ASCII characters in any component trigger automatic GUID generation +- Once an ID is created (either manually or automatically), it cannot be easily changed +- All references to a term (in `inherits`, `contains`, etc.) must use its correct ID +- Moving terms in the hierarchy does NOT update their IDs: + - The ID retains its original path components even after moving + - This can lead to IDs that don't match the current location + - Consider using `enable_auto_id: true` if you plan to reorganize your glossary +- For terms that other terms will reference, consider using explicit IDs or enable auto_id + +Example of how different names are handled: + +```yaml +nodes: + - name: "Data Services" # Node ID: Data-Services + terms: + # Basic term name + - name: "Response Time" # Term ID: Data-Services.Response-Time + description: "SLA metrics" + + # Term name with special characters + - name: "API @ Response" # Term ID: Data-Services.API-Response + description: "API metrics" + + # Term with non-ASCII (triggers GUID) + - name: "パフォーマンス" # Term ID will be a 32-character GUID + description: "Performance" +``` + +To see how these all work together, check out this comprehensive example business glossary file below: + +```yaml +version: "1" +source: DataHub +owners: + users: + - mjames +url: "https://github.com/datahub-project/datahub/" +nodes: + - name: "Data Classification" + id: "Data-Classification" # Custom ID for stable references + description: A set of terms related to Data Classification + knowledge_links: + - label: Wiki link for classification + url: "https://en.wikipedia.org/wiki/Classification" + terms: + - name: "Sensitive Data" # Will generate: Data-Classification.Sensitive-Data + description: Sensitive Data + custom_properties: + is_confidential: "false" + - name: "Confidential Information" # Will generate: Data-Classification.Confidential-Information + description: Confidential Data + custom_properties: + is_confidential: "true" + - name: "Highly Confidential" # Will generate: Data-Classification.Highly-Confidential + description: Highly Confidential Data + custom_properties: + is_confidential: "true" + domain: Marketing + + - name: "Personal Information" + description: All terms related to personal information + owners: + users: + - mjames + terms: + - name: "Email" # Will generate: Personal-Information.Email + description: An individual's email address + inherits: + - Data-Classification.Confidential # References parent node path + owners: + groups: + - Trust and Safety + - name: "Address" # Will generate: Personal-Information.Address + description: A physical address + - name: "Gender" # Will generate: Personal-Information.Gender + description: The gender identity of the individual + inherits: + - Data-Classification.Sensitive # References parent node path + + - name: "Clients And Accounts" + description: Provides basic concepts such as account, account holder, account provider, relationship manager that are commonly used by financial services providers to describe customers and to determine counterparty identities + owners: + groups: + - finance + type: DATAOWNER + terms: + - name: "Account" # Will generate: Clients-And-Accounts.Account + description: Container for records associated with a business arrangement for regular transactions and services + term_source: "EXTERNAL" + source_ref: FIBO + source_url: "https://spec.edmcouncil.org/fibo/ontology/FBC/ProductsAndServices/ClientsAndAccounts/Account" + inherits: + - Data-Classification.Highly-Confidential # References parent node path + contains: + - Clients-And-Accounts.Balance # References term in same node + - name: "Balance" # Will generate: Clients-And-Accounts.Balance + description: Amount of money available or owed + term_source: "EXTERNAL" + source_ref: FIBO + source_url: "https://spec.edmcouncil.org/fibo/ontology/FBC/ProductsAndServices/ClientsAndAccounts/Balance" + + - name: "KPIs" + description: Common Business KPIs + terms: + - name: "CSAT %" # Will generate: KPIs.CSAT + description: Customer Satisfaction Score +``` + +#### Custom ID Specification + +Custom IDs can be specified in two ways, both of which are fully supported and acceptable: + +1. Just the ID portion (simpler approach): + +```yaml +terms: + - name: "Email" + id: "company-email" # Will become urn:li:glossaryTerm:company-email + description: "Company email address" +``` + +2. Full URN format: + +```yaml +terms: + - name: "Email" + id: "urn:li:glossaryTerm:company-email" + description: "Company email address" +``` + +Both methods are valid and will work correctly. The system will automatically handle the URN prefix if you specify just the ID portion. + +The same applies for nodes: + +```yaml +nodes: + - name: "Communications" + id: "internal-comms" # Will become urn:li:glossaryNode:internal-comms + description: "Internal communication methods" +``` + +Note: Once you select a custom ID, it cannot be easily changed. + +#### Compatibility + +Compatible with version 1 of business glossary format. The source will be evolved as newer versions of this format are published. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.metadata.business_glossary.BusinessGlossaryFileSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/metadata/business_glossary.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Business Glossary, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cassandra.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cassandra.md new file mode 100644 index 00000000..ef474754 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cassandra.md @@ -0,0 +1,726 @@ +--- +sidebar_position: 8 +title: Cassandra +slug: /generated/ingestion/sources/cassandra +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Cassandra + +## Overview + +Cassandra is a data platform used to store and query analytical or operational data. Learn more in the [official Cassandra documentation](https://cassandra.apache.org/). + +The DataHub integration for Cassandra covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `cassandra` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | + +### Overview + +The `cassandra` module ingests metadata from Cassandra into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +#### Setup + +Supports **DataStax Astra DB** and **Cassandra Enterprise Edition (EE)**. + +##### Steps to Get the Required Information + +1. **Set Up User Credentials**: + + - **For Astra DB**: + - Log in to your Astra DB Console. + - Navigate to **Organization Settings** > **Token Management**. + - Generate an **Application Token** with the required permissions for read access. + - Download the **Secure Connect Bundle** from the Astra DB Console. + - **For Cassandra EE**: + - Ensure you have a **username** and **password** with read access to the necessary keyspaces. + +2. **Permissions**: + + - The user or token must have `SELECT` permissions that allow it to: + - Access metadata in system keyspaces (e.g., `system_schema`) to retrieve information about keyspaces, tables, columns, and views. + - Perform `SELECT` operations on the data tables if data profiling is enabled. + +3. **Verify Database Access**: + - For Astra DB: Ensure the **Secure Connect Bundle** is used and configured correctly. + - For Cassandra Opensource: Ensure the **contact point** and **port** are accessible. + +:::caution + +When enabling profiling, make sure to set a limit on the number of rows to sample. Profiling large tables without a limit may lead to excessive resource consumption and slow performance. + +::: + +:::note + +For cloud configuration with Astra DB, it is necessary to specify the Secure Connect Bundle path in the configuration. For that reason, use the CLI to ingest metadata into DataHub. + +::: + + +### Install the Plugin +```shell +pip install 'acryl-datahub[cassandra]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "cassandra" + config: + # Credentials for on prem cassandra + contact_point: "localhost" + port: 9042 + username: "admin" + password: "password" + + # SSL Configuration (optional) + #ssl_ca_certs: "/path/to/ca-certificate.pem" + #ssl_certfile: "/path/to/client-certificate.pem" + #ssl_keyfile: "/path/to/client-private-key.pem" + #ssl_version: "TLS_CLIENT" # Options: TLS_CLIENT, TLSv1, TLSv1_1, TLSv1_2, TLSv1_3 + + # Or + # Credentials Astra Cloud + #cloud_config: + # secure_connect_bundle: "Path to Secure Connect Bundle (.zip)" + # token: "Application Token" + + # Optional Allow / Deny extraction of particular keyspaces. + keyspace_pattern: + allow: [".*"] + + # Optional Allow / Deny extraction of particular tables. + table_pattern: + allow: [".*"] + + # Optional + profiling: + enabled: true + profile_table_level_only: true + +sink: + # config sinks + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
contact_point
string
| Domain or IP address of the Cassandra instance (excluding port).
Default: localhost
| +|
password
One of string(password), null
| Password credential associated with the specified username.
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
port
integer
| Port number to connect to the Cassandra instance.
Default: 9042
| +|
ssl_ca_certs
One of string, null
| Path to the CA certificate file for SSL connections.
Default: None
| +|
ssl_certfile
One of string, null
| Path to the SSL certificate file for SSL connections.
Default: None
| +|
ssl_keyfile
One of string, null
| Path to the SSL key file for SSL connections.
Default: None
| +|
ssl_version
One of string, null
| SSL protocol version to use for connections. Options: TLS_CLIENT, TLSv1, TLSv1_1, TLSv1_2, TLSv1_3. Defaults to TLS_CLIENT.
Default: TLS_CLIENT
| +|
username
One of string, null
| Username credential with read access to the system_schema keyspace.
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
cloud_config
One of CassandraCloudConfig, null
| Configuration for cloud-based Cassandra, such as DataStax Astra DB.
Default: None
| +|
cloud_config.secure_connect_bundle ❓
string
| File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax Astra DB. | +|
cloud_config.token ❓
string(password)
| The Astra DB application token used for authentication. | +|
cloud_config.connect_timeout
integer
| Timeout in seconds for establishing new connections to Cassandra.
Default: 600
| +|
cloud_config.request_timeout
integer
| Timeout in seconds for individual Cassandra requests.
Default: 600
| +|
keyspace_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
keyspace_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingBaseConfig
| | +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Configuration for stateful ingestion and stale metadata removal.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "CassandraCloudConfig": { + "additionalProperties": false, + "description": "Configuration for connecting to DataStax Astra DB in the cloud.", + "properties": { + "token": { + "description": "The Astra DB application token used for authentication.", + "format": "password", + "title": "Token", + "type": "string", + "writeOnly": true + }, + "secure_connect_bundle": { + "description": "File path to the Secure Connect Bundle (.zip) used for a secure connection to DataStax Astra DB.", + "title": "Secure Connect Bundle", + "type": "string" + }, + "connect_timeout": { + "default": 600, + "description": "Timeout in seconds for establishing new connections to Cassandra.", + "title": "Connect Timeout", + "type": "integer" + }, + "request_timeout": { + "default": 600, + "description": "Timeout in seconds for individual Cassandra requests.", + "title": "Request Timeout", + "type": "integer" + } + }, + "required": [ + "token", + "secure_connect_bundle" + ], + "title": "CassandraCloudConfig", + "type": "object" + }, + "GEProfilingBaseConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + } + }, + "title": "GEProfilingBaseConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Configuration for connecting to a Cassandra or DataStax Astra DB source.", + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for stateful ingestion and stale metadata removal." + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "contact_point": { + "default": "localhost", + "description": "Domain or IP address of the Cassandra instance (excluding port).", + "title": "Contact Point", + "type": "string" + }, + "port": { + "default": 9042, + "description": "Port number to connect to the Cassandra instance.", + "title": "Port", + "type": "integer" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username credential with read access to the system_schema keyspace.", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Password credential associated with the specified username.", + "title": "Password" + }, + "cloud_config": { + "anyOf": [ + { + "$ref": "#/$defs/CassandraCloudConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for cloud-based Cassandra, such as DataStax Astra DB." + }, + "ssl_ca_certs": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to the CA certificate file for SSL connections.", + "title": "Ssl Ca Certs" + }, + "ssl_certfile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to the SSL certificate file for SSL connections.", + "title": "Ssl Certfile" + }, + "ssl_keyfile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to the SSL key file for SSL connections.", + "title": "Ssl Keyfile" + }, + "ssl_version": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "TLS_CLIENT", + "description": "SSL protocol version to use for connections. Options: TLS_CLIENT, TLSv1, TLSv1_1, TLSv1_2, TLSv1_3. Defaults to TLS_CLIENT.", + "title": "Ssl Version" + }, + "keyspace_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter keyspaces for ingestion." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter keyspaces.tables for ingestion." + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to profile" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingBaseConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50 + }, + "description": "Configuration for profiling" + } + }, + "title": "CassandraSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.cassandra.cassandra.CassandraSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/cassandra/cassandra.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Cassandra, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/clickhouse.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/clickhouse.md new file mode 100644 index 00000000..cdd4ff2d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/clickhouse.md @@ -0,0 +1,2556 @@ +--- +sidebar_position: 9 +title: ClickHouse +slug: /generated/ingestion/sources/clickhouse +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# ClickHouse +There are 2 sources that provide integration with ClickHouse + + + + + + + + + + +
Source ModuleDocumentation
+ +`clickhouse` + + + + + +Source that extracts tables, views, and dictionaries from ClickHouse via SQLAlchemy. + +Implementation notes: +- Extends TwoTierSQLAlchemySource (database.table hierarchy) +- Optionally parses system.query_log for usage and lineage +- Supports ClickHouse-specific objects (dictionaries, materialized views) +- Some column types not fully supported (*AggregateFunction, DateTime with timezone) + [Read more...](#module-clickhouse) + + +
+ +`clickhouse-usage` + + + + + +Source that extracts usage statistics from ClickHouse by analyzing system.query_log. + +Implementation notes: +- Queries system.query_log (or custom view via query_log_table config) for SELECT queries +- Filters out system tables, table functions, and internal queries +- Uses SQLAlchemy to connect to ClickHouse +- Parses tables and columns arrays from query log +- Aggregates usage by time bucket using BaseUsageConfig +- Only processes QueryFinish events with Select query_kind + [Read more...](#module-clickhouse-usage) + + +
+ + +## Overview + +ClickHouse is a data platform used to store and query analytical or operational data. Learn more in the [official ClickHouse documentation](https://clickhouse.com/). + +The DataHub integration for ClickHouse covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `clickhouse` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Enabled by default via `include_view_column_lineage`. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Dataset Usage | ✅ | Optionally enabled via `include_usage_statistics`. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_lineage`. Supported for types - View, Table. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `clickhouse` module ingests metadata from Clickhouse into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[clickhouse]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: clickhouse + config: + # Coordinates + host_port: localhost:9000 + + # Credentials + username: user + password: pass + + # Options + platform_instance: DatabaseNameToBeIngested + + include_views: True # whether to include views, defaults to True + include_tables: True # whether to include views, defaults to True + +sink: + # sink configs + +#--------------------------------------------------------------------------- +# For the HTTP interface: +#--------------------------------------------------------------------------- +source: + type: clickhouse + config: + host_port: localhost:8443 + uri_opts: + protocol: https + +#--------------------------------------------------------------------------- +# For the Native interface: +#--------------------------------------------------------------------------- + +source: + type: clickhouse + config: + host_port: localhost:9440 + scheme: clickhouse+native + uri_opts: + secure: True + +#------------------------------------------------------------------------ +# Example: using ingestion with configured SSL-TLS and uri_opts +# See https://clickhouse.com/docs/en/guides/sre/configuring-ssl +# ------------------------------------------------------------------------ +source: + type: clickhouse + config: + # Url form, prefered + sqlalchemy_uri: 'clickhouse+native://user:pass@localhost:9000/db?&ca_certs=ca.crt' + + # Non url form + username: user + password: pass + + host_port: localhost:9000 + + uri_opts: + secure: True + ca_certs: "ca.crt" + certfile: "clickhouse.crt" + keyfile: "clickhouse.key" + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
host_port
string
| ClickHouse host URL.
Default: localhost:8123
| +|
include_materialized_views
One of boolean, null
|
Default: True
| +|
include_query_log_lineage
boolean
| Whether to extract lineage from query_log (INSERT/CREATE queries). This complements the definition-based lineage from views/materialized views.
Default: False
| +|
include_query_log_operations
boolean
| Whether to show recent write activity on datasets. Displays INSERT, UPDATE, DELETE history in the dataset's activity tab.
Default: False
| +|
include_table_lineage
One of boolean, null
| Whether table lineage should be ingested.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_usage_statistics
boolean
| Whether to extract usage statistics from query_log. Tracks which tables and columns are queried by SELECT statements.
Default: False
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
string(password)
| password
Default:
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
platform_instance_map
One of string, null
| A holder for platform -> platform_instance mappings to generate correct dataset urns
Default: None
| +|
protocol
One of string, null
| [deprecated] Use uri_opts instead.
Default: None
| +|
secure
One of boolean, null
| [deprecated] Use uri_opts instead.
Default: None
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
top_n_queries
integer
| Number of top queries to save to each table for usage statistics.
Default: 10
| +|
uri_opts
map(str,string)
| | +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
query_log_deny_usernames
array
| List of ClickHouse usernames to exclude from query log extraction.
Default: []
| +|
query_log_deny_usernames.string
string
| | +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
temporary_tables_pattern
array
| Regex patterns for temporary tables to filter from lineage.
Default: ['^_.*', '.*\\.tmp_.*', '.*\\.temp_.*', '.*\\._inn...
| +|
temporary_tables_pattern.string
string
| | +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "BucketDuration": { + "enum": [ + "DAY", + "HOUR" + ], + "title": "BucketDuration", + "type": "string" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance_map": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A holder for platform -> platform_instance mappings to generate correct dataset urns", + "title": "Platform Instance Map" + }, + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "default": "", + "description": "password", + "format": "password", + "title": "Password", + "type": "string", + "writeOnly": true + }, + "host_port": { + "default": "localhost:8123", + "description": "ClickHouse host URL.", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for databases to filter in ingestion." + }, + "secure": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[deprecated] Use uri_opts instead.", + "title": "Secure" + }, + "protocol": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[deprecated] Use uri_opts instead.", + "title": "Protocol" + }, + "uri_opts": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "The part of the URI and it's used to provide additional configuration options or parameters for the database connection.", + "title": "Uri Opts", + "type": "object" + }, + "include_table_lineage": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether table lineage should be ingested.", + "title": "Include Table Lineage" + }, + "include_materialized_views": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "", + "title": "Include Materialized Views" + }, + "include_query_log_lineage": { + "default": false, + "description": "Whether to extract lineage from query_log (INSERT/CREATE queries). This complements the definition-based lineage from views/materialized views.", + "title": "Include Query Log Lineage", + "type": "boolean" + }, + "include_usage_statistics": { + "default": false, + "description": "Whether to extract usage statistics from query_log. Tracks which tables and columns are queried by SELECT statements.", + "title": "Include Usage Statistics", + "type": "boolean" + }, + "include_query_log_operations": { + "default": false, + "description": "Whether to show recent write activity on datasets. Displays INSERT, UPDATE, DELETE history in the dataset's activity tab.", + "title": "Include Query Log Operations", + "type": "boolean" + }, + "query_log_deny_usernames": { + "default": [], + "description": "List of ClickHouse usernames to exclude from query log extraction.", + "items": { + "type": "string" + }, + "title": "Query Log Deny Usernames", + "type": "array" + }, + "temporary_tables_pattern": { + "default": [ + "^_.*", + ".*\\.tmp_.*", + ".*\\.temp_.*", + ".*\\._inner.*" + ], + "description": "Regex patterns for temporary tables to filter from lineage.", + "items": { + "type": "string" + }, + "title": "Temporary Tables Pattern", + "type": "array" + }, + "top_n_queries": { + "default": 10, + "description": "Number of top queries to save to each table for usage statistics.", + "exclusiveMinimum": 0, + "title": "Top N Queries", + "type": "integer" + } + }, + "title": "ClickHouseConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Query Log Extraction + +Enable query-log based metadata extraction to augment definition-based lineage: + +- `include_query_log_lineage`: derive lineage from INSERT/CREATE queries in `system.query_log` +- `include_usage_statistics`: derive usage statistics from SELECT query activity + +This complements view/materialized-view lineage and improves operational usage visibility. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.clickhouse.ClickHouseSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/clickhouse.py) + + + +## Module `clickhouse-usage` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Dataset Usage | ✅ | Enabled by default to get usage stats. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | + +### Overview + +The `clickhouse-usage` module ingests metadata from Clickhouse into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin has the below functionalities: + +- For a specific dataset this plugin ingests the following statistics - + - top n queries. + - top users. + - usage of each column in the dataset. +- Aggregation of these statistics into buckets, by day or hour granularity. +- Usage information is computed by querying the `system.query_log` table. In case you have a cluster or need to apply additional + transformation/filters you can create a view and put to the `query_log_table` setting. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[clickhouse-usage]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: clickhouse-usage + config: + # Coordinates + host_port: db_host:port + platform_instance: dev_cluster + email_domain: acryl.io + + # Credentials + username: username + password: "password" + +sink: +# sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
email_domain ✅
string
| | +|
bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
format_sql_queries
boolean
| Whether to format sql queries
Default: False
| +|
host_port
string
| ClickHouse host URL.
Default: localhost:8123
| +|
include_materialized_views
One of boolean, null
|
Default: True
| +|
include_operational_stats
boolean
| Whether to display operational stats.
Default: True
| +|
include_query_log_lineage
boolean
| Whether to extract lineage from query_log (INSERT/CREATE queries). This complements the definition-based lineage from views/materialized views.
Default: False
| +|
include_query_log_operations
boolean
| Whether to show recent write activity on datasets. Displays INSERT, UPDATE, DELETE history in the dataset's activity tab.
Default: False
| +|
include_read_operational_stats
boolean
| Whether to report read operational stats. Experimental.
Default: False
| +|
include_table_lineage
One of boolean, null
| Whether table lineage should be ingested.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_top_n_queries
boolean
| Whether to ingest the top_n_queries.
Default: True
| +|
include_usage_statistics
boolean
| Whether to extract usage statistics from query_log. Tracks which tables and columns are queried by SELECT statements.
Default: False
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
|
Default: {}
| +|
password
string(password)
| password
Default:
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
platform_instance_map
One of string, null
| A holder for platform -> platform_instance mappings to generate correct dataset urns
Default: None
| +|
protocol
One of string, null
| [deprecated] Use uri_opts instead.
Default: None
| +|
query_log_table
string
|
Default: system.query_log
| +|
secure
One of boolean, null
| [deprecated] Use uri_opts instead.
Default: None
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
top_n_queries
integer
| Number of top queries to save to each table for usage statistics.
Default: 10
| +|
uri_opts
map(str,string)
| | +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
query_log_deny_usernames
array
| List of ClickHouse usernames to exclude from query log extraction.
Default: []
| +|
query_log_deny_usernames.string
string
| | +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
temporary_tables_pattern
array
| Regex patterns for temporary tables to filter from lineage.
Default: ['^_.*', '.*\\.tmp_.*', '.*\\.temp_.*', '.*\\._inn...
| +|
temporary_tables_pattern.string
string
| | +|
user_email_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
user_email_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "BucketDuration": { + "enum": [ + "DAY", + "HOUR" + ], + "title": "BucketDuration", + "type": "string" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance_map": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A holder for platform -> platform_instance mappings to generate correct dataset urns", + "title": "Platform Instance Map" + }, + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "top_n_queries": { + "default": 10, + "description": "Number of top queries to save to each table for usage statistics.", + "exclusiveMinimum": 0, + "title": "Top N Queries", + "type": "integer" + }, + "user_email_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for user emails to filter in usage." + }, + "include_operational_stats": { + "default": true, + "description": "Whether to display operational stats.", + "title": "Include Operational Stats", + "type": "boolean" + }, + "include_read_operational_stats": { + "default": false, + "description": "Whether to report read operational stats. Experimental.", + "title": "Include Read Operational Stats", + "type": "boolean" + }, + "format_sql_queries": { + "default": false, + "description": "Whether to format sql queries", + "title": "Format Sql Queries", + "type": "boolean" + }, + "include_top_n_queries": { + "default": true, + "description": "Whether to ingest the top_n_queries.", + "title": "Include Top N Queries", + "type": "boolean" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "default": {}, + "description": "", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "default": "", + "description": "password", + "format": "password", + "title": "Password", + "type": "string", + "writeOnly": true + }, + "host_port": { + "default": "localhost:8123", + "description": "ClickHouse host URL.", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for databases to filter in ingestion." + }, + "secure": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[deprecated] Use uri_opts instead.", + "title": "Secure" + }, + "protocol": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[deprecated] Use uri_opts instead.", + "title": "Protocol" + }, + "uri_opts": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "The part of the URI and it's used to provide additional configuration options or parameters for the database connection.", + "title": "Uri Opts", + "type": "object" + }, + "include_table_lineage": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether table lineage should be ingested.", + "title": "Include Table Lineage" + }, + "include_materialized_views": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "", + "title": "Include Materialized Views" + }, + "include_query_log_lineage": { + "default": false, + "description": "Whether to extract lineage from query_log (INSERT/CREATE queries). This complements the definition-based lineage from views/materialized views.", + "title": "Include Query Log Lineage", + "type": "boolean" + }, + "include_usage_statistics": { + "default": false, + "description": "Whether to extract usage statistics from query_log. Tracks which tables and columns are queried by SELECT statements.", + "title": "Include Usage Statistics", + "type": "boolean" + }, + "include_query_log_operations": { + "default": false, + "description": "Whether to show recent write activity on datasets. Displays INSERT, UPDATE, DELETE history in the dataset's activity tab.", + "title": "Include Query Log Operations", + "type": "boolean" + }, + "query_log_deny_usernames": { + "default": [], + "description": "List of ClickHouse usernames to exclude from query log extraction.", + "items": { + "type": "string" + }, + "title": "Query Log Deny Usernames", + "type": "array" + }, + "temporary_tables_pattern": { + "default": [ + "^_.*", + ".*\\.tmp_.*", + ".*\\.temp_.*", + ".*\\._inner.*" + ], + "description": "Regex patterns for temporary tables to filter from lineage.", + "items": { + "type": "string" + }, + "title": "Temporary Tables Pattern", + "type": "array" + }, + "email_domain": { + "description": "", + "title": "Email Domain", + "type": "string" + }, + "query_log_table": { + "default": "system.query_log", + "title": "Query Log Table", + "type": "string" + } + }, + "required": [ + "email_domain" + ], + "title": "ClickHouseUsageConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Usage Extraction Scope + +The `clickhouse-usage` module focuses on usage extraction from query logs. Use it when you only need usage telemetry without full schema/lineage extraction from the main `clickhouse` module. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.usage.clickhouse_usage.ClickHouseUsageSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/usage/clickhouse_usage.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for ClickHouse, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cockroachdb.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cockroachdb.md new file mode 100644 index 00000000..b6fcdd33 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/cockroachdb.md @@ -0,0 +1,1452 @@ +--- +sidebar_position: 10 +title: CockroachDB +slug: /generated/ingestion/sources/cockroachdb +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# CockroachDB + +## Overview + +Cockroachdb is a data platform used to store and query analytical or operational data. Learn more in the [official Cockroachdb documentation](https://www.cockroachlabs.com/product/cockroachdb/). + +The DataHub integration for Cockroachdb covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `cockroachdb` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_column_lineage`. Supported for types - View. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_lineage`. Supported for types - View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `cockroachdb` module ingests metadata from Cockroachdb into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[cockroachdb]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: cockroachdb + config: + # Coordinates + host_port: localhost:26257 + database: DemoDatabase + + # Credentials + username: user + password: pass + + # Optional: SSL configuration. + # options: + # connect_args: + # sslcert: "<>" + # sslkey: "<>" + # sslrootcert: "<>" + # sslmode: "verify-full" + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
host_port ✅
string
| host URL | +|
auth_mode
Enum
| One of: "PASSWORD", "AWS_IAM" | +|
bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog). If set to Null, all databases will be considered for ingestion.
Default: None
| +|
end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
format_sql_queries
boolean
| Whether to format sql queries
Default: False
| +|
include_operational_stats
boolean
| Whether to display operational stats.
Default: True
| +|
include_query_lineage
boolean
| Enable query-based lineage extraction from pg_stat_statements. Requires the pg_stat_statements extension to be installed and enabled. See documentation for setup instructions.
Default: False
| +|
include_read_operational_stats
boolean
| Whether to report read operational stats. Experimental.
Default: False
| +|
include_stored_procedures
boolean
| Include ingest of stored procedures.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_top_n_queries
boolean
| Whether to ingest the top_n_queries.
Default: True
| +|
include_usage_statistics
boolean
| Generate usage statistics from query history. Requires include_query_lineage to be enabled. Collects metrics like unique user counts, query frequencies, and column access patterns. Statistics appear in DataHub UI under the Dataset Profile > Usage tab.
Default: False
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
initial_database
string
| Initial database used to query for the list of databases, when ingesting multiple databases. Note: this is not used if `database` or `sqlalchemy_uri` are provided.
Default: postgres
| +|
max_queries_to_extract
integer
| Maximum number of queries to extract from pg_stat_statements for lineage analysis. Queries are prioritized by execution time and frequency.
Default: 1000
| +|
min_query_calls
integer
| Minimum number of executions required for a query to be included. Set higher to focus on frequently-used queries.
Default: 1
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
top_n_queries
integer
| Number of top queries to save to each table.
Default: 10
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
aws_config
AwsConnectionConfig
| Common AWS credentials config.

Currently used by:
- Glue source
- SageMaker source
- dbt source | +|
aws_config.aws_access_key_id
One of string, null
| AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.aws_advanced_config
object
| Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html). | +|
aws_config.aws_endpoint_url
One of string, null
| The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.
Default: None
| +|
aws_config.aws_profile
One of string, null
| The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.
Default: None
| +|
aws_config.aws_proxy
One of string, null
| A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.
Default: None
| +|
aws_config.aws_region
One of string, null
| AWS region code.
Default: None
| +|
aws_config.aws_retry_mode
Enum
| One of: "legacy", "standard", "adaptive"
Default: standard
| +|
aws_config.aws_retry_num
integer
| Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.
Default: 5
| +|
aws_config.aws_secret_access_key
One of string(password), null
| AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.aws_session_token
One of string(password), null
| AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.read_timeout
number
| The timeout for reading from the connection (in seconds).
Default: 60
| +|
aws_config.aws_role
One of string, array, null
| AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).
Default: None
| +|
aws_config.aws_role.union
One of string, AwsAssumeRoleConfig
| | +|
aws_config.aws_role.union.RoleArn ❓
string
| ARN of the role to assume. | +|
aws_config.aws_role.union.ExternalId
One of string, null
| External ID to use when assuming the role.
Default: None
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
procedure_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
procedure_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
query_exclude_patterns
One of array, null
| SQL LIKE patterns to exclude from query extraction. Example: ['%pg_catalog%', '%temp_%'] to exclude catalog and temp tables.
Default: None
| +|
query_exclude_patterns.string
string
| | +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
user_email_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
user_email_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AwsAssumeRoleConfig": { + "additionalProperties": true, + "properties": { + "RoleArn": { + "description": "ARN of the role to assume.", + "title": "Rolearn", + "type": "string" + }, + "ExternalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "External ID to use when assuming the role.", + "title": "Externalid" + } + }, + "required": [ + "RoleArn" + ], + "title": "AwsAssumeRoleConfig", + "type": "object" + }, + "AwsConnectionConfig": { + "additionalProperties": false, + "description": "Common AWS credentials config.\n\nCurrently used by:\n - Glue source\n - SageMaker source\n - dbt source", + "properties": { + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Session Token" + }, + "aws_role": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/AwsAssumeRoleConfig" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).", + "title": "Aws Role" + }, + "aws_profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.", + "title": "Aws Profile" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region code.", + "title": "Aws Region" + }, + "aws_endpoint_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.", + "title": "Aws Endpoint Url" + }, + "aws_proxy": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", + "title": "Aws Proxy" + }, + "aws_retry_num": { + "default": 5, + "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "title": "Aws Retry Num", + "type": "integer" + }, + "aws_retry_mode": { + "default": "standard", + "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "enum": [ + "legacy", + "standard", + "adaptive" + ], + "title": "Aws Retry Mode", + "type": "string" + }, + "read_timeout": { + "default": 60, + "description": "The timeout for reading from the connection (in seconds).", + "title": "Read Timeout", + "type": "number" + }, + "aws_advanced_config": { + "additionalProperties": true, + "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).", + "title": "Aws Advanced Config", + "type": "object" + } + }, + "title": "AwsConnectionConfig", + "type": "object" + }, + "BucketDuration": { + "enum": [ + "DAY", + "HOUR" + ], + "title": "BucketDuration", + "type": "string" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "PostgresAuthMode": { + "description": "Authentication mode for PostgreSQL connection.", + "enum": [ + "PASSWORD", + "AWS_IAM" + ], + "title": "PostgresAuthMode", + "type": "string" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "top_n_queries": { + "default": 10, + "description": "Number of top queries to save to each table.", + "exclusiveMinimum": 0, + "title": "Top N Queries", + "type": "integer" + }, + "user_email_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for user emails to filter in usage." + }, + "include_operational_stats": { + "default": true, + "description": "Whether to display operational stats.", + "title": "Include Operational Stats", + "type": "boolean" + }, + "include_read_operational_stats": { + "default": false, + "description": "Whether to report read operational stats. Experimental.", + "title": "Include Read Operational Stats", + "type": "boolean" + }, + "format_sql_queries": { + "default": false, + "description": "Whether to format sql queries", + "title": "Format Sql Queries", + "type": "boolean" + }, + "include_top_n_queries": { + "default": true, + "description": "Whether to ingest the top_n_queries.", + "title": "Include Top N Queries", + "type": "boolean" + }, + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [ + "information_schema", + "crdb_internal" + ], + "ignoreCase": true + } + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "description": "host URL", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog). If set to Null, all databases will be considered for ingestion.", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "auth_mode": { + "$ref": "#/$defs/PostgresAuthMode", + "default": "PASSWORD", + "description": "Authentication mode to use for the PostgreSQL connection. Options are 'PASSWORD' (default) for standard username/password authentication, or 'AWS_IAM' for AWS RDS IAM authentication." + }, + "aws_config": { + "$ref": "#/$defs/AwsConnectionConfig", + "description": "AWS configuration for RDS IAM authentication (only used when auth_mode is AWS_IAM). Provides full control over AWS credentials, region, profiles, role assumption, retry logic, and proxy settings. If not explicitly configured, boto3 will automatically use the default credential chain and region from environment variables (AWS_DEFAULT_REGION, AWS_REGION), AWS config files (~/.aws/config), or IAM role metadata." + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for databases to filter in ingestion. Note: this is not used if `database` or `sqlalchemy_uri` are provided." + }, + "initial_database": { + "default": "postgres", + "description": "Initial database used to query for the list of databases, when ingesting multiple databases. Note: this is not used if `database` or `sqlalchemy_uri` are provided.", + "title": "Initial Database", + "type": "string" + }, + "include_stored_procedures": { + "default": true, + "description": "Include ingest of stored procedures.", + "title": "Include Stored Procedures", + "type": "boolean" + }, + "procedure_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for stored procedures to filter in ingestion.Specify regex to match the entire procedure name in database.schema.procedure_name format. e.g. to match all procedures starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "include_query_lineage": { + "default": false, + "description": "Enable query-based lineage extraction from pg_stat_statements. Requires the pg_stat_statements extension to be installed and enabled. See documentation for setup instructions.", + "title": "Include Query Lineage", + "type": "boolean" + }, + "max_queries_to_extract": { + "default": 1000, + "description": "Maximum number of queries to extract from pg_stat_statements for lineage analysis. Queries are prioritized by execution time and frequency.", + "title": "Max Queries To Extract", + "type": "integer" + }, + "min_query_calls": { + "default": 1, + "description": "Minimum number of executions required for a query to be included. Set higher to focus on frequently-used queries.", + "title": "Min Query Calls", + "type": "integer" + }, + "query_exclude_patterns": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "SQL LIKE patterns to exclude from query extraction. Example: ['%pg_catalog%', '%temp_%'] to exclude catalog and temp tables.", + "title": "Query Exclude Patterns" + }, + "include_usage_statistics": { + "default": false, + "description": "Generate usage statistics from query history. Requires include_query_lineage to be enabled. Collects metrics like unique user counts, query frequencies, and column access patterns. Statistics appear in DataHub UI under the Dataset Profile > Usage tab.", + "title": "Include Usage Statistics", + "type": "boolean" + } + }, + "required": [ + "host_port" + ], + "title": "CockroachDBConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.cockroachdb.CockroachDBSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/cockroachdb.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for CockroachDB, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/confluence.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/confluence.md new file mode 100644 index 00000000..e6f4ccfe --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/confluence.md @@ -0,0 +1,1771 @@ +--- +sidebar_position: 11 +title: Confluence +slug: /generated/ingestion/sources/confluence +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Confluence + +## Overview + +Confluence is a documentation or collaboration platform. Learn more in the [official Confluence documentation](https://www.atlassian.com/software/confluence). + +The DataHub integration for Confluence covers document/workspace entities and hierarchy context for knowledge assets. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +:::warning Not Supported with Remote Executor +This source is not supported with the Remote Executor in DataHub Cloud. It must be run using a self-hosted ingestion setup. +::: + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `confluence` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +:::caution Not Supported with Remote Executor +This source is available as a private beta feature on DataHub Cloud. Note that running the connector using the Remote Executor is not yet supported. +::: + +The Confluence source ingests pages and spaces from Confluence workspaces (Cloud or Data Center) as DataHub Document entities with optional semantic embeddings for semantic search. + +#### Key Features + +##### 1. Content Extraction + +- **Page Content**: Full text extraction from Confluence pages including all content types +- **Space Discovery**: Automatic discovery of all pages within specified spaces +- **Hierarchical Structure**: Maintains parent-child relationships between pages +- **Metadata Extraction**: Captures creation/modification timestamps, authors, labels, and custom properties + +##### 2. Hierarchical Relationships + +- **Parent-Child Links**: Preserves Confluence page hierarchy in DataHub +- **Recursive Discovery**: Recursively discovers nested pages starting from root pages or entire spaces +- **Space Organization**: Maintains space context as custom properties +- **Flexible Navigation**: Browse documentation structure in DataHub UI + +##### 3. Embedding Generation + +Optional semantic search support with sensible defaults: + +- **Supported providers**: Cohere (API key), AWS Bedrock (IAM roles) +- **Automatic chunking**: Documents are automatically chunked for optimal embedding generation +- **Automatic deduplication**: Prevents duplicate chunk embeddings + +See [Semantic Search Configuration](../../../how-to/semantic-search-configuration.md) for detailed setup and advanced options. + +##### 4. Stateful Ingestion + +Supports smart incremental updates via stateful ingestion: + +- **Content Change Detection**: Only reprocesses documents when content or embeddings config changes +- **Deletion Detection**: Automatically removes stale entities from DataHub +- **Flexible Discovery**: Ingest entire spaces, specific pages, or page trees +- **State Persistence**: Maintains processing state between runs to skip unchanged documents + +#### Related Documentation + +- [Confluence Cloud REST API](https://developer.atlassian.com/cloud/confluence/rest/v1/intro/) +- [Confluence Data Center REST API](https://docs.atlassian.com/ConfluenceServer/rest/) +- [Semantic Search Configuration](../../../how-to/semantic-search-configuration.md) +- [DataHub Document Ingestion](https://datahubproject.io/docs/generated/ingestion/sources/) + +### Prerequisites + +#### 1. Confluence API Access + +##### For Confluence Cloud + +Create an API token: + +1. Go to https://id.atlassian.com/manage-profile/security/api-tokens +2. Click **"Create API token"** +3. Give it a name (e.g., "DataHub Integration") +4. Copy the token (you won't be able to see it again) + +You'll need: + +- **Base URL**: Your Confluence Cloud URL (e.g., `https://your-domain.atlassian.net/wiki`) +- **Username**: Your Atlassian account email +- **API Token**: The token you just created + +##### For Confluence Data Center / Server + +Create a Personal Access Token: + +1. Go to your Confluence → Profile → Personal Access Tokens +2. Click **"Create token"** +3. Give it a name and set expiration +4. Copy the token + +You'll need: + +- **Base URL**: Your Confluence server URL (e.g., `https://confluence.company.com`) +- **Personal Access Token**: The token you created + +**Note**: For Data Center, you can also use username/password, but Personal Access Tokens are recommended. + +#### 2. Required Permissions + +The API credentials must have: + +- **Read access** to all spaces and pages you want to ingest +- For Cloud: User must be added to spaces or have site-wide read access +- For Data Center: User must have "View" permissions on spaces + +#### 3. Embedding Provider (Optional) + +If you want semantic search capabilities, configure an embedding provider in your DataHub instance. + +Supported providers include Cohere (API key) and AWS Bedrock (IAM roles). The connector will use sensible defaults for chunking and embedding configuration. + +See [Semantic Search Configuration](../../../how-to/semantic-search-configuration.md) for detailed provider setup and configuration options. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[confluence]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: confluence + config: + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@example.com" + api_token: "${CONFLUENCE_API_TOKEN}" + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
url ✅
string
| Base URL of your Confluence instance. Examples: 'https://your-domain.atlassian.net/wiki' (Cloud) or 'https://confluence.your-company.com' (Data Center) | +|
api_token
One of string(password), null
| API token for Confluence Cloud authentication. Generate at: https://id.atlassian.com/manage-profile/security/api-tokens
Default: None
| +|
cloud
boolean
| Whether this is a Confluence Cloud instance (True) or Data Center/Server (False).
Default: True
| +|
max_documents
integer
| Maximum number of documents to process per ingestion run. The job will stop and fail with an error once this limit is reached. Set to 0 or -1 to disable the limit.
Default: 10000
| +|
max_pages_per_space
integer
| Maximum number of pages to ingest per space.
Default: 1000
| +|
max_spaces
integer
| Maximum number of spaces to ingest when auto-discovering (applies when urls is not set).
Default: 100
| +|
personal_access_token
One of string(password), null
| Personal Access Token for Confluence Data Center authentication. Generate from: User Profile > Settings > Personal Access Tokens
Default: None
| +|
platform_instance
One of string, null
| Optional human-readable identifier for this Confluence instance (e.g., 'mycompany-prod', 'team-a-confluence'). If not provided, automatically generated by hashing the base URL, which guarantees global uniqueness across all Confluence installations (both Cloud and Data Center). Use explicit values for more readable URNs, but auto-generated hashes are perfectly fine and require no manual configuration.
Default: None
| +|
recursive
boolean
| Whether to recursively fetch child pages (applies to page URLs only).
Default: True
| +|
username
One of string, null
| Username for Confluence Cloud authentication (required for Cloud).
Default: None
| +|
advanced
AdvancedConfig
| Advanced configuration options. | +|
advanced.continue_on_failure
boolean
|
Default: True
| +|
advanced.max_errors
integer
|
Default: 10
| +|
advanced.output_format
Enum
| One of: "json", "xml"
Default: json
| +|
advanced.preserve_outputs
boolean
|
Default: False
| +|
advanced.raise_on_error
boolean
|
Default: False
| +|
advanced.work_dir
string
|
Default: /tmp/unstructured_datahub
| +|
advanced.cache
CacheConfig
| Cache configuration. | +|
advanced.cache.cache_dir
string
|
Default: ~/.cache/unstructured_datahub
| +|
advanced.cache.enabled
boolean
|
Default: True
| +|
advanced.cache.ttl
integer
| Cache TTL in seconds
Default: 86400
| +|
advanced.retry
RetryConfig
| Retry configuration. | +|
advanced.retry.backoff_factor
integer
|
Default: 2
| +|
advanced.retry.enabled
boolean
|
Default: True
| +|
advanced.retry.max_attempts
integer
|
Default: 3
| +|
advanced.retry.retry_on_timeout
boolean
|
Default: True
| +|
chunking
ChunkingConfig
| Chunking strategy configuration. | +|
chunking.combine_text_under_n_chars
integer
| Combine chunks smaller than this size
Default: 100
| +|
chunking.max_characters
integer
| Maximum characters per chunk
Default: 500
| +|
chunking.overlap
integer
| Character overlap between chunks
Default: 0
| +|
chunking.strategy
Enum
| One of: "basic", "by_title"
Default: by_title
| +|
document_mapping
DocumentMappingConfig
| Document entity mapping configuration. | +|
document_mapping.id_pattern
string
| Pattern for generating document IDs
Default: {source_type}-{directory}-{basename}
| +|
document_mapping.status
Enum
| One of: "PUBLISHED", "UNPUBLISHED"
Default: PUBLISHED
| +|
document_mapping.id_normalization
IdNormalizationConfig
| Document ID normalization rules. | +|
document_mapping.id_normalization.lowercase
boolean
| Convert to lowercase
Default: True
| +|
document_mapping.id_normalization.max_length
integer
| Maximum ID length
Default: 200
| +|
document_mapping.id_normalization.remove_special_chars
boolean
| Remove special characters except _ and -
Default: True
| +|
document_mapping.id_normalization.replace_spaces_with
string
| Replace spaces with this character
Default: -
| +|
document_mapping.source
SourceConfig
| Document source configuration. | +|
document_mapping.source.include_external_id
boolean
| Include external ID in DocumentSource
Default: True
| +|
document_mapping.source.include_external_url
boolean
| Include external URL in DocumentSource
Default: True
| +|
document_mapping.source.type
Enum
| One of: "NATIVE", "EXTERNAL"
Default: EXTERNAL
| +|
document_mapping.title
TitleExtractionConfig
| Title extraction configuration. | +|
document_mapping.title.extract_from_content
boolean
| Try to extract title from document content
Default: True
| +|
document_mapping.title.fallback_to_filename
boolean
| Use filename as title if not found in content
Default: True
| +|
document_mapping.title.max_length
integer
| Maximum title length
Default: 500
| +|
embedding
EmbeddingConfig
| Embedding generation configuration.

Default behavior: Fetches configuration from DataHub server automatically.
Override behavior: Validates local config against server when explicitly set. | +|
embedding.allow_local_embedding_config
boolean
| BREAK-GLASS: Allow local config without server validation. NOT RECOMMENDED - may break semantic search.
Default: False
| +|
embedding.api_key
One of string(password), null
| API key for Cohere (not needed for Bedrock with IAM roles)
Default: None
| +|
embedding.aws_region
One of string, null
| AWS region for Bedrock. If not set, loads from server.
Default: None
| +|
embedding.batch_size
integer
| Batch size for embedding API calls
Default: 25
| +|
embedding.documents_per_minute
integer
| Maximum number of documents to embed per minute when rate_limit is enabled.
Default: 300
| +|
embedding.input_type
One of string, null
| Input type for Cohere embeddings
Default: search_document
| +|
embedding.model
One of string, null
| Model name. If not set, loads from server.
Default: None
| +|
embedding.model_embedding_key
One of string, null
| Storage key for embeddings (e.g., 'cohere_embed_v3'). Required if overriding server config. If not set, loads from server.
Default: None
| +|
embedding.provider
One of Enum, null
| Embedding provider (bedrock uses AWS, cohere/openai use API key). If not set, loads from server.
Default: None
| +|
embedding.rate_limit
boolean
| Enable rate limiting for embedding API calls.
Default: True
| +|
filtering
FilteringConfig
| File filtering configuration. | +|
filtering.max_file_size
One of integer, null
| Maximum file size in bytes
Default: None
| +|
filtering.min_file_size
One of integer, null
| Minimum file size in bytes
Default: None
| +|
filtering.min_text_length
integer
| Minimum text length in characters
Default: 50
| +|
filtering.modified_after
One of string, null
| Only files modified after this date (ISO format)
Default: None
| +|
filtering.modified_before
One of string, null
| Only files modified before this date (ISO format)
Default: None
| +|
filtering.skip_empty_documents
boolean
| Skip documents with no text content
Default: True
| +|
filtering.exclude_patterns
array
| Glob patterns to exclude | +|
filtering.exclude_patterns.string
string
| | +|
filtering.include_patterns
array
| Glob patterns to include | +|
filtering.include_patterns.string
string
| | +|
hierarchy
HierarchyConfig
| Hierarchy configuration. | +|
hierarchy.enabled
boolean
| Enable parent-child relationships
Default: True
| +|
hierarchy.parent_strategy
Enum
| One of: "folder", "none", "custom", "notion", "confluence"
Default: folder
| +|
hierarchy.custom_mapping
One of CustomMappingConfig, null
| Custom mapping configuration
Default: None
| +|
hierarchy.custom_mapping.rules
array
| Custom parent mapping rules | +|
hierarchy.custom_mapping.rules.CustomParentRule
CustomParentRule
| Custom parent mapping rule. | +|
hierarchy.custom_mapping.rules.CustomParentRule.parent_id ❓
string
| Parent document ID for matching files | +|
hierarchy.custom_mapping.rules.CustomParentRule.pattern ❓
string
| Glob pattern to match file paths | +|
hierarchy.folder_mapping
FolderMappingConfig
| Folder hierarchy mapping configuration. | +|
hierarchy.folder_mapping.create_parent_docs
boolean
| Create Document entities for folders
Default: True
| +|
hierarchy.folder_mapping.max_depth
integer
| Maximum hierarchy depth
Default: 10
| +|
hierarchy.folder_mapping.parent_id_pattern
string
| Pattern for parent document IDs
Default: {source_type}-{directory}
| +|
hierarchy.folder_mapping.root_parent
One of string, null
| Optional root document URN
Default: None
| +|
pages
PageFilterConfig
| Configuration for filtering Confluence pages. | +|
pages.allow
One of array, null
| List of specific Confluence pages to include in ingestion. By default, all pages in discovered spaces are included. Specify page IDs or URLs to limit ingestion to specific pages and their children.

Examples:
- Page IDs: ['123456', '789012']
- Page URLs: ['https://domain.atlassian.net/wiki/spaces/ENG/pages/123456/API-Docs']

When specified, only these page trees will be ingested (if recursive=true). This allows focusing on specific documentation sections.
Default: None
| +|
pages.allow.string
string
| | +|
pages.deny
One of array, null
| List of specific Confluence pages to exclude from ingestion. Applies after allow filtering.

Examples:
- Exclude specific pages: ['123456', '789012']
- Page URLs: ['https://domain.atlassian.net/wiki/spaces/ENG/pages/999999/Draft']

Useful for excluding specific pages within otherwise included spaces.
Default: None
| +|
pages.deny.string
string
| | +|
processing
ProcessingConfig
| Processing configuration (partitioning only, no chunking). | +|
processing.parallelism
ParallelismConfig
| Parallelism configuration. | +|
processing.parallelism.disable_parallelism
boolean
| Disable all parallelism
Default: False
| +|
processing.parallelism.max_connections
integer
| Max concurrent connections for async operations
Default: 10
| +|
processing.parallelism.num_processes
integer
| Number of worker processes
Default: 2
| +|
processing.partition
PartitionConfig
| Unstructured partitioning configuration. | +|
processing.partition.additional_args
object
| Additional partition arguments | +|
processing.partition.api_key
One of string(password), null
| Unstructured API key
Default: None
| +|
processing.partition.partition_by_api
boolean
| Use Unstructured API for partitioning
Default: False
| +|
processing.partition.split_pdf_concurrency_level
integer
| Number of parallel requests for PDF pages
Default: 5
| +|
processing.partition.split_pdf_page
boolean
| Enable page-level splitting for large PDFs
Default: False
| +|
processing.partition.strategy
Enum
| One of: "auto", "hi_res", "fast", "ocr_only"
Default: auto
| +|
processing.partition.ocr_languages
array
| Languages for OCR
Default: ['eng']
| +|
processing.partition.ocr_languages.string
string
| | +|
spaces
SpaceFilterConfig
| Configuration for filtering Confluence spaces. | +|
spaces.allow
One of array, null
| List of Confluence spaces to include in ingestion. By default, all accessible spaces are discovered. Specify space keys or URLs to limit ingestion to specific spaces.

Examples:
- Space keys: ['ENGINEERING', 'PRODUCT', 'DESIGN']
- Space URLs: ['https://domain.atlassian.net/wiki/spaces/TEAM']
- Mixed: ['ENGINEERING', 'https://domain.atlassian.net/wiki/spaces/PRODUCT']

If specified, only these spaces will be ingested. Use deny to exclude specific spaces from discovery.
Default: None
| +|
spaces.allow.string
string
| | +|
spaces.deny
One of array, null
| List of Confluence spaces to exclude from ingestion. Applies after allow filtering.

Examples:
- Exclude personal spaces: ['~user1', '~user2']
- Exclude specific spaces: ['ARCHIVE', 'OLD_DOCS']
- Space URLs: ['https://domain.atlassian.net/wiki/spaces/TEST']

Useful for excluding personal spaces or archived content.
Default: None
| +|
spaces.deny.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Stateful Ingestion Config
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AdvancedConfig": { + "additionalProperties": false, + "description": "Advanced configuration options.", + "properties": { + "work_dir": { + "default": "/tmp/unstructured_datahub", + "title": "Work Dir", + "type": "string" + }, + "preserve_outputs": { + "default": false, + "title": "Preserve Outputs", + "type": "boolean" + }, + "output_format": { + "default": "json", + "enum": [ + "json", + "xml" + ], + "title": "Output Format", + "type": "string" + }, + "raise_on_error": { + "default": false, + "title": "Raise On Error", + "type": "boolean" + }, + "max_errors": { + "default": 10, + "title": "Max Errors", + "type": "integer" + }, + "continue_on_failure": { + "default": true, + "title": "Continue On Failure", + "type": "boolean" + }, + "retry": { + "$ref": "#/$defs/RetryConfig" + }, + "cache": { + "$ref": "#/$defs/CacheConfig" + } + }, + "title": "AdvancedConfig", + "type": "object" + }, + "CacheConfig": { + "additionalProperties": false, + "description": "Cache configuration.", + "properties": { + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "cache_dir": { + "default": "~/.cache/unstructured_datahub", + "title": "Cache Dir", + "type": "string" + }, + "ttl": { + "default": 86400, + "description": "Cache TTL in seconds", + "title": "Ttl", + "type": "integer" + } + }, + "title": "CacheConfig", + "type": "object" + }, + "ChunkingConfig": { + "additionalProperties": false, + "description": "Chunking strategy configuration.", + "properties": { + "strategy": { + "default": "by_title", + "description": "Chunking strategy to use", + "enum": [ + "basic", + "by_title" + ], + "title": "Strategy", + "type": "string" + }, + "max_characters": { + "default": 500, + "description": "Maximum characters per chunk", + "title": "Max Characters", + "type": "integer" + }, + "overlap": { + "default": 0, + "description": "Character overlap between chunks", + "title": "Overlap", + "type": "integer" + }, + "combine_text_under_n_chars": { + "default": 100, + "description": "Combine chunks smaller than this size", + "title": "Combine Text Under N Chars", + "type": "integer" + } + }, + "title": "ChunkingConfig", + "type": "object" + }, + "CustomMappingConfig": { + "additionalProperties": false, + "description": "Custom parent mapping configuration.", + "properties": { + "rules": { + "description": "Custom parent mapping rules", + "items": { + "$ref": "#/$defs/CustomParentRule" + }, + "title": "Rules", + "type": "array" + } + }, + "title": "CustomMappingConfig", + "type": "object" + }, + "CustomParentRule": { + "additionalProperties": false, + "description": "Custom parent mapping rule.", + "properties": { + "pattern": { + "description": "Glob pattern to match file paths", + "title": "Pattern", + "type": "string" + }, + "parent_id": { + "description": "Parent document ID for matching files", + "title": "Parent Id", + "type": "string" + } + }, + "required": [ + "pattern", + "parent_id" + ], + "title": "CustomParentRule", + "type": "object" + }, + "DocumentMappingConfig": { + "additionalProperties": false, + "description": "Document entity mapping configuration.", + "properties": { + "id_pattern": { + "default": "{source_type}-{directory}-{basename}", + "description": "Pattern for generating document IDs", + "title": "Id Pattern", + "type": "string" + }, + "id_normalization": { + "$ref": "#/$defs/IdNormalizationConfig", + "description": "ID normalization rules" + }, + "title": { + "$ref": "#/$defs/TitleExtractionConfig", + "description": "Title extraction configuration" + }, + "source": { + "$ref": "#/$defs/SourceConfig", + "description": "Source configuration" + }, + "status": { + "default": "PUBLISHED", + "description": "Default publication status", + "enum": [ + "PUBLISHED", + "UNPUBLISHED" + ], + "title": "Status", + "type": "string" + } + }, + "title": "DocumentMappingConfig", + "type": "object" + }, + "EmbeddingConfig": { + "additionalProperties": false, + "description": "Embedding generation configuration.\n\nDefault behavior: Fetches configuration from DataHub server automatically.\nOverride behavior: Validates local config against server when explicitly set.", + "properties": { + "provider": { + "anyOf": [ + { + "enum": [ + "bedrock", + "cohere", + "openai" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Embedding provider (bedrock uses AWS, cohere/openai use API key). If not set, loads from server.", + "title": "Provider" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Model name. If not set, loads from server.", + "title": "Model" + }, + "model_embedding_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Storage key for embeddings (e.g., 'cohere_embed_v3'). Required if overriding server config. If not set, loads from server.", + "title": "Model Embedding Key" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region for Bedrock. If not set, loads from server.", + "title": "Aws Region" + }, + "api_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "API key for Cohere (not needed for Bedrock with IAM roles)", + "title": "Api Key" + }, + "batch_size": { + "default": 25, + "description": "Batch size for embedding API calls", + "title": "Batch Size", + "type": "integer" + }, + "input_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "search_document", + "description": "Input type for Cohere embeddings", + "title": "Input Type" + }, + "rate_limit": { + "default": true, + "description": "Enable rate limiting for embedding API calls.", + "title": "Rate Limit", + "type": "boolean" + }, + "documents_per_minute": { + "default": 300, + "description": "Maximum number of documents to embed per minute when rate_limit is enabled.", + "exclusiveMinimum": 0, + "title": "Documents Per Minute", + "type": "integer" + }, + "allow_local_embedding_config": { + "default": false, + "description": "BREAK-GLASS: Allow local config without server validation. NOT RECOMMENDED - may break semantic search.", + "title": "Allow Local Embedding Config", + "type": "boolean" + } + }, + "title": "EmbeddingConfig", + "type": "object" + }, + "FilteringConfig": { + "additionalProperties": false, + "description": "File filtering configuration.", + "properties": { + "include_patterns": { + "description": "Glob patterns to include", + "items": { + "type": "string" + }, + "title": "Include Patterns", + "type": "array" + }, + "exclude_patterns": { + "description": "Glob patterns to exclude", + "items": { + "type": "string" + }, + "title": "Exclude Patterns", + "type": "array" + }, + "min_file_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Minimum file size in bytes", + "title": "Min File Size" + }, + "max_file_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum file size in bytes", + "title": "Max File Size" + }, + "modified_after": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only files modified after this date (ISO format)", + "title": "Modified After" + }, + "modified_before": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Only files modified before this date (ISO format)", + "title": "Modified Before" + }, + "skip_empty_documents": { + "default": true, + "description": "Skip documents with no text content", + "title": "Skip Empty Documents", + "type": "boolean" + }, + "min_text_length": { + "default": 50, + "description": "Minimum text length in characters", + "title": "Min Text Length", + "type": "integer" + } + }, + "title": "FilteringConfig", + "type": "object" + }, + "FolderMappingConfig": { + "additionalProperties": false, + "description": "Folder hierarchy mapping configuration.", + "properties": { + "create_parent_docs": { + "default": true, + "description": "Create Document entities for folders", + "title": "Create Parent Docs", + "type": "boolean" + }, + "parent_id_pattern": { + "default": "{source_type}-{directory}", + "description": "Pattern for parent document IDs", + "title": "Parent Id Pattern", + "type": "string" + }, + "max_depth": { + "default": 10, + "description": "Maximum hierarchy depth", + "maximum": 50, + "minimum": 1, + "title": "Max Depth", + "type": "integer" + }, + "root_parent": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional root document URN", + "title": "Root Parent" + } + }, + "title": "FolderMappingConfig", + "type": "object" + }, + "HierarchyConfig": { + "additionalProperties": false, + "description": "Hierarchy configuration.", + "properties": { + "enabled": { + "default": true, + "description": "Enable parent-child relationships", + "title": "Enabled", + "type": "boolean" + }, + "parent_strategy": { + "default": "folder", + "description": "Parent document creation strategy. 'notion' extracts parent from Notion API metadata. 'confluence' extracts parent from Confluence page ancestors.", + "enum": [ + "folder", + "none", + "custom", + "notion", + "confluence" + ], + "title": "Parent Strategy", + "type": "string" + }, + "folder_mapping": { + "$ref": "#/$defs/FolderMappingConfig", + "description": "Folder mapping configuration" + }, + "custom_mapping": { + "anyOf": [ + { + "$ref": "#/$defs/CustomMappingConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Custom mapping configuration" + } + }, + "title": "HierarchyConfig", + "type": "object" + }, + "IdNormalizationConfig": { + "additionalProperties": false, + "description": "Document ID normalization rules.", + "properties": { + "lowercase": { + "default": true, + "description": "Convert to lowercase", + "title": "Lowercase", + "type": "boolean" + }, + "replace_spaces_with": { + "default": "-", + "description": "Replace spaces with this character", + "title": "Replace Spaces With", + "type": "string" + }, + "remove_special_chars": { + "default": true, + "description": "Remove special characters except _ and -", + "title": "Remove Special Chars", + "type": "boolean" + }, + "max_length": { + "default": 200, + "description": "Maximum ID length", + "title": "Max Length", + "type": "integer" + } + }, + "title": "IdNormalizationConfig", + "type": "object" + }, + "PageFilterConfig": { + "additionalProperties": false, + "description": "Configuration for filtering Confluence pages.", + "properties": { + "allow": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of specific Confluence pages to include in ingestion. By default, all pages in discovered spaces are included. Specify page IDs or URLs to limit ingestion to specific pages and their children.\n\nExamples:\n - Page IDs: ['123456', '789012']\n - Page URLs: ['https://domain.atlassian.net/wiki/spaces/ENG/pages/123456/API-Docs']\n\nWhen specified, only these page trees will be ingested (if recursive=true). This allows focusing on specific documentation sections.", + "title": "Allow" + }, + "deny": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of specific Confluence pages to exclude from ingestion. Applies after allow filtering.\n\nExamples:\n - Exclude specific pages: ['123456', '789012']\n - Page URLs: ['https://domain.atlassian.net/wiki/spaces/ENG/pages/999999/Draft']\n\nUseful for excluding specific pages within otherwise included spaces.", + "title": "Deny" + } + }, + "title": "PageFilterConfig", + "type": "object" + }, + "ParallelismConfig": { + "additionalProperties": false, + "description": "Parallelism configuration.", + "properties": { + "num_processes": { + "default": 2, + "description": "Number of worker processes", + "maximum": 32, + "minimum": 1, + "title": "Num Processes", + "type": "integer" + }, + "disable_parallelism": { + "default": false, + "description": "Disable all parallelism", + "title": "Disable Parallelism", + "type": "boolean" + }, + "max_connections": { + "default": 10, + "description": "Max concurrent connections for async operations", + "title": "Max Connections", + "type": "integer" + } + }, + "title": "ParallelismConfig", + "type": "object" + }, + "PartitionConfig": { + "additionalProperties": false, + "description": "Unstructured partitioning configuration.", + "properties": { + "strategy": { + "default": "auto", + "description": "Partitioning strategy", + "enum": [ + "auto", + "hi_res", + "fast", + "ocr_only" + ], + "title": "Strategy", + "type": "string" + }, + "partition_by_api": { + "default": false, + "description": "Use Unstructured API for partitioning", + "title": "Partition By Api", + "type": "boolean" + }, + "api_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unstructured API key", + "title": "Api Key" + }, + "split_pdf_page": { + "default": false, + "description": "Enable page-level splitting for large PDFs", + "title": "Split Pdf Page", + "type": "boolean" + }, + "split_pdf_concurrency_level": { + "default": 5, + "description": "Number of parallel requests for PDF pages", + "title": "Split Pdf Concurrency Level", + "type": "integer" + }, + "ocr_languages": { + "default": [ + "eng" + ], + "description": "Languages for OCR", + "items": { + "type": "string" + }, + "title": "Ocr Languages", + "type": "array" + }, + "additional_args": { + "additionalProperties": true, + "description": "Additional partition arguments", + "title": "Additional Args", + "type": "object" + } + }, + "title": "PartitionConfig", + "type": "object" + }, + "ProcessingConfig": { + "additionalProperties": false, + "description": "Processing configuration (partitioning only, no chunking).", + "properties": { + "partition": { + "$ref": "#/$defs/PartitionConfig", + "description": "Partition configuration" + }, + "parallelism": { + "$ref": "#/$defs/ParallelismConfig", + "description": "Parallelism configuration" + } + }, + "title": "ProcessingConfig", + "type": "object" + }, + "RetryConfig": { + "additionalProperties": false, + "description": "Retry configuration.", + "properties": { + "enabled": { + "default": true, + "title": "Enabled", + "type": "boolean" + }, + "max_attempts": { + "default": 3, + "title": "Max Attempts", + "type": "integer" + }, + "backoff_factor": { + "default": 2, + "title": "Backoff Factor", + "type": "integer" + }, + "retry_on_timeout": { + "default": true, + "title": "Retry On Timeout", + "type": "boolean" + } + }, + "title": "RetryConfig", + "type": "object" + }, + "SourceConfig": { + "additionalProperties": false, + "description": "Document source configuration.", + "properties": { + "type": { + "default": "EXTERNAL", + "description": "Source type (always EXTERNAL for ingested docs)", + "enum": [ + "NATIVE", + "EXTERNAL" + ], + "title": "Type", + "type": "string" + }, + "include_external_url": { + "default": true, + "description": "Include external URL in DocumentSource", + "title": "Include External Url", + "type": "boolean" + }, + "include_external_id": { + "default": true, + "description": "Include external ID in DocumentSource", + "title": "Include External Id", + "type": "boolean" + } + }, + "title": "SourceConfig", + "type": "object" + }, + "SpaceFilterConfig": { + "additionalProperties": false, + "description": "Configuration for filtering Confluence spaces.", + "properties": { + "allow": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of Confluence spaces to include in ingestion. By default, all accessible spaces are discovered. Specify space keys or URLs to limit ingestion to specific spaces.\n\nExamples:\n - Space keys: ['ENGINEERING', 'PRODUCT', 'DESIGN']\n - Space URLs: ['https://domain.atlassian.net/wiki/spaces/TEAM']\n - Mixed: ['ENGINEERING', 'https://domain.atlassian.net/wiki/spaces/PRODUCT']\n\nIf specified, only these spaces will be ingested. Use deny to exclude specific spaces from discovery.", + "title": "Allow" + }, + "deny": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of Confluence spaces to exclude from ingestion. Applies after allow filtering.\n\nExamples:\n - Exclude personal spaces: ['~user1', '~user2']\n - Exclude specific spaces: ['ARCHIVE', 'OLD_DOCS']\n - Space URLs: ['https://domain.atlassian.net/wiki/spaces/TEST']\n\nUseful for excluding personal spaces or archived content.", + "title": "Deny" + } + }, + "title": "SpaceFilterConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + }, + "TitleExtractionConfig": { + "additionalProperties": false, + "description": "Title extraction configuration.", + "properties": { + "extract_from_content": { + "default": true, + "description": "Try to extract title from document content", + "title": "Extract From Content", + "type": "boolean" + }, + "fallback_to_filename": { + "default": true, + "description": "Use filename as title if not found in content", + "title": "Fallback To Filename", + "type": "boolean" + }, + "max_length": { + "default": 500, + "description": "Maximum title length", + "title": "Max Length", + "type": "integer" + } + }, + "title": "TitleExtractionConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Configuration for Confluence source connector.", + "properties": { + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Stateful Ingestion Config" + }, + "url": { + "description": "Base URL of your Confluence instance. Examples: 'https://your-domain.atlassian.net/wiki' (Cloud) or 'https://confluence.your-company.com' (Data Center)", + "title": "Url", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional human-readable identifier for this Confluence instance (e.g., 'mycompany-prod', 'team-a-confluence'). If not provided, automatically generated by hashing the base URL, which guarantees global uniqueness across all Confluence installations (both Cloud and Data Center). Use explicit values for more readable URNs, but auto-generated hashes are perfectly fine and require no manual configuration.", + "title": "Platform Instance" + }, + "cloud": { + "default": true, + "description": "Whether this is a Confluence Cloud instance (True) or Data Center/Server (False).", + "title": "Cloud", + "type": "boolean" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Username for Confluence Cloud authentication (required for Cloud).", + "title": "Username" + }, + "api_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "API token for Confluence Cloud authentication. Generate at: https://id.atlassian.com/manage-profile/security/api-tokens", + "title": "Api Token" + }, + "personal_access_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Personal Access Token for Confluence Data Center authentication. Generate from: User Profile > Settings > Personal Access Tokens", + "title": "Personal Access Token" + }, + "spaces": { + "$ref": "#/$defs/SpaceFilterConfig" + }, + "pages": { + "$ref": "#/$defs/PageFilterConfig" + }, + "max_spaces": { + "default": 100, + "description": "Maximum number of spaces to ingest when auto-discovering (applies when urls is not set).", + "title": "Max Spaces", + "type": "integer" + }, + "max_pages_per_space": { + "default": 1000, + "description": "Maximum number of pages to ingest per space.", + "title": "Max Pages Per Space", + "type": "integer" + }, + "recursive": { + "default": true, + "description": "Whether to recursively fetch child pages (applies to page URLs only).", + "title": "Recursive", + "type": "boolean" + }, + "processing": { + "$ref": "#/$defs/ProcessingConfig", + "description": "Document processing configuration (partitioning strategy, OCR, etc.)." + }, + "document_mapping": { + "$ref": "#/$defs/DocumentMappingConfig", + "description": "Configuration for mapping Confluence pages to DataHub documents." + }, + "hierarchy": { + "$ref": "#/$defs/HierarchyConfig", + "description": "Parent-child relationship configuration." + }, + "filtering": { + "$ref": "#/$defs/FilteringConfig", + "description": "Filtering options for document content." + }, + "chunking": { + "$ref": "#/$defs/ChunkingConfig", + "description": "Configuration for document chunking (required for embeddings)." + }, + "embedding": { + "$ref": "#/$defs/EmbeddingConfig", + "description": "Configuration for generating vector embeddings for semantic search." + }, + "max_documents": { + "default": 10000, + "description": "Maximum number of documents to process per ingestion run. The job will stop and fail with an error once this limit is reached. Set to 0 or -1 to disable the limit.", + "minimum": -1, + "title": "Max Documents", + "type": "integer" + }, + "advanced": { + "$ref": "#/$defs/AdvancedConfig", + "description": "Advanced ingestion options." + } + }, + "required": [ + "url" + ], + "title": "ConfluenceSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Common Use Cases + +##### 1. Auto-Discover All Spaces (Default) + +By default, the connector discovers and ingests all accessible spaces: + +```yaml +source: + type: confluence + config: + # Confluence Cloud + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@company.com" + api_token: "${CONFLUENCE_API_TOKEN}" + + # No filtering - discovers all accessible spaces + # Optional: limit number of spaces for large instances + max_spaces: 100 +``` + +##### 2. Include Specific Spaces + +Ingest only specific Confluence spaces: + +```yaml +source: + type: confluence + config: + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@company.com" + api_token: "${CONFLUENCE_API_TOKEN}" + + # Include only these spaces + spaces: + allow: + - "ENGINEERING" + - "PRODUCT" + - "DESIGN" +``` + +##### 3. Exclude Personal and Archive Spaces + +Ingest all spaces except specific ones: + +```yaml +source: + type: confluence + config: + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@company.com" + api_token: "${CONFLUENCE_API_TOKEN}" + + # Exclude personal spaces and archived content + spaces: + deny: + - "~john.doe" + - "~jane.smith" + - "ARCHIVE" + - "OLD_DOCS" +``` + +##### 4. Specific Page Trees Only + +Ingest specific pages and their descendants: + +```yaml +source: + type: confluence + config: + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@company.com" + api_token: "${CONFLUENCE_API_TOKEN}" + + # Start from specific pages + pages: + allow: + - "123456789" # API Documentation page tree + - "987654321" # User Guides page tree + recursive: true # Include all child pages +``` + +##### 5. Combined Space and Page Filtering + +Combine space and page filters for fine-grained control: + +```yaml +source: + type: confluence + config: + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@company.com" + api_token: "${CONFLUENCE_API_TOKEN}" + + # Include specific spaces + spaces: + allow: + - "ENGINEERING" + - "PRODUCT" + # Exclude personal spaces even if in allow list + deny: + - "~admin" + + # Exclude specific pages (e.g., drafts, archived content) + pages: + deny: + - "999999" # Draft page + - "888888" # Archived page +``` + +##### 6. Data Center / Server Setup + +Connect to Confluence Data Center or Server: + +```yaml +source: + type: confluence + config: + # Data Center / Server + cloud: false + url: "https://confluence.company.com" + personal_access_token: "${CONFLUENCE_PAT}" + + spaces: + allow: + - "WIKI" + - "DOCS" +``` + +##### 7. Production Setup with Stateful Ingestion + +Enterprise setup with incremental updates: + +```yaml +source: + type: confluence + config: + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@company.com" + api_token: "${CONFLUENCE_API_TOKEN}" + + spaces: + allow: + - "COMPANY" + - "PUBLIC" + + # Enable stateful ingestion for incremental updates + stateful_ingestion: + enabled: true +``` + +**Note**: Embedding configuration is managed by your DataHub instance. See [Semantic Search Configuration](../../../how-to/semantic-search-configuration.md) for setup. + +##### 8. Using URLs for Allow/Deny + +You can specify spaces and pages using full URLs for both allow and deny lists: + +```yaml +source: + type: confluence + config: + cloud: true + url: "https://your-domain.atlassian.net/wiki" + username: "user@company.com" + api_token: "${CONFLUENCE_API_TOKEN}" + + # Use full URLs - connector extracts keys/IDs automatically + spaces: + allow: + - "https://your-domain.atlassian.net/wiki/spaces/ENG" + - "https://your-domain.atlassian.net/wiki/spaces/PRODUCT" + deny: + - "https://your-domain.atlassian.net/wiki/spaces/ARCHIVE" + - "~john.doe" # Can mix URLs and keys + + pages: + allow: + - "https://your-domain.atlassian.net/wiki/spaces/ENG/pages/123456/Getting+Started" + deny: + - "https://your-domain.atlassian.net/wiki/spaces/ENG/pages/999999/Draft" +``` + +#### Filtering Content + +The connector provides flexible filtering options through allow and deny lists for both spaces and pages. + +##### Space Filtering + +Control which Confluence spaces are ingested: + +**`spaces.allow`**: Include only specific spaces (by default, all accessible spaces are discovered) + +```yaml +spaces: + allow: + - "ENGINEERING" # Space key + - "PRODUCT" + - "https://your-domain.atlassian.net/wiki/spaces/DESIGN" # Or full URL +``` + +**`spaces.deny`**: Exclude specific spaces (applied after `spaces.allow`) + +```yaml +spaces: + deny: + - "~john.doe" # Personal space + - "ARCHIVE" # Archived content + - "TEST" # Test space +``` + +##### Page Filtering + +Control which pages are ingested: + +**`pages.allow`**: Include only specific pages (triggers page-based mode, bypasses space discovery) + +```yaml +pages: + allow: + - "123456789" # Page ID + - "987654321" + - "https://your-domain.atlassian.net/wiki/spaces/ENG/pages/111111/API+Docs" # Or full URL +recursive: true # Include child pages +``` + +**`pages.deny`**: Exclude specific pages (works in both space-based and page-based modes) + +```yaml +pages: + deny: + - "999999" # Draft page + - "888888" # Archived page +``` + +##### Filtering Rules + +**Precedence**: + +- Deny lists always take precedence over allow lists +- If a space/page is in both allow and deny lists, it will be excluded + +**Modes**: + +- **Space-based mode** (default): Discovers spaces, then ingests all pages within allowed spaces +- **Page-based mode**: When `page_allow` is specified, bypasses space discovery and fetches specific page trees + +**Format Support**: + +- Space keys: `"ENGINEERING"`, `"~username"` (for personal spaces) +- Page IDs: `"123456789"` (numeric string) +- Full URLs: Both space URLs and page URLs are automatically parsed + +##### Common Filtering Patterns + +**Exclude all personal spaces:** + +```yaml +spaces: + deny: + - "~*" # Note: Use explicit user IDs, wildcard not supported + # Instead, list specific personal spaces: + - "~john.doe" + - "~jane.smith" +``` + +**Ingest only documentation spaces:** + +```yaml +spaces: + allow: + - "DOCS" + - "API_DOCS" + - "USER_GUIDES" +``` + +**Focus on specific documentation trees:** + +```yaml +pages: + allow: + - "123456" # API Documentation root page + - "789012" # User Guides root page +recursive: true +``` + +**Exclude drafts and WIP pages:** + +```yaml +pages: + deny: + - "999999" # Draft page ID + - "888888" # WIP page ID +``` + +#### How It Works + +##### Processing Pipeline + +1. **Discovery**: Confluence API discovers spaces and pages +2. **Download**: Downloads page content via Confluence REST API +3. **Extraction**: Extracts text, metadata, and hierarchy from pages +4. **Chunking**: Splits documents into semantic chunks (if embeddings enabled) +5. **Embedding**: Generates vector embeddings for each chunk (if embeddings enabled) +6. **Emission**: Emits Document entities with SemanticContent aspects to DataHub + +##### URL Format Support + +The connector supports multiple input formats for spaces and pages in allow/deny lists: + +**Space Identifiers:** + +- Space key: `"ENGINEERING"`, `"~username"` (for personal spaces) +- Full URL: `"https://your-domain.atlassian.net/wiki/spaces/ENGINEERING"` + +**Page Identifiers:** + +- Page ID: `"123456789"` (numeric string) +- Full URL (Cloud): `"https://your-domain.atlassian.net/wiki/spaces/ENG/pages/123456/Page+Title"` +- Full URL (Data Center): `"https://confluence.company.com/pages/viewpage.action?pageId=123456"` + +The connector automatically extracts space keys and page IDs from URLs, so you can use either format interchangeably in `space_allow`, `space_deny`, `page_allow`, and `page_deny` lists. + +##### Stateful Ingestion Details + +The source uses content-based change detection: + +- Calculates SHA-256 hash of document content + embedding configuration +- Compares hash with previous run to detect changes +- Only reprocesses documents when hash changes +- Tracks all emitted URNs to detect deletions + +This means: + +- **First run**: Processes all documents +- **Subsequent runs**: Only processes new/changed documents +- **Deleted pages**: Automatically soft-deleted from DataHub + +#### Performance Tuning + +##### Parallelism Settings + +```yaml +processing: + parallelism: + num_processes: 4 # Increase for faster processing (default: 2) + max_connections: 20 # Concurrent API connections (default: 10) +``` + +**Guidelines:** + +- Small spaces (<100 pages): `num_processes: 2` +- Medium spaces (100-500 pages): `num_processes: 4` +- Large spaces (>500 pages): `num_processes: 8` + +##### Filtering + +```yaml +filtering: + min_text_length: 100 # Skip short pages (default: 50) + skip_empty_documents: true # Skip empty pages (default: true) +``` + +##### Space Selection + +Instead of ingesting all spaces, select specific ones: + +```yaml +spaces: + allow: + - "ENGINEERING" # High-value documentation space + - "PRODUCT" # Product requirements space + deny: + - "~*" # Exclude personal spaces (list specific users) + - "ARCHIVE" # Exclude archived content + - "TEST" # Exclude test spaces +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +:::caution Not Supported with Remote Executor +This source is not supported with the Remote Executor in DataHub Cloud. It must be run using a self-hosted ingestion setup. +::: + +#### Limitations and Considerations + +##### Confluence API Limits + +- **Rate Limits**: Confluence enforces rate limits (Cloud: varies by plan, Data Center: configurable) +- **Content Types**: Complex macros may not extract perfectly (e.g., embedded content, custom macros) +- **Attachments**: File attachments are not ingested (only page content) + +##### Performance Considerations + +- **Large Spaces**: First run may take significant time for large spaces (1000+ pages) +- **Embedding Generation**: Adds processing time proportional to content volume +- **API Costs**: Embedding providers may incur costs based on usage + +##### Content Extraction + +- **Supported Content**: Text, headings, lists, code blocks, tables, panels +- **Limited Support**: Some macros extract as text/links +- **Not Supported**: Attachments, complex custom macros, embedded Jira issues (content only) + +### Troubleshooting + +#### Common Issues + +**"401 Unauthorized" or "Authentication failed" errors:** + +- **Cloud**: Verify `username` (email) and `api_token` are correct +- **Data Center**: Verify `personal_access_token` is valid and not expired +- Check that `cloud: true/false` matches your Confluence type +- Ensure the URL includes `/wiki` suffix for Cloud (e.g., `https://domain.atlassian.net/wiki`) + +**"403 Forbidden" or "Space not found" errors:** + +- Verify the user has read access to the specified spaces +- Check that space keys are correct (case-sensitive) +- For Cloud, ensure user is added to private spaces +- For Data Center, verify "View Space" permissions + +**Empty or missing content:** + +- Verify pages contain text (empty pages are skipped by default with `skip_empty_documents: true`) +- Check `min_text_length` filter setting (default: 50 characters) +- Ensure `recursive: true` if expecting child pages +- Check that pages are not restricted or have special permissions + +**Slow ingestion:** + +- Increase `processing.parallelism.num_processes` (default: 2) +- Consider filtering specific spaces instead of all spaces +- First run is always slower - subsequent runs use incremental updates +- Large spaces with 1000+ pages may take several minutes + +**Embedding generation failures:** + +- Verify provider API key is correct +- Check provider-specific rate limits (Cohere: 10k requests/min) +- Ensure embedding model name is valid for your provider +- For Bedrock: verify IAM permissions and model access is enabled in AWS Console + +**Stateful ingestion not working:** + +- Ensure `stateful_ingestion.enabled: true` in config +- Check DataHub connection (source needs to query previous state) +- Verify state file path is writable (if using file-based state) +- Look for state persistence logs in ingestion output + +**Missing hierarchy/parent relationships:** + +- Verify `hierarchy.enabled: true` (default) +- Check that parent pages are being ingested +- Ensure `recursive: true` to discover parent-child relationships +- Parent pages must be accessible to the API credentials + +**Page IDs not working:** + +- For Cloud, use the numeric page ID from the URL (after `/pages/`) +- For Data Center, page IDs may differ - use the ID from the page URL or query param `?pageId=` +- Alternatively, use full page URLs instead of IDs in `page_allow` or `page_deny` + +**How to find space keys and page IDs:** + +- **Space key**: Visible in the space URL: `https://domain.atlassian.net/wiki/spaces/ENGINEERING` → key is `ENGINEERING` +- **Page ID (Cloud)**: In the page URL after `/pages/`: `https://domain.atlassian.net/wiki/spaces/ENG/pages/123456/Title` → ID is `123456` +- **Page ID (Data Center)**: In the URL query parameter: `https://confluence.company.com/pages/viewpage.action?pageId=123456` → ID is `123456` +- **Personal space key**: Format is `~username` (e.g., `~john.doe` for user john.doe) + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.confluence.confluence_source.ConfluenceSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/confluence/confluence_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Confluence, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/csv-enricher.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/csv-enricher.md new file mode 100644 index 00000000..34cfe325 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/csv-enricher.md @@ -0,0 +1,196 @@ +--- +sidebar_position: 12 +title: CSV Enricher +slug: /generated/ingestion/sources/csv-enricher +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# CSV Enricher + +## Overview + +Csv Enricher is a DataHub utility or metadata-focused integration. Learn more in the [official Csv Enricher documentation](https://datahub.com/docs/). + +The DataHub integration for Csv Enricher covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +:::info Looking to ingest a CSV data file into DataHub, as an asset? + +Use the Local File ingestion source. The CSV enricher is used for enriching entities already ingested into DataHub. +::: + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `csv-enricher` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Descriptions | ✅ | Supported by default. | +| [Domains](../../../domains.md) | ✅ | Supported by default. | +| Extract Ownership | ✅ | Supported by default. | +| Extract Tags | ✅ | Supported by default. | + +### Overview + +The `csv-enricher` module ingests metadata from Csv Enricher into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin is used to bulk upload metadata to Datahub. It will apply glossary terms, tags, description, owners and domain at the entity level. It can also be used to apply tags, glossary terms, and documentation at the column level. These values are read from a CSV file. You have the option to either overwrite or append existing values. + +The format of the CSV is demonstrated below. The header is required and URNs should be surrounded by quotes when they contains commas (most URNs contains commas). + +``` +resource,subresource,glossary_terms,tags,owners,ownership_type,description,domain,ownership_type_urn +"urn:li:dataset:(urn:li:dataPlatform:snowflake,datahub.growth.users,PROD)",,[urn:li:glossaryTerm:Users],[urn:li:tag:HighQuality],[urn:li:corpuser:lfoe|urn:li:corpuser:jdoe],CUSTOM,"description for users table",urn:li:domain:Engineering,urn:li:ownershipType:a0e9176c-d8cf-4b11-963b-f7a1bc2333c9 +"urn:li:dataset:(urn:li:dataPlatform:hive,datahub.growth.users,PROD)",first_name,[urn:li:glossaryTerm:FirstName],,,,"first_name description", +"urn:li:dataset:(urn:li:dataPlatform:hive,datahub.growth.users,PROD)",last_name,[urn:li:glossaryTerm:LastName],,,,"last_name description", +``` + +Note that the first row does not have a subresource populated. That means any glossary terms, tags, and owners will be applied at the entity field. If a subresource is populated (as it is for the second and third rows), glossary terms and tags will be applied on the column. Every row MUST have a resource. Also note that owners can only be applied at the resource level. + +If ownership_type_urn is set then ownership_type must be set to CUSTOM. + +Note that you have the option in your recipe config to write as a PATCH or as an OVERRIDE. This choice will apply to all metadata for the entity, not just a single aspect. So OVERRIDE will override all metadata, including performing deletes if a metadata field is empty. The default is PATCH. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[csv-enricher]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: csv-enricher + config: + # relative path to your csv file to ingest + filename: ./path/to/your/file.csv + +# Default sink is datahub-rest and doesn't need to be configured +# See https://docs.datahub.com/docs/metadata-ingestion/sink_docs/datahub for customization options + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
filename ✅
string
| File path or URL of CSV file to ingest. | +|
array_delimiter
string
| Delimiter to use when parsing array fields (tags, terms and owners)
Default: |
| +|
delimiter
string
| Delimiter to use when parsing CSV
Default: ,
| +|
write_semantics
string
| Whether the new tags, terms and owners to be added will override the existing ones added only by this source or not. Value for this config can be "PATCH" or "OVERRIDE". NOTE: this will apply to all metadata for the entity, not just a single aspect.
Default: PATCH
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "additionalProperties": false, + "properties": { + "filename": { + "description": "File path or URL of CSV file to ingest.", + "title": "Filename", + "type": "string" + }, + "write_semantics": { + "default": "PATCH", + "description": "Whether the new tags, terms and owners to be added will override the existing ones added only by this source or not. Value for this config can be \"PATCH\" or \"OVERRIDE\". NOTE: this will apply to all metadata for the entity, not just a single aspect.", + "title": "Write Semantics", + "type": "string" + }, + "delimiter": { + "default": ",", + "description": "Delimiter to use when parsing CSV", + "title": "Delimiter", + "type": "string" + }, + "array_delimiter": { + "default": "|", + "description": "Delimiter to use when parsing array fields (tags, terms and owners)", + "title": "Array Delimiter", + "type": "string" + } + }, + "required": [ + "filename" + ], + "title": "CSVEnricherConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +:::warning Performance Considerations + +This source will not work on very large csv files that do not fit in memory. +::: + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.csv_enricher.CSVEnricherSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/csv_enricher.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for CSV Enricher, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/databricks.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/databricks.md new file mode 100644 index 00000000..67c87966 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/databricks.md @@ -0,0 +1,2119 @@ +--- +sidebar_position: 13 +title: Databricks +slug: /generated/ingestion/sources/databricks +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Databricks + +## Overview + +Databricks is a data platform used to store and query analytical or operational data. Learn more in the [official Databricks documentation](https://www.databricks.com/). + +The DataHub integration for Databricks covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +DataHub supports integration with Databricks ecosystem using a multitude of connectors, depending on your exact setup. + +### Databricks Unity Catalog (new) + +The recently introduced [Unity Catalog](https://www.databricks.com/product/unity-catalog) provides a new way to govern your assets within the Databricks lakehouse. If you have Unity Catalog Enabled Workspace, you can use the `databricks` source (aka `unity-catalog` source, see below for details) to integrate your metadata into DataHub as an alternate to the Hive pathway. This also ingests hive metastore catalog in Databricks and is recommended approach to ingest Databricks ecosystem in DataHub. + +### Databricks Hive (old) + +The alternative way to integrate is via the Hive connector. The [Hive starter recipe](http://datahubproject.io/docs/generated/ingestion/sources/hive#starter-recipe) has a section describing how to connect to your Databricks workspace. + +### Databricks Spark + +To complete the picture, we recommend adding push-based ingestion from your Spark jobs to see real-time activity and lineage between your Databricks tables and your Spark jobs. Use the Spark agent to push metadata to DataHub using the instructions [here](../../../../metadata-integration/java/acryl-spark-lineage/README.md#configuration-instructions-databricks). + +### Watch the DataHub Talk at the Data and AI Summit 2022 + +For a deeper look at how to think about DataHub within and across your Databricks ecosystem, watch the recording of our talk at the Data and AI Summit 2022. + +

+ + + +

+ +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `unity-catalog` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Catalog, Schema. | +| Column-level Lineage | ✅ | Enabled by default. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Supported via the `profiling.enabled` config. | +| Dataset Usage | ✅ | Enabled by default. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| Extract Ownership | ✅ | Supported via the `include_ownership` config. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `unity-catalog` module ingests metadata from Databricks into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +- Get your Databricks instance's [workspace url](https://docs.databricks.com/workspace/workspace-details.html#workspace-instance-names-urls-and-ids) +- Create a [Databricks Service Principal](https://docs.databricks.com/administration-guide/users-groups/service-principals.html#what-is-a-service-principal) + - You can skip this step and use your own account to get things running quickly, + but we strongly recommend creating a dedicated service principal for production use. + +#### Authentication Options + +You can authenticate with Databricks using OAuth, Azure authentication, a Personal Access Token (legacy), or Databricks unified authentication: + +**Option 1: OAuth** + +- Generate a Databricks OAuth secret using the following guide: + - [Authorize service principal access to Databricks with OAuth](https://docs.databricks.com/aws/en/dev-tools/auth/oauth-m2m#oauth-m2m-manual) + +**Option 2: Azure Authentication (for Azure Databricks)** + +- Create an Azure Active Directory application: + - Follow the [Azure AD app registration guide](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app) + - Note down the `client_id` (Application ID), `tenant_id` (Directory ID), and create a `client_secret` +- Grant the Azure AD application access to your Databricks workspace: + - Add the service principal to your Databricks workspace following [this guide](https://docs.databricks.com/administration-guide/users-groups/service-principals.html#add-a-service-principal-to-your-azure-databricks-account-using-the-account-console) + +**Option 3: Personal Access Token (PAT) (legacy)** + +- Generate a Databricks Personal Access token following the following guides: + - [Service Principals](https://docs.databricks.com/administration-guide/users-groups/service-principals.html#personal-access-tokens) + - [Personal Access Tokens](https://docs.databricks.com/dev-tools/auth.html#databricks-personal-access-tokens) + +**Option 4: Unified authentication** + +- Set authentication configuration via environment variables or Databricks configuration profiles, as specified in the [Databricks unified authentication guide](https://docs.databricks.com/aws/en/dev-tools/auth/unified-auth). + +#### Provision your service account: + +- To ingest your workspace's metadata and lineage, your service principal must have all of the following: + - One of: metastore admin role, ownership of, or `USE CATALOG` privilege on any catalogs you want to ingest + - One of: metastore admin role, ownership of, or `USE SCHEMA` privilege on any schemas you want to ingest + - Ownership of or `SELECT` privilege on any tables and views you want to ingest + - [Ownership documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/ownership.html) + - [Privileges documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/privileges.html) +- To ingest legacy hive_metastore catalog (`include_hive_metastore` - enabled by default), your service principal must have all of the following: + - `READ_METADATA` and `USAGE` privilege on `hive_metastore` catalog + - `READ_METADATA` and `USAGE` privilege on schemas you want to ingest + - `READ_METADATA` and `USAGE` privilege on tables and views you want to ingest + - [Hive Metastore Privileges documentation](https://docs.databricks.com/en/sql/language-manual/sql-ref-privileges-hms.html) +- To ingest your workspace's notebooks and respective lineage, your service principal must have `CAN_READ` privileges on the folders containing the notebooks you want to ingest: [guide](https://docs.databricks.com/en/security/auth-authz/access-control/workspace-acl.html#folder-permissions). +- To `include_usage_statistics` (enabled by default), your service principal must have one of the following: + - `CAN_MANAGE` permissions on any SQL Warehouses you want to ingest: [guide](https://docs.databricks.com/security/auth-authz/access-control/sql-endpoint-acl.html). + - When `usage_data_source` is set to `SYSTEM_TABLES` or `AUTO` (default) with `warehouse_id` configured: `SELECT` privilege on `system.query.history` table for improved performance with large query volumes and multi-workspace setups. +- To ingest `profiling` information with `method: ge`, you need `SELECT` privileges on all profiled tables. +- To ingest `profiling` information with `method: analyze` and `call_analyze: true` (enabled by default), your service principal must have ownership or `MODIFY` privilege on any tables you want to profile. + - Alternatively, you can run [ANALYZE TABLE](https://docs.databricks.com/sql/language-manual/sql-ref-syntax-aux-analyze-table.html) yourself on any tables you want to profile, then set `call_analyze` to `false`. + You will still need `SELECT` privilege on those tables to fetch the results. +- Check the starter recipe below and replace `workspace_url` and either `token` (for PAT authentication) or `azure_auth` credentials (for Azure authentication) with your information from the previous steps. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[unity-catalog]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: databricks + config: + workspace_url: https://my-workspace.cloud.databricks.com + + # Authentication Option 1: OAuth + client_id: "" + client_secret: "" + + # Authentication Option 2: Azure Authentication (for Azure Databricks) + # Uncomment the following section and comment out the token above to use Azure auth + # azure_auth: + # client_id: "" + # tenant_id: "" + # client_secret: "" + + # Authentication Option 3: Personal Access Token + # Uncomment the following section and comment out client_id/client_secret above + # token: "" + + # Authentication Option 4: Databricks unified auth + # Comment out client_id/client_secret above to use Databricks unified auth (reads environment variables + # and local Databricks configuration profiles) + + include_metastore: false + include_ownership: true + include_ml_model_aliases: false + ml_model_max_results: 1000 + profiling: + method: "ge" + enabled: true + warehouse_id: "" + profile_table_level_only: false + max_wait_secs: 60 + pattern: + deny: + - ".*\\.unwanted_schema" + +# emit_siblings: true +# delta_lake_options: +# platform_instance_name: null +# env: 'PROD' + +# profiling: +# method: "analyze" +# enabled: true +# warehouse_id: "" +# profile_table_level_only: true +# call_analyze: true + +# catalogs: ["my_catalog"] +# schema_pattern: +# deny: +# - information_schema +# table_pattern: +# allow: +# - my_catalog.my_schema.my_table +# First you have to create domains on Datahub by following this guide -> https://docs.datahub.com/docs/domains/#domains-setup-prerequisites-and-permissions +# domain: +# urn:li:domain:1111-222-333-444-555: +# allow: +# - main.* + + stateful_ingestion: + enabled: true + +pipeline_name: acme-corp-unity + + +# sink configs if needed + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
workspace_url ✅
string
| Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com | +|
bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
client_id
One of string, null
| Databricks service principal client ID
Default: None
| +|
client_secret
One of string(password), null
| Databricks service principal client secret
Default: None
| +|
column_lineage_column_limit
integer
| Limit the number of columns to get column level lineage.
Default: 300
| +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
databricks_api_page_size
integer
| Page size for Databricks API calls when listing resources (catalogs, schemas, tables, etc.). When set to 0 (default), uses server-side configured page length (recommended). When set to a positive value, the page length is the minimum of this value and the server configured value. Must be a non-negative integer.
Default: 0
| +|
emit_siblings
boolean
| Whether to emit siblings relation with corresponding delta-lake platform's table. If enabled, this will also ingest the corresponding delta-lake table.
Default: True
| +|
enable_stateful_profiling
boolean
| Enable stateful profiling. This will store profiling timestamps per dataset after successful profiling. and will not run profiling again in subsequent run if table has not been updated.
Default: True
| +|
end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
extra_client_options
object
| Additional options to pass to Databricks SQLAlchemy client.
Default: {}
| +|
format_sql_queries
boolean
| Whether to format sql queries
Default: False
| +|
ignore_start_time_lineage
boolean
| Option to ignore the start_time and retrieve all available lineage. When enabled, the start_time filter will be set to zero to extract all lineage events regardless of the configured time window.
Default: False
| +|
include_column_lineage
boolean
| Option to enable/disable lineage generation. Currently we have to call a rest call per column to get column level lineage due to the Databrick api which can slow down ingestion.
Default: True
| +|
include_external_lineage
boolean
| Option to enable/disable lineage generation for external tables. Only external S3 tables are supported at the moment.
Default: True
| +|
include_hive_metastore
boolean
| Whether to ingest legacy `hive_metastore` catalog. This requires executing queries on SQL warehouse.
Default: True
| +|
include_metastore
boolean
| Whether to ingest the workspace's metastore as a container and include it in all urns. Changing this will affect the urns of all entities in the workspace. This config is deprecated and will be removed in the future, so it is recommended to not set this to `True` for new ingestions. If you have an existing unity catalog ingestion, you'll want to avoid duplicates by soft deleting existing data. If stateful ingestion is enabled, running with `include_metastore: false` should be sufficient. Otherwise, we recommend deleting via the cli: `datahub delete --platform databricks` and re-ingesting with `include_metastore: false`.
Default: False
| +|
include_ml_model_aliases
boolean
| Whether to include ML model aliases in the ingestion.
Default: False
| +|
include_notebooks
boolean
| Ingest notebooks, represented as DataHub datasets.
Default: False
| +|
include_operational_stats
boolean
| Whether to display operational stats.
Default: True
| +|
include_ownership
boolean
| Option to enable/disable ownership generation for metastores, catalogs, schemas, and tables.
Default: False
| +|
include_read_operational_stats
boolean
| Whether to report read operational stats. Experimental.
Default: False
| +|
include_table_lineage
boolean
| Option to enable/disable lineage generation.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_tags
boolean
| Option to enable/disable column/table tag extraction. Requires warehouse_id to be set since tag extraction needs to query system.information_schema.tags. If warehouse_id is not provided, this will be automatically disabled to allow ingestion to continue.
Default: True
| +|
include_top_n_queries
boolean
| Whether to ingest the top_n_queries.
Default: True
| +|
include_usage_statistics
boolean
| Generate usage statistics.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
ingest_data_platform_instance_aspect
One of boolean, null
| Option to enable/disable ingestion of the data platform instance aspect. The default data platform instance id for a dataset is workspace_name
Default: False
| +|
lineage_data_source
Enum
| One of: "AUTO", "SYSTEM_TABLES", "API" | +|
ml_model_max_results
integer
| Maximum number of ML models to ingest.
Default: 1000
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. | +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
scheme
string
|
Default: databricks
| +|
start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
token
One of string(password), null
| Databricks personal access token
Default: None
| +|
top_n_queries
integer
| Number of top queries to save to each table.
Default: 10
| +|
usage_data_source
Enum
| One of: "AUTO", "SYSTEM_TABLES", "API" | +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
warehouse_id
One of string, null
| SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. Required for the following features that need SQL access: 1) Tag extraction (include_tags=True) - queries system.information_schema.tags 2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog 3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage 4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue.
Default: None
| +|
workspace_name
One of string, null
| Name of the workspace. Default to deployment name present in workspace_url
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
azure_auth
One of AzureAuthConfig, null
| Azure configuration
Default: None
| +|
azure_auth.client_id ❓
string
| Azure application (client) ID. This is the unique identifier for the registered Azure AD application. | +|
azure_auth.client_secret ❓
string(password)
| Azure application client secret used for authentication. This is a confidential credential that should be kept secure. | +|
azure_auth.tenant_id ❓
string
| Azure tenant (directory) ID. This identifies the Azure AD tenant where the application is registered. | +|
catalog_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
catalog_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
catalogs
One of array, null
| Fixed list of catalogs to ingest. If not specified, catalogs will be ingested based on `catalog_pattern`.
Default: None
| +|
catalogs.string
string
| | +|
delta_lake_options
DeltaLakeDetails
| | +|
delta_lake_options.platform_instance_name
One of string, null
| Delta-lake paltform instance name
Default: None
| +|
delta_lake_options.env
string
| Delta-lake environment
Default: PROD
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
notebook_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
notebook_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
user_email_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
user_email_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
One of UnityCatalogGEProfilerConfig, UnityCatalogAnalyzeProfilerConfig, UnityCatalogSQLAlchemyProfilerConfig
| Data profiling configuration
Default: {'method': 'ge', 'enabled': False, 'operation_conf...
| +|
profiling.call_analyze
boolean
| Whether to call ANALYZE TABLE as part of profile ingestion.If false, will ingest the results of the most recent ANALYZE TABLE call, if any.
Default: True
| +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
One of boolean, boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null, union(anyOf), integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null, union(anyOf), integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_wait_secs
One of integer, null, integer, union(anyOf), integer, null
| Maximum time to wait for a table to be profiled.
Default: None
| +|
profiling.max_workers
One of integer, integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
One of string, string
| Const value: ge
Default: ge
| +|
profiling.offset
One of integer, null, union(anyOf), integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null, union(anyOf), string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null, union(anyOf), number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
One of boolean, boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null, union(anyOf), integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null, union(anyOf), integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.warehouse_id
One of string, null, union(anyOf), string, null, union(anyOf), string, null
| SQL Warehouse id, for running profiling queries.
Default: None
| +|
profiling.operation_config
One of OperationConfig, OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
One of boolean, boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null, union(anyOf), integer, null, union(anyOf), integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null, union(anyOf), integer, null, union(anyOf), integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.pattern
One of AllowDenyPattern, AllowDenyPattern
| A class to store allow deny regexes | +|
profiling.pattern.ignoreCase
One of boolean, null, union(anyOf), boolean, null, union(anyOf), boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling.tags_to_ignore_sampling
One of array, null, union(anyOf), array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Unity Catalog Stateful Ingestion Config.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AzureAuthConfig": { + "additionalProperties": false, + "properties": { + "client_secret": { + "description": "Azure application client secret used for authentication. This is a confidential credential that should be kept secure.", + "format": "password", + "title": "Client Secret", + "type": "string", + "writeOnly": true + }, + "client_id": { + "description": "Azure application (client) ID. This is the unique identifier for the registered Azure AD application.", + "title": "Client Id", + "type": "string" + }, + "tenant_id": { + "description": "Azure tenant (directory) ID. This identifies the Azure AD tenant where the application is registered.", + "title": "Tenant Id", + "type": "string" + } + }, + "required": [ + "client_secret", + "client_id", + "tenant_id" + ], + "title": "AzureAuthConfig", + "type": "object" + }, + "BucketDuration": { + "enum": [ + "DAY", + "HOUR" + ], + "title": "BucketDuration", + "type": "string" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DeltaLakeDetails": { + "additionalProperties": false, + "properties": { + "platform_instance_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Delta-lake paltform instance name", + "title": "Platform Instance Name" + }, + "env": { + "default": "PROD", + "description": "Delta-lake environment", + "title": "Env", + "type": "string" + } + }, + "title": "DeltaLakeDetails", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "LineageDataSource": { + "enum": [ + "AUTO", + "SYSTEM_TABLES", + "API" + ], + "title": "LineageDataSource", + "type": "string" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + }, + "UnityCatalogAnalyzeProfilerConfig": { + "additionalProperties": false, + "properties": { + "method": { + "const": "analyze", + "default": "analyze", + "title": "Method", + "type": "string" + }, + "warehouse_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "SQL Warehouse id, for running profiling queries.", + "title": "Warehouse Id" + }, + "pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for profiling during ingestion. Specify regex to match the `catalog.schema.table` format. Note that only tables allowed by the `table_pattern` will be considered." + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "call_analyze": { + "default": true, + "description": "Whether to call ANALYZE TABLE as part of profile ingestion.If false, will ingest the results of the most recent ANALYZE TABLE call, if any.", + "title": "Call Analyze", + "type": "boolean" + }, + "max_wait_secs": { + "default": 3600, + "description": "Maximum time to wait for an ANALYZE TABLE query to complete.", + "title": "Max Wait Secs", + "type": "integer" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + } + }, + "title": "UnityCatalogAnalyzeProfilerConfig", + "type": "object" + }, + "UnityCatalogGEProfilerConfig": { + "additionalProperties": false, + "properties": { + "method": { + "const": "ge", + "default": "ge", + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + }, + "warehouse_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "SQL Warehouse id, for running profiling queries.", + "title": "Warehouse Id" + }, + "pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for profiling during ingestion. Specify regex to match the `catalog.schema.table` format. Note that only tables allowed by the `table_pattern` will be considered." + }, + "max_wait_secs": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum time to wait for a table to be profiled.", + "title": "Max Wait Secs" + } + }, + "title": "UnityCatalogGEProfilerConfig", + "type": "object" + }, + "UnityCatalogSQLAlchemyProfilerConfig": { + "additionalProperties": false, + "properties": { + "method": { + "const": "sqlalchemy", + "default": "sqlalchemy", + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + }, + "warehouse_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "SQL Warehouse id, for running profiling queries.", + "title": "Warehouse Id" + }, + "pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for profiling during ingestion. Specify regex to match the `catalog.schema.table` format. Note that only tables allowed by the `table_pattern` will be considered." + }, + "max_wait_secs": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum time to wait for a table to be profiled.", + "title": "Max Wait Secs" + } + }, + "title": "UnityCatalogSQLAlchemyProfilerConfig", + "type": "object" + }, + "UsageDataSource": { + "enum": [ + "AUTO", + "SYSTEM_TABLES", + "API" + ], + "title": "UsageDataSource", + "type": "string" + } + }, + "additionalProperties": false, + "properties": { + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schemas to filter in ingestion. Specify regex to the full `metastore.catalog.schema` name. e.g. to match all tables in schema analytics, use the regex `^mymetastore\\.mycatalog\\.analytics$`." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in `catalog.schema.table` format. e.g. to match all tables starting with customer in Customer catalog and public schema, use the regex `Customer\\.public\\.customer.*`." + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "enable_stateful_profiling": { + "default": true, + "description": "Enable stateful profiling. This will store profiling timestamps per dataset after successful profiling. and will not run profiling again in subsequent run if table has not been updated. ", + "title": "Enable Stateful Profiling", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "top_n_queries": { + "default": 10, + "description": "Number of top queries to save to each table.", + "exclusiveMinimum": 0, + "title": "Top N Queries", + "type": "integer" + }, + "user_email_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for user emails to filter in usage." + }, + "include_operational_stats": { + "default": true, + "description": "Whether to display operational stats.", + "title": "Include Operational Stats", + "type": "boolean" + }, + "include_read_operational_stats": { + "default": false, + "description": "Whether to report read operational stats. Experimental.", + "title": "Include Read Operational Stats", + "type": "boolean" + }, + "format_sql_queries": { + "default": false, + "description": "Whether to format sql queries", + "title": "Format Sql Queries", + "type": "boolean" + }, + "include_top_n_queries": { + "default": true, + "description": "Whether to ingest the top_n_queries.", + "title": "Include Top N Queries", + "type": "boolean" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Unity Catalog Stateful Ingestion Config." + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to catalogs, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false, + "warehouse_id": null, + "pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "max_wait_secs": null + }, + "description": "Data profiling configuration", + "discriminator": { + "mapping": { + "analyze": "#/$defs/UnityCatalogAnalyzeProfilerConfig", + "ge": "#/$defs/UnityCatalogGEProfilerConfig", + "sqlalchemy": "#/$defs/UnityCatalogSQLAlchemyProfilerConfig" + }, + "propertyName": "method" + }, + "oneOf": [ + { + "$ref": "#/$defs/UnityCatalogGEProfilerConfig" + }, + { + "$ref": "#/$defs/UnityCatalogAnalyzeProfilerConfig" + }, + { + "$ref": "#/$defs/UnityCatalogSQLAlchemyProfilerConfig" + } + ], + "title": "Profiling" + }, + "scheme": { + "default": "databricks", + "title": "Scheme", + "type": "string" + }, + "token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Databricks personal access token", + "title": "Token" + }, + "azure_auth": { + "anyOf": [ + { + "$ref": "#/$defs/AzureAuthConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure configuration" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Databricks service principal client ID", + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Databricks service principal client secret", + "title": "Client Secret" + }, + "workspace_url": { + "description": "Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com", + "title": "Workspace Url", + "type": "string" + }, + "warehouse_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. Required for the following features that need SQL access: 1) Tag extraction (include_tags=True) - queries system.information_schema.tags 2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog 3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage 4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue.", + "title": "Warehouse Id" + }, + "extra_client_options": { + "additionalProperties": true, + "default": {}, + "description": "Additional options to pass to Databricks SQLAlchemy client.", + "title": "Extra Client Options", + "type": "object" + }, + "include_metastore": { + "default": false, + "description": "Whether to ingest the workspace's metastore as a container and include it in all urns. Changing this will affect the urns of all entities in the workspace. This config is deprecated and will be removed in the future, so it is recommended to not set this to `True` for new ingestions. If you have an existing unity catalog ingestion, you'll want to avoid duplicates by soft deleting existing data. If stateful ingestion is enabled, running with `include_metastore: false` should be sufficient. Otherwise, we recommend deleting via the cli: `datahub delete --platform databricks` and re-ingesting with `include_metastore: false`.", + "title": "Include Metastore", + "type": "boolean" + }, + "ingest_data_platform_instance_aspect": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Option to enable/disable ingestion of the data platform instance aspect. The default data platform instance id for a dataset is workspace_name", + "title": "Ingest Data Platform Instance Aspect" + }, + "catalogs": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of catalogs to ingest. If not specified, catalogs will be ingested based on `catalog_pattern`.", + "title": "Catalogs" + }, + "catalog_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for catalogs to filter in ingestion. Specify regex to match the full `metastore.catalog` name." + }, + "notebook_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for notebooks to filter in ingestion, based on notebook *path*. Specify regex to match the entire notebook path in `//.../` format. e.g. to match all notebooks in the root Shared directory, use the regex `/Shared/.*`." + }, + "include_table_lineage": { + "default": true, + "description": "Option to enable/disable lineage generation.", + "title": "Include Table Lineage", + "type": "boolean" + }, + "include_external_lineage": { + "default": true, + "description": "Option to enable/disable lineage generation for external tables. Only external S3 tables are supported at the moment.", + "title": "Include External Lineage", + "type": "boolean" + }, + "include_notebooks": { + "default": false, + "description": "Ingest notebooks, represented as DataHub datasets.", + "title": "Include Notebooks", + "type": "boolean" + }, + "include_ownership": { + "default": false, + "description": "Option to enable/disable ownership generation for metastores, catalogs, schemas, and tables.", + "title": "Include Ownership", + "type": "boolean" + }, + "include_tags": { + "default": true, + "description": "Option to enable/disable column/table tag extraction. Requires warehouse_id to be set since tag extraction needs to query system.information_schema.tags. If warehouse_id is not provided, this will be automatically disabled to allow ingestion to continue.", + "title": "Include Tags", + "type": "boolean" + }, + "include_column_lineage": { + "default": true, + "description": "Option to enable/disable lineage generation. Currently we have to call a rest call per column to get column level lineage due to the Databrick api which can slow down ingestion. ", + "title": "Include Column Lineage", + "type": "boolean" + }, + "lineage_data_source": { + "$ref": "#/$defs/LineageDataSource", + "default": "AUTO", + "description": "Source for lineage data extraction. Options: 'AUTO' - Use system tables when SQL warehouse is available, fallback to API; 'SYSTEM_TABLES' - Force use of system.access.table_lineage and system.access.column_lineage tables (requires SQL warehouse); 'API' - Force use of REST API endpoints for lineage data" + }, + "ignore_start_time_lineage": { + "default": false, + "description": "Option to ignore the start_time and retrieve all available lineage. When enabled, the start_time filter will be set to zero to extract all lineage events regardless of the configured time window.", + "title": "Ignore Start Time Lineage", + "type": "boolean" + }, + "column_lineage_column_limit": { + "default": 300, + "description": "Limit the number of columns to get column level lineage. ", + "title": "Column Lineage Column Limit", + "type": "integer" + }, + "databricks_api_page_size": { + "default": 0, + "description": "Page size for Databricks API calls when listing resources (catalogs, schemas, tables, etc.). When set to 0 (default), uses server-side configured page length (recommended). When set to a positive value, the page length is the minimum of this value and the server configured value. Must be a non-negative integer.", + "minimum": 0, + "title": "Databricks Api Page Size", + "type": "integer" + }, + "include_usage_statistics": { + "default": true, + "description": "Generate usage statistics.", + "title": "Include Usage Statistics", + "type": "boolean" + }, + "usage_data_source": { + "$ref": "#/$defs/UsageDataSource", + "default": "AUTO", + "description": "Source for usage/query history data extraction. Options: 'AUTO' (default) - Automatically use system.query.history table when SQL warehouse is configured, otherwise fall back to REST API. This provides better performance for multi-workspace setups and large query volumes when warehouse_id is set. 'SYSTEM_TABLES' - Force use of system.query.history table (requires SQL warehouse and SELECT permission on system.query.history). 'API' - Force use of REST API endpoints for query history (legacy method, may have limitations with multiple workspaces)." + }, + "emit_siblings": { + "default": true, + "description": "Whether to emit siblings relation with corresponding delta-lake platform's table. If enabled, this will also ingest the corresponding delta-lake table.", + "title": "Emit Siblings", + "type": "boolean" + }, + "delta_lake_options": { + "$ref": "#/$defs/DeltaLakeDetails", + "default": { + "platform_instance_name": null, + "env": "PROD" + }, + "description": "Details about the delta lake, incase to emit siblings" + }, + "include_ml_model_aliases": { + "default": false, + "description": "Whether to include ML model aliases in the ingestion.", + "title": "Include Ml Model Aliases", + "type": "boolean" + }, + "ml_model_max_results": { + "default": 1000, + "description": "Maximum number of ML models to ingest.", + "minimum": 0, + "title": "Ml Model Max Results", + "type": "integer" + }, + "include_hive_metastore": { + "default": true, + "description": "Whether to ingest legacy `hive_metastore` catalog. This requires executing queries on SQL warehouse.", + "title": "Include Hive Metastore", + "type": "boolean" + }, + "workspace_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the workspace. Default to deployment name present in workspace_url", + "title": "Workspace Name" + } + }, + "required": [ + "workspace_url" + ], + "title": "UnityCatalogSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Advanced + +##### Multiple Databricks Workspaces + +If you have multiple databricks workspaces **that point to the same Unity Catalog metastore**, our suggestion is to use separate recipes for ingesting the workspace-specific Hive Metastore catalog and Unity Catalog metastore's information schema. + +To ingest Hive metastore information schema + +- Setup one ingestion recipe per workspace +- Use platform instance equivalent to workspace name +- Ingest only hive_metastore catalog in the recipe using config `catalogs: ["hive_metastore"]` + +To ingest Unity Catalog information schema + +- Disable hive metastore catalog ingestion in the recipe using config `include_hive_metastore: False` +- Ideally, just ingest from one workspace +- To ingest from both workspaces (e.g. if each workspace has different permissions and therefore restricted view of the UC metastore): + - Use same platform instance for all workspaces using same UC metastore + - Ingest usage from only one workspace (you lose usage from other workspace) + - Use filters to only ingest each catalog once, but shouldn’t be necessary + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +#### No data lineage captured or missing lineage + +Check that you meet the [Unity Catalog lineage requirements](https://docs.databricks.com/data-governance/unity-catalog/data-lineage.html#requirements). + +Also check the [Unity Catalog limitations](https://docs.databricks.com/data-governance/unity-catalog/data-lineage.html#limitations) to make sure that lineage would be expected to exist in this case. + +#### Lineage extraction is too slow + +Unity Catalog REST API requires one call per table (table lineage) and one call per column (column lineage). To improve performance, disable column lineage with `include_column_lineage: false`. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.unity.source.UnityCatalogSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/unity/source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Databricks, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub-documents.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub-documents.md new file mode 100644 index 00000000..9181c07e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub-documents.md @@ -0,0 +1,1303 @@ +--- +sidebar_position: 17 +title: DataHubDocuments +slug: /generated/ingestion/sources/datahub-documents +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHubDocuments + +## Overview + +Datahub Documents is a documentation or collaboration platform. Learn more in the [official Datahub Documents documentation](https://datahub.com/docs/). + +The DataHub integration for Datahub Documents covers document/workspace entities and hierarchy context for knowledge assets. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +Process Document entities from DataHub and generate semantic embeddings for semantic search. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `datahub-documents` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | + +### Overview + +The DataHub Documents source processes Document entities already stored in DataHub and enriches them with semantic embeddings for semantic search. This source is designed to work with DataHub's native Document entities that have been created via GraphQL, Python SDK, or other ingestion sources (like Notion, Confluence, etc.). + +#### Key Features + +##### 1. Smart Defaults with Server Configuration Sync + +The source **automatically fetches embedding configuration from your DataHub server**, ensuring perfect alignment: + +- Connects using environment variables (`DATAHUB_GMS_URL`, `DATAHUB_GMS_TOKEN`) +- Fetches embedding provider config (provider, model, region, dimensions) +- Validates local config against server (if provided) +- Works with minimal `config: {}` in your recipe + +##### 2. Dual Operating Modes + +**Event-Driven Mode (Recommended):** + +- Processes documents in real-time from Kafka MCL events +- Only reprocesses when document content changes +- Efficient for continuous updates +- Automatically falls back to batch mode on first run + +**Batch Mode:** + +- Fetches all documents via GraphQL +- Good for initial setup or periodic full refreshes +- Processes all matching documents + +##### 3. Incremental Processing + +- Tracks document content hashes to skip unchanged documents +- Uses DataHub's stateful ingestion framework +- Reduces processing time and API costs +- Force reprocess option available + +##### 4. Flexible Platform Filtering + +- Process all NATIVE documents (default) +- Filter by specific platforms (e.g., `["notion", "confluence"]`) +- Process all documents regardless of source type + +#### Related Documentation + +- [Semantic Search Configuration](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/docs/how-to/semantic-search-configuration) - **Start Here** +- [Notion Source](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/docs/generated/ingestion/sources/notion) - Example document source with embeddings +- [AWS Bedrock Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/embeddings.html) - Embedding models +- [AWS Bedrock Pricing](https://aws.amazon.com/bedrock/pricing/) - Cost estimation +- [DataHub Developer Guides](/docs/developers) - Additional developer documentation + +### Prerequisites + +#### 1. DataHub Server Configuration + +**You MUST configure semantic search on your DataHub server before using this source.** + +See the [Semantic Search Configuration Guide](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/docs/how-to/semantic-search-configuration) for complete setup instructions. + +Required server configuration: + +- OpenSearch 2.17+ with k-NN plugin enabled +- AWS Bedrock or Cohere embedding provider configured +- Semantic search enabled in `application.yml` + +#### 2. Document Entities in DataHub + +This source processes existing Document entities. Documents can be created through: + +- **GraphQL API**: Create documents programmatically +- **Python SDK**: Use `datahub.sdk.document.Document` +- **External Sources**: Notion, Confluence, SharePoint sources that emit Document entities + +#### 3. AWS Credentials (for Bedrock) + +If using AWS Bedrock for embeddings: + +- IAM permissions for `bedrock:InvokeModel` +- AWS credentials configured (env vars, instance role, or ECS task role) +- Bedrock model access enabled in AWS Console + +#### 4. Environment Variables + +```bash +# Required for DataHub connection +export DATAHUB_GMS_URL="http://localhost:8080" +export DATAHUB_GMS_TOKEN="your-token-here" + +# Optional: AWS credentials (if not using instance/task roles) +export AWS_PROFILE="datahub-dev" +# OR +export AWS_ACCESS_KEY_ID="your-key" +export AWS_SECRET_ACCESS_KEY="your-secret" +export AWS_REGION="us-west-2" +``` + + +### Install the Plugin +```shell +pip install 'acryl-datahub[datahub-documents]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: datahub-documents + config: + # Minimal configuration - uses smart defaults + # Automatically connects to DataHub using DATAHUB_GMS_URL and DATAHUB_GMS_TOKEN environment variables + # Fetches embedding configuration from server + # Enables event-driven mode and incremental processing + # No sink needed when deploying to DataHub (writes back to itself) + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
max_documents
integer
| Maximum number of documents to process per ingestion run. The job will stop and fail with an error once this limit is reached. Set to 0 or -1 to disable the limit.
Default: 10000
| +|
min_text_length
integer
| Minimum text length in characters to process (shorter documents are skipped)
Default: 50
| +|
partition_strategy
string
| Text partitioning strategy. Currently only 'markdown' is supported. This field is included in the document hash to trigger reprocessing if the strategy changes.
Default: markdown
| +|
skip_empty_text
boolean
| Skip documents with no text content
Default: True
| +|
chunking
ChunkingConfig
| Chunking strategy configuration. | +|
chunking.combine_text_under_n_chars
integer
| Combine chunks smaller than this size
Default: 100
| +|
chunking.max_characters
integer
| Maximum characters per chunk
Default: 500
| +|
chunking.overlap
integer
| Character overlap between chunks
Default: 0
| +|
chunking.strategy
Enum
| One of: "basic", "by_title"
Default: by_title
| +|
datahub
DataHubConnectionConfig
| DataHub connection configuration.

Defaults are loaded in this priority:
1. Explicitly configured values in the recipe
2. Environment variables (DATAHUB_GMS_URL, DATAHUB_GMS_TOKEN)
3. ~/.datahubenv file (created by `datahub init`)
4. Hardcoded defaults (http://localhost:8080, no token) | +|
datahub.server
string
| DataHub GMS server URL (defaults to DATAHUB_GMS_URL env var, ~/.datahubenv, or localhost:8080) | +|
datahub.token
One of string(password), null
| DataHub API token for authentication (defaults to DATAHUB_GMS_TOKEN env var or ~/.datahubenv) | +|
document_urns
One of array, null
| Specific document URNs to process (if None, process all matching platforms)
Default: None
| +|
document_urns.string
string
| | +|
embedding
EmbeddingConfig
| Embedding generation configuration.

Default behavior: Fetches configuration from DataHub server automatically.
Override behavior: Validates local config against server when explicitly set. | +|
embedding.allow_local_embedding_config
boolean
| BREAK-GLASS: Allow local config without server validation. NOT RECOMMENDED - may break semantic search.
Default: False
| +|
embedding.api_key
One of string(password), null
| API key for Cohere (not needed for Bedrock with IAM roles)
Default: None
| +|
embedding.aws_region
One of string, null
| AWS region for Bedrock. If not set, loads from server.
Default: None
| +|
embedding.batch_size
integer
| Batch size for embedding API calls
Default: 25
| +|
embedding.documents_per_minute
integer
| Maximum number of documents to embed per minute when rate_limit is enabled.
Default: 300
| +|
embedding.input_type
One of string, null
| Input type for Cohere embeddings
Default: search_document
| +|
embedding.model
One of string, null
| Model name. If not set, loads from server.
Default: None
| +|
embedding.model_embedding_key
One of string, null
| Storage key for embeddings (e.g., 'cohere_embed_v3'). Required if overriding server config. If not set, loads from server.
Default: None
| +|
embedding.provider
One of Enum, null
| Embedding provider (bedrock uses AWS, cohere/openai use API key). If not set, loads from server.
Default: None
| +|
embedding.rate_limit
boolean
| Enable rate limiting for embedding API calls.
Default: True
| +|
event_mode
EventModeConfig
| Event-driven mode configuration. | +|
event_mode.consumer_id
One of string, null
| Consumer ID for offset tracking (defaults to 'datahub-documents-{pipeline_name}')
Default: None
| +|
event_mode.enabled
boolean
| Enable event-driven mode (polls MCL events instead of GraphQL batch)
Default: False
| +|
event_mode.idle_timeout_seconds
integer
| Exit after this many seconds with no new events (incremental batch mode)
Default: 30
| +|
event_mode.lookback_days
One of integer, null
| Number of days to look back for events on first run (None means start from latest)
Default: None
| +|
event_mode.poll_limit
integer
| Maximum number of events to fetch per poll
Default: 100
| +|
event_mode.poll_timeout_seconds
integer
| Timeout for each poll request
Default: 2
| +|
event_mode.reset_offsets
boolean
| Reset consumer offsets to start from beginning
Default: False
| +|
event_mode.topics
array
| Topics to consume for document changes
Default: ['MetadataChangeLog_Versioned_v1']
| +|
event_mode.topics.string
string
| | +|
incremental
IncrementalConfig
| Incremental processing configuration. | +|
incremental.enabled
boolean
| Only process documents whose text content has changed (tracks content hash). Uses stateful ingestion when enabled. The state_file_path option is deprecated and ignored when stateful ingestion is enabled.
Default: True
| +|
incremental.force_reprocess
boolean
| Force reprocess all documents regardless of content hash
Default: False
| +|
incremental.state_file_path
One of string, null
| [DEPRECATED] Path to state file. This option is ignored when stateful ingestion is enabled. State is now managed through DataHub's stateful ingestion framework.
Default: None
| +|
platform_filter
One of array, null
| Filter documents by platforms. Default (None): Process all NATIVE documents (sourceType=NATIVE) regardless of platform. To include external documents from specific platforms, add them here (e.g., ['notion', 'confluence']). This will process NATIVE documents + EXTERNAL documents from the specified platforms. Use ['*'] or ['ALL'] to process all documents regardless of source type or platform.
Default: None
| +|
platform_filter.string
string
| | +|
stateful_ingestion
DocumentChunkingStatefulIngestionConfig
| Configuration for document chunking stateful ingestion. | +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "ChunkingConfig": { + "additionalProperties": false, + "description": "Chunking strategy configuration.", + "properties": { + "strategy": { + "default": "by_title", + "description": "Chunking strategy to use", + "enum": [ + "basic", + "by_title" + ], + "title": "Strategy", + "type": "string" + }, + "max_characters": { + "default": 500, + "description": "Maximum characters per chunk", + "title": "Max Characters", + "type": "integer" + }, + "overlap": { + "default": 0, + "description": "Character overlap between chunks", + "title": "Overlap", + "type": "integer" + }, + "combine_text_under_n_chars": { + "default": 100, + "description": "Combine chunks smaller than this size", + "title": "Combine Text Under N Chars", + "type": "integer" + } + }, + "title": "ChunkingConfig", + "type": "object" + }, + "DataHubConnectionConfig": { + "additionalProperties": false, + "description": "DataHub connection configuration.\n\nDefaults are loaded in this priority:\n1. Explicitly configured values in the recipe\n2. Environment variables (DATAHUB_GMS_URL, DATAHUB_GMS_TOKEN)\n3. ~/.datahubenv file (created by `datahub init`)\n4. Hardcoded defaults (http://localhost:8080, no token)", + "properties": { + "server": { + "description": "DataHub GMS server URL (defaults to DATAHUB_GMS_URL env var, ~/.datahubenv, or localhost:8080)", + "title": "Server", + "type": "string" + }, + "token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "description": "DataHub API token for authentication (defaults to DATAHUB_GMS_TOKEN env var or ~/.datahubenv)", + "title": "Token" + } + }, + "title": "DataHubConnectionConfig", + "type": "object" + }, + "DocumentChunkingStatefulIngestionConfig": { + "additionalProperties": false, + "description": "Configuration for document chunking stateful ingestion.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + } + }, + "title": "DocumentChunkingStatefulIngestionConfig", + "type": "object" + }, + "EmbeddingConfig": { + "additionalProperties": false, + "description": "Embedding generation configuration.\n\nDefault behavior: Fetches configuration from DataHub server automatically.\nOverride behavior: Validates local config against server when explicitly set.", + "properties": { + "provider": { + "anyOf": [ + { + "enum": [ + "bedrock", + "cohere", + "openai" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Embedding provider (bedrock uses AWS, cohere/openai use API key). If not set, loads from server.", + "title": "Provider" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Model name. If not set, loads from server.", + "title": "Model" + }, + "model_embedding_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Storage key for embeddings (e.g., 'cohere_embed_v3'). Required if overriding server config. If not set, loads from server.", + "title": "Model Embedding Key" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region for Bedrock. If not set, loads from server.", + "title": "Aws Region" + }, + "api_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "API key for Cohere (not needed for Bedrock with IAM roles)", + "title": "Api Key" + }, + "batch_size": { + "default": 25, + "description": "Batch size for embedding API calls", + "title": "Batch Size", + "type": "integer" + }, + "input_type": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "search_document", + "description": "Input type for Cohere embeddings", + "title": "Input Type" + }, + "rate_limit": { + "default": true, + "description": "Enable rate limiting for embedding API calls.", + "title": "Rate Limit", + "type": "boolean" + }, + "documents_per_minute": { + "default": 300, + "description": "Maximum number of documents to embed per minute when rate_limit is enabled.", + "exclusiveMinimum": 0, + "title": "Documents Per Minute", + "type": "integer" + }, + "allow_local_embedding_config": { + "default": false, + "description": "BREAK-GLASS: Allow local config without server validation. NOT RECOMMENDED - may break semantic search.", + "title": "Allow Local Embedding Config", + "type": "boolean" + } + }, + "title": "EmbeddingConfig", + "type": "object" + }, + "EventModeConfig": { + "additionalProperties": false, + "description": "Event-driven mode configuration.", + "properties": { + "enabled": { + "default": false, + "description": "Enable event-driven mode (polls MCL events instead of GraphQL batch)", + "title": "Enabled", + "type": "boolean" + }, + "consumer_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Consumer ID for offset tracking (defaults to 'datahub-documents-{pipeline_name}')", + "title": "Consumer Id" + }, + "topics": { + "default": [ + "MetadataChangeLog_Versioned_v1" + ], + "description": "Topics to consume for document changes", + "items": { + "type": "string" + }, + "title": "Topics", + "type": "array" + }, + "lookback_days": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number of days to look back for events on first run (None means start from latest)", + "title": "Lookback Days" + }, + "reset_offsets": { + "default": false, + "description": "Reset consumer offsets to start from beginning", + "title": "Reset Offsets", + "type": "boolean" + }, + "idle_timeout_seconds": { + "default": 30, + "description": "Exit after this many seconds with no new events (incremental batch mode)", + "title": "Idle Timeout Seconds", + "type": "integer" + }, + "poll_timeout_seconds": { + "default": 2, + "description": "Timeout for each poll request", + "title": "Poll Timeout Seconds", + "type": "integer" + }, + "poll_limit": { + "default": 100, + "description": "Maximum number of events to fetch per poll", + "title": "Poll Limit", + "type": "integer" + } + }, + "title": "EventModeConfig", + "type": "object" + }, + "IncrementalConfig": { + "additionalProperties": false, + "description": "Incremental processing configuration.", + "properties": { + "enabled": { + "default": true, + "description": "Only process documents whose text content has changed (tracks content hash). Uses stateful ingestion when enabled. The state_file_path option is deprecated and ignored when stateful ingestion is enabled.", + "title": "Enabled", + "type": "boolean" + }, + "state_file_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[DEPRECATED] Path to state file. This option is ignored when stateful ingestion is enabled. State is now managed through DataHub's stateful ingestion framework.", + "title": "State File Path" + }, + "force_reprocess": { + "default": false, + "description": "Force reprocess all documents regardless of content hash", + "title": "Force Reprocess", + "type": "boolean" + } + }, + "title": "IncrementalConfig", + "type": "object" + } + }, + "description": "Configuration for DataHub Documents Source.", + "properties": { + "stateful_ingestion": { + "$ref": "#/$defs/DocumentChunkingStatefulIngestionConfig", + "description": "Stateful ingestion configuration. Enabled by default to support incremental mode (document hash tracking) and event mode (offset tracking)." + }, + "datahub": { + "$ref": "#/$defs/DataHubConnectionConfig", + "description": "DataHub connection configuration. Only used when running standalone (e.g., CLI ingestion). In managed ingestion (deployed sources), the connection is automatically configured from the sink." + }, + "platform_filter": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Filter documents by platforms. Default (None): Process all NATIVE documents (sourceType=NATIVE) regardless of platform. To include external documents from specific platforms, add them here (e.g., ['notion', 'confluence']). This will process NATIVE documents + EXTERNAL documents from the specified platforms. Use ['*'] or ['ALL'] to process all documents regardless of source type or platform.", + "title": "Platform Filter" + }, + "document_urns": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Specific document URNs to process (if None, process all matching platforms)", + "title": "Document Urns" + }, + "event_mode": { + "$ref": "#/$defs/EventModeConfig", + "description": "Event-driven mode configuration (polls Kafka MCL events)" + }, + "incremental": { + "$ref": "#/$defs/IncrementalConfig", + "description": "Incremental processing configuration (skip unchanged documents)" + }, + "chunking": { + "$ref": "#/$defs/ChunkingConfig", + "description": "Text chunking strategy configuration" + }, + "embedding": { + "$ref": "#/$defs/EmbeddingConfig", + "description": "Embedding generation configuration (LiteLLM with Cohere/Bedrock)" + }, + "max_documents": { + "default": 10000, + "description": "Maximum number of documents to process per ingestion run. The job will stop and fail with an error once this limit is reached. Set to 0 or -1 to disable the limit.", + "minimum": -1, + "title": "Max Documents", + "type": "integer" + }, + "partition_strategy": { + "const": "markdown", + "default": "markdown", + "description": "Text partitioning strategy. Currently only 'markdown' is supported. This field is included in the document hash to trigger reprocessing if the strategy changes.", + "title": "Partition Strategy", + "type": "string" + }, + "skip_empty_text": { + "default": true, + "description": "Skip documents with no text content", + "title": "Skip Empty Text", + "type": "boolean" + }, + "min_text_length": { + "default": 50, + "description": "Minimum text length in characters to process (shorter documents are skipped)", + "title": "Min Text Length", + "type": "integer" + } + }, + "title": "DataHubDocumentsSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +:::tip Quick Start: Auto-Deploy for Semantic Search + +To enable automatic semantic search indexing for your documents, deploy this source to DataHub with a simple command: + +```bash +# Create minimal recipe and deploy with hourly schedule +cat > /tmp/datahub-docs.yml << 'EOF' +source: + type: datahub-documents + config: {} +EOF + +datahub ingest deploy -c /tmp/datahub-docs.yml --name "document-embeddings" --schedule "0 * * * *" +``` + +This creates a managed ingestion source in DataHub that automatically processes documents every hour and generates embeddings for semantic search. + +**What this does:** + +- ✅ Deploys ingestion recipe to DataHub +- ✅ Runs hourly (cron: `0 * * * *`) to keep embeddings up-to-date +- ✅ Uses event-driven mode (only processes changed documents) +- ✅ Auto-configures from server (no manual embedding setup needed) + +**Alternative schedules:** + +```bash +# Every 15 minutes: "*/15 * * * *" +# Every 6 hours: "0 */6 * * *" +# Daily at 2 AM: "0 2 * * *" +``` + +> **Note:** In future DataHub versions, GMS will run this automatically. For now, manual deployment is required. + +::: + +#### Common Use Cases + +##### 1. Event-Driven Processing (Production) + +Process documents in real-time as they're created or updated: + +```yaml +source: + type: datahub-documents + config: + # Event mode enabled by default in recent versions + event_mode: + enabled: true + idle_timeout_seconds: 60 + +sink: + type: datahub-rest + config: {} +``` + +**When to use:** + +- Production deployments +- Continuous document updates +- Real-time semantic search needs + +##### 2. Batch Processing (Initial Load) + +Process all documents in a single run: + +```yaml +source: + type: datahub-documents + config: + event_mode: + enabled: false + + # Optional: Process specific platforms + platform_filter: ["notion", "confluence"] + +sink: + type: datahub-rest + config: {} +``` + +**When to use:** + +- Initial setup +- Periodic full refreshes +- Backfilling embeddings + +##### 3. Platform-Specific Processing + +Process documents from specific platforms only: + +```yaml +source: + type: datahub-documents + config: + # Process NATIVE documents + EXTERNAL from these platforms + platform_filter: ["notion", "confluence"] + + incremental: + enabled: true + +sink: + type: datahub-rest + config: {} +``` + +##### 4. Force Reprocessing + +Reprocess all documents regardless of content changes: + +```yaml +source: + type: datahub-documents + config: + incremental: + enabled: true + force_reprocess: true # Reprocess everything + + # Useful when: + # - Changing chunking strategy + # - Updating embedding model + # - Fixing processing issues + +sink: + type: datahub-rest + config: {} +``` + +##### 5. Custom Chunking and Embedding + +Override server configuration with local settings: + +```yaml +source: + type: datahub-documents + config: + # Custom chunking + chunking: + strategy: by_title # or 'basic' + max_characters: 1000 # Larger chunks + combine_text_under_n_chars: 200 + + # Override embedding config (validates against server) + embedding: + provider: bedrock + model: cohere.embed-english-v3 + model_embedding_key: cohere_embed_v3 + aws_region: us-west-2 + batch_size: 50 + +sink: + type: datahub-rest + config: {} +``` + +**⚠️ Warning:** Custom embedding configs are validated against the server. Mismatches will cause errors. + +#### How It Works + +##### Processing Pipeline + +``` +1. Fetch Mode Selection + ├─ Event Mode: Subscribe to Kafka MCL events + └─ Batch Mode: GraphQL query for all documents + +2. For Each Document: + ├─ Check incremental state (skip if unchanged) + ├─ Partition markdown → structured elements + ├─ Chunk elements → semantic chunks + │ ├─ by_title: Preserves document structure + │ └─ basic: Fixed-size chunks with overlap + ├─ Generate embeddings via LiteLLM + │ └─ Batches of 25 (configurable) + └─ Emit SemanticContent aspect → DataHub + +3. State Management + ├─ Batch Mode: Track document content hashes + └─ Event Mode: Track Kafka offsets +``` + +##### Event Mode Flow + +**First Run (No State):** + +1. Falls back to batch mode +2. Captures current Kafka offset BEFORE processing +3. Processes all documents +4. Saves offset to state +5. Next run continues from captured offset + +**Subsequent Runs:** + +1. Loads last committed offset from state +2. Consumes events from last position +3. Processes only changed documents +4. Updates offset after each batch +5. Exits after idle timeout (no new events) + +##### Incremental Processing + +**Content Hash Calculation:** + +```python +hash_input = { + "text": document.text, + "partition_strategy": config.partition_strategy, + "chunking_strategy": config.chunking.strategy, + "max_characters": config.chunking.max_characters, + # ... other chunking params +} +content_hash = sha256(json.dumps(hash_input)) +``` + +**When Documents Are Reprocessed:** + +- Text content changes +- Chunking configuration changes +- Partition strategy changes +- `force_reprocess: true` is set + +#### Configuration Deep Dive + +##### Platform Filtering + +The `platform_filter` setting controls which documents are processed: + +**`None` (default):** + +```yaml +platform_filter: null # or omit the field +``` + +- Processes all NATIVE documents (sourceType=NATIVE) +- Ignores EXTERNAL documents from other platforms + +**Specific Platforms:** + +```yaml +platform_filter: ["notion", "confluence"] +``` + +- Processes NATIVE documents +- PLUS EXTERNAL documents from specified platforms + +**All Documents:** + +```yaml +platform_filter: ["*"] # or ["ALL"] +``` + +- Processes ALL documents regardless of source type or platform + +##### Event Mode Configuration + +```yaml +event_mode: + enabled: true + + # Consumer ID for offset tracking + consumer_id: "datahub-documents-{pipeline_name}" # Default + + # Kafka topics to consume + topics: + - "MetadataChangeLog_Versioned_v1" + + # Lookback window for first run + lookback_days: null # null = start from latest, or specify days + + # Reset offsets to beginning (DANGEROUS - reprocesses everything) + reset_offsets: false + + # Exit after N seconds with no new events + idle_timeout_seconds: 30 + + # Kafka poll settings + poll_timeout_seconds: 2 + poll_limit: 100 +``` + +##### Chunking Strategies + +**by_title (Recommended):** + +```yaml +chunking: + strategy: by_title + max_characters: 500 + combine_text_under_n_chars: 100 +``` + +- Preserves document structure +- Groups text under section headers +- Combines small chunks intelligently +- Better semantic coherence + +**basic:** + +```yaml +chunking: + strategy: basic + max_characters: 500 + overlap: 50 # Character overlap between chunks +``` + +- Simple fixed-size chunks +- Configurable overlap +- No structure awareness + +##### Embedding Configuration + +**Default (Fetch from Server):** + +```yaml +embedding: {} # or omit entirely +``` + +- Automatically fetches config from server +- Ensures alignment with server's semantic search +- **Recommended for production** + +**Override (Validated Against Server):** + +```yaml +embedding: + provider: bedrock # bedrock, cohere, openai + model: cohere.embed-english-v3 + model_embedding_key: cohere_embed_v3 # Must match server! + aws_region: us-west-2 + batch_size: 25 + input_type: search_document # Cohere-specific +``` + +- Validates that config matches server +- Fails if mismatch detected +- Prevents broken semantic search + +**Break-Glass Override (NOT RECOMMENDED):** + +```yaml +embedding: + allow_local_embedding_config: true + provider: bedrock + model: cohere.embed-english-v3 + # ... other settings +``` + +- Bypasses server validation +- **May break semantic search** +- Only use for debugging or special cases + +##### Stateful Ingestion + +```yaml +stateful_ingestion: + enabled: true # Enabled by default + + # State backend configuration + state_provider: + type: datahub # Store state in DataHub + config: + datahub_api: + server: "http://localhost:8080" + token: "${DATAHUB_TOKEN}" + + # Ignore previous state (fresh start) + ignore_old_state: false + + # Don't commit new state (dry run) + ignore_new_state: false +``` + +#### Performance Tuning + +##### Batch Size + +```yaml +embedding: + batch_size: 25 # Default + # Increase for faster processing (if provider supports): + # - Cohere: Up to 96 + # - Bedrock: Up to 100 (but rate-limited) +``` + +##### Event Mode Settings + +```yaml +event_mode: + poll_limit: 100 # Fetch up to 100 events per poll + # Increase for high-volume scenarios: + poll_limit: 500 # Process more events per batch +``` + +##### Filtering + +```yaml +# Skip short or empty documents +skip_empty_text: true +min_text_length: 50 # Characters + +# Process fewer documents +platform_filter: ["notion"] # Only one platform +document_urns: # Specific documents only + - "urn:li:document:abc123" +``` + +#### Monitoring and Observability + +##### Report Metrics + +The source reports the following metrics: + +```python +report = { + "num_documents_fetched": 100, # Total documents fetched + "num_documents_processed": 85, # Successfully processed + "num_documents_skipped": 15, # Skipped (various reasons) + "num_documents_skipped_unchanged": 10, # Unchanged content + "num_documents_skipped_empty": 5, # Empty or too short + "num_chunks_created": 425, # Total chunks generated + "num_embeddings_generated": 425, # Total embeddings + "processing_errors": [] # List of errors +} +``` + +##### Logging + +Enable debug logging for detailed insights: + +```yaml +# In your ingestion recipe +source: + type: datahub-documents + config: + # ... your config +# Set log level via environment variable +# export DATAHUB_DEBUG=true +``` + +Look for these log messages: + +- `"Loading embedding configuration from DataHub server..."` +- `"✓ Loaded embedding configuration from server"` +- `"Incremental mode enabled, state file: ..."` +- `"Skipping document {urn} (unchanged content hash)"` + +#### Cost Estimation + +##### AWS Bedrock Pricing (Cohere Embed v3) + +As of December 2024 in us-west-2: + +- **$0.0001 per 1,000 input tokens** (~750 words) + +**Example Costs:** + +**One-time Processing:** + +- 1,000 documents × 500 tokens each = 500,000 tokens = **$0.05** +- 10,000 documents × 500 tokens each = 5M tokens = **$0.50** +- 100,000 documents × 500 tokens each = 50M tokens = **$5.00** + +**Incremental Updates (Event Mode):** + +- 100 changed documents/day × 500 tokens = 50,000 tokens/day +- Monthly: 1.5M tokens = **$0.15/month** + +**Query Embeddings (GMS):** + +- Separate from this source (handled by GMS at search time) +- ~50 tokens per search query +- 10,000 queries = **$0.05** + +### Limitations + +#### Processing Limitations + +- **Text Only:** Only processes `Document.text` field (markdown format expected) +- **No Binary Content:** Images, PDFs, etc. must be converted to text first +- **Markdown Partitioning:** Uses `unstructured.partition.md` which may not handle all markdown variants + +#### Platform Filtering + +- **Source Type Required:** Documents must have `sourceType` field (defaults to NATIVE if missing) +- **Platform Identification:** Relies on `dataPlatformInstance` or URL-based platform extraction + +#### State Management + +- **State Size:** State file grows with number of documents (includes hash for each) +- **State Backend:** Requires DataHub or file-based state provider + +#### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +#### Issue: "Semantic search is not enabled on the DataHub server" + +**Cause:** Server does not have semantic search configured. + +**Solution:** + +1. Configure semantic search on your DataHub server first +2. See [Semantic Search Configuration Guide](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/docs/how-to/semantic-search-configuration) +3. Verify `ELASTICSEARCH_SEMANTIC_SEARCH_ENABLED=true` in server config + +#### Issue: "Server does not support semantic search configuration API" + +**Cause:** Old DataHub server version (pre-v0.14.0). + +**Solutions:** + +**Option 1 (Recommended):** Upgrade DataHub server to v0.14.0+ + +**Option 2:** Provide local embedding config: + +```yaml +embedding: + provider: bedrock + model: cohere.embed-english-v3 + model_embedding_key: cohere_embed_v3 + aws_region: us-west-2 +``` + +#### Issue: Embedding Configuration Validation Fails + +**Error:** + +``` +Embedding configuration mismatch with server: +- Model: local='cohere.embed-english-v3', server='amazon.titan-embed-text-v1' +``` + +**Cause:** Local config doesn't match server configuration. + +**Solution:** + +1. Either remove local embedding config (use server config) +2. Or update server config to match local settings +3. Or update local config to match server + +#### Issue: No Documents Being Processed + +**Possible Causes:** + +1. **Platform Filter Too Restrictive:** + + ```yaml + # If you have NATIVE documents but filter for external platforms: + platform_filter: ["notion"] # Won't process NATIVE documents! + + # Solution: Remove filter or use null + platform_filter: null + ``` + +2. **All Documents Unchanged:** + + - Check incremental mode is working correctly + - Force reprocess if needed: `incremental.force_reprocess: true` + +3. **Documents Have No Text:** + - Verify documents have content in `Document.text` field + - Check `min_text_length` threshold + +#### Issue: Event Mode Not Working + +**Symptoms:** Falls back to batch mode every run. + +**Possible Causes:** + +1. **Stateful Ingestion Disabled:** + + ```yaml + stateful_ingestion: + enabled: true # Must be enabled for event mode + ``` + +2. **Kafka Connection Issues:** + + - Check DataHub Kafka is accessible + - Verify network connectivity + - Check Kafka broker configuration + +3. **State Provider Misconfigured:** + ```yaml + stateful_ingestion: + state_provider: + type: datahub + config: + datahub_api: + server: "http://correct-host:8080" # Correct URL + ``` + +#### Issue: AWS Credentials Error + +**Error:** + +``` +Unable to load credentials from any provider in the chain +``` + +**Solutions:** + +1. **Verify AWS_PROFILE:** + + ```bash + export AWS_PROFILE=datahub-dev + cat ~/.aws/credentials # Check profile exists + ``` + +2. **For EC2 Instance Role:** + + ```bash + # Check instance role is attached + curl http://169.254.169.254/latest/meta-data/iam/security-credentials/ + ``` + +3. **For ECS Task Role:** + - Verify task definition has correct IAM role + - Check ECS task logs for IAM-related errors + +#### Issue: Slow Processing + +**Optimization Strategies:** + +1. **Increase Batch Size:** + + ```yaml + embedding: + batch_size: 50 # Up from default 25 + ``` + +2. **Use Event Mode:** + + - Only processes changed documents + - Much faster than batch mode for updates + +3. **Filter Documents:** + + ```yaml + platform_filter: ["notion"] # Process fewer platforms + min_text_length: 100 # Skip short documents + ``` + +4. **Optimize Chunking:** + ```yaml + chunking: + max_characters: 1000 # Larger chunks = fewer embeddings + ``` + +#### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.datahub_documents.datahub_documents_source.DataHubDocumentsSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/datahub_documents/datahub_documents_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for DataHubDocuments, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub.md new file mode 100644 index 00000000..d40b357d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahub.md @@ -0,0 +1,582 @@ +--- +sidebar_position: 14 +title: DataHub +slug: /generated/ingestion/sources/datahub +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHub + +## Overview + +Datahub is a DataHub utility or metadata-focused integration. Learn more in the [official Datahub documentation](https://datahub.com/). + +The DataHub integration for Datahub covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `datahub` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | + +### Overview + +Migrate data from one DataHub instance to another. + +Pulls data from two locations: + +- **DataHub database**: Versioned aspects +- **DataHub Kafka**: [MCL Log](../../../../docs/what/mxe.md#metadata-change-log-mcl) timeseries aspects + +All data is first read from the database, before timeseries data is ingested from kafka. To prevent this source from potentially running forever, it will not ingest data produced after the datahub_source ingestion job is started. This stop_time is reflected in the report. + +Data from the database and kafka are read in chronological order, specifically by the createdon timestamp in the database and by kafka offset per partition. In order to properly read from the database, please ensure that the createdon column is indexed. Newly created databases should have this index, named timeIndex, by default, but older ones you may have to create yourself, with the statement: + +``` +CREATE INDEX timeIndex ON metadata_aspect_v2 (createdon); +``` + +:::warning Indexing createdon + +If you do not have this index, the source may run incredibly slowly and produce significant database load. +::: + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +Requires direct access to the database, kafka broker, and kafka schema registry +of the source DataHub instance. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[datahub]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +pipeline_name: datahub_source_1 +datahub_api: + server: "http://localhost:8080" # Migrate data from DataHub instance on localhost:8080 + token: "" +source: + type: datahub + config: + include_all_versions: false + database_connection: + scheme: "mysql+pymysql" # or "postgresql+psycopg2" for Postgres + host_port: ":" + username: "" + password: "" + database: "" + kafka_connection: + bootstrap: ":9092" + schema_registry_url: ":8081" + stateful_ingestion: + enabled: true + ignore_old_state: false + urn_pattern: + deny: + # Ignores all datahub metadata where the urn matches the regex + - ^denied.urn.* + allow: + # Ingests all datahub metadata where the urn matches the regex. + - ^allowed.urn.* + +flags: + set_system_metadata: false # Replicate system metadata + +# Here, we write to a DataHub instance +# You can also use a different sink, e.g. to write the data to a file instead +sink: + type: datahub-rest + config: + server: "" + token: "" + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
commit_state_interval
One of integer, null
| Number of records to process before committing state
Default: 1000
| +|
commit_with_parse_errors
boolean
| Whether to update createdon timestamp and kafka offset despite parse errors. Enable if you want to ignore the errors.
Default: False
| +|
database_query_batch_size
integer
| Number of records to fetch from the database at a time
Default: 10000
| +|
database_table_name
string
| Name of database table containing all versioned aspects
Default: metadata_aspect_v2
| +|
drop_duplicate_schema_fields
boolean
| Whether to drop duplicate schema fields in the schemaMetadata aspect. Useful if the source system has duplicate field paths in the db, but we're pushing to a system with server-side duplicate checking.
Default: False
| +|
include_all_versions
boolean
| If enabled, include all versions of each aspect. Otherwise, only include the latest version of each aspect.
Default: False
| +|
include_soft_deleted_entities
boolean
| If enabled, include entities that have been soft deleted. Otherwise, include all entities regardless of removal status.
Default: True
| +|
kafka_topic_name
string
| Name of kafka topic containing timeseries MCLs
Default: MetadataChangeLog_Timeseries_v1
| +|
preserve_system_metadata
boolean
| Copy system metadata from the source system
Default: True
| +|
query_timeout
One of integer, null
| Timeout for each query in seconds.
Default: None
| +|
database_connection
One of SQLAlchemyConnectionConfig, null
| Database connection config
Default: None
| +|
database_connection.host_port ❓
string
| host URL | +|
database_connection.scheme ❓
string
| scheme | +|
database_connection.database
One of string, null
| database (catalog)
Default: None
| +|
database_connection.options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
database_connection.password
One of string(password), null
| password
Default: None
| +|
database_connection.sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
database_connection.username
One of string, null
| username
Default: None
| +|
exclude_aspects
array
| Aspect names to exclude from entities that are being ingested. Note: This only makes sense for entities you want to ingest but without certain aspects. To completely exclude entity types, use 'urn_pattern.deny' instead. Warning: Excluding key aspects while keeping others can create invalid entities.
Default: ['datahubIngestionRunSummary', 'testResults', 'dat...
| +|
exclude_aspects.string
string
| | +|
kafka_connection
One of KafkaConsumerConnectionConfig, null
| Kafka connection config
Default: None
| +|
kafka_connection.bootstrap
string
|
Default: localhost:9092
| +|
kafka_connection.client_timeout_seconds
integer
| The request timeout used when interacting with the Kafka APIs.
Default: 60
| +|
kafka_connection.consumer_config
object
| Extra consumer config serialized as JSON. These options will be passed into Kafka's DeserializingConsumer. See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md . | +|
kafka_connection.schema_registry_config
object
| Extra schema registry config serialized as JSON. These options will be passed into Kafka's SchemaRegistryClient. https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html?#schemaregistryclient | +|
kafka_connection.schema_registry_url
string
| Schema registry URL. Can be overridden with KAFKA_SCHEMAREGISTRY_URL environment variable, or will use DATAHUB_GMS_BASE_PATH if not set. | +|
urn_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
urn_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
StatefulIngestionConfig
| Basic Stateful Ingestion Specific Configuration for any source. | +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "KafkaConsumerConnectionConfig": { + "additionalProperties": false, + "description": "Configuration class for holding connectivity information for Kafka consumers", + "properties": { + "bootstrap": { + "default": "localhost:9092", + "title": "Bootstrap", + "type": "string" + }, + "schema_registry_url": { + "description": "Schema registry URL. Can be overridden with KAFKA_SCHEMAREGISTRY_URL environment variable, or will use DATAHUB_GMS_BASE_PATH if not set.", + "title": "Schema Registry Url", + "type": "string" + }, + "schema_registry_config": { + "additionalProperties": true, + "description": "Extra schema registry config serialized as JSON. These options will be passed into Kafka's SchemaRegistryClient. https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html?#schemaregistryclient", + "title": "Schema Registry Config", + "type": "object" + }, + "client_timeout_seconds": { + "default": 60, + "description": "The request timeout used when interacting with the Kafka APIs.", + "title": "Client Timeout Seconds", + "type": "integer" + }, + "consumer_config": { + "additionalProperties": true, + "description": "Extra consumer config serialized as JSON. These options will be passed into Kafka's DeserializingConsumer. See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md .", + "title": "Consumer Config", + "type": "object" + } + }, + "title": "KafkaConsumerConnectionConfig", + "type": "object" + }, + "SQLAlchemyConnectionConfig": { + "additionalProperties": false, + "properties": { + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "description": "host URL", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "scheme": { + "description": "scheme", + "title": "Scheme", + "type": "string" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + } + }, + "required": [ + "host_port", + "scheme" + ], + "title": "SQLAlchemyConnectionConfig", + "type": "object" + }, + "StatefulIngestionConfig": { + "additionalProperties": false, + "description": "Basic Stateful Ingestion Specific Configuration for any source.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + } + }, + "title": "StatefulIngestionConfig", + "type": "object" + } + }, + "properties": { + "stateful_ingestion": { + "$ref": "#/$defs/StatefulIngestionConfig", + "default": { + "enabled": true, + "max_checkpoint_state_size": 16777216, + "state_provider": { + "config": {}, + "type": "datahub" + }, + "ignore_old_state": false, + "ignore_new_state": false + }, + "description": "Stateful Ingestion Config" + }, + "database_connection": { + "anyOf": [ + { + "$ref": "#/$defs/SQLAlchemyConnectionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Database connection config" + }, + "kafka_connection": { + "anyOf": [ + { + "$ref": "#/$defs/KafkaConsumerConnectionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kafka connection config" + }, + "include_all_versions": { + "default": false, + "description": "If enabled, include all versions of each aspect. Otherwise, only include the latest version of each aspect. ", + "title": "Include All Versions", + "type": "boolean" + }, + "include_soft_deleted_entities": { + "default": true, + "description": "If enabled, include entities that have been soft deleted. Otherwise, include all entities regardless of removal status. ", + "title": "Include Soft Deleted Entities", + "type": "boolean" + }, + "exclude_aspects": { + "default": [ + "datahubIngestionRunSummary", + "testResults", + "datahubIngestionCheckpoint" + ], + "description": "Aspect names to exclude from entities that are being ingested. Note: This only makes sense for entities you want to ingest but without certain aspects. To completely exclude entity types, use 'urn_pattern.deny' instead. Warning: Excluding key aspects while keeping others can create invalid entities.", + "items": { + "type": "string" + }, + "title": "Exclude Aspects", + "type": "array", + "uniqueItems": true + }, + "database_query_batch_size": { + "default": 10000, + "description": "Number of records to fetch from the database at a time", + "title": "Database Query Batch Size", + "type": "integer" + }, + "database_table_name": { + "default": "metadata_aspect_v2", + "description": "Name of database table containing all versioned aspects", + "title": "Database Table Name", + "type": "string" + }, + "kafka_topic_name": { + "default": "MetadataChangeLog_Timeseries_v1", + "description": "Name of kafka topic containing timeseries MCLs", + "title": "Kafka Topic Name", + "type": "string" + }, + "commit_state_interval": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1000, + "description": "Number of records to process before committing state", + "title": "Commit State Interval" + }, + "commit_with_parse_errors": { + "default": false, + "description": "Whether to update createdon timestamp and kafka offset despite parse errors. Enable if you want to ignore the errors.", + "title": "Commit With Parse Errors", + "type": "boolean" + }, + "urn_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [ + "urn:li:dataHubIngestionSource:.*", + "urn:li:dataHubSecret:.*", + "urn:li:globalSettings:.*", + "urn:li:dataHubExecutionRequest:.*" + ], + "ignoreCase": true + }, + "description": "URN patterns to filter entities. Defaults exclude environment-specific entities (ingestion sources, secrets, settings) to prevent copying encrypted credentials and creating broken entities. Recommended to keep these defaults when customizing." + }, + "drop_duplicate_schema_fields": { + "default": false, + "description": "Whether to drop duplicate schema fields in the schemaMetadata aspect. Useful if the source system has duplicate field paths in the db, but we're pushing to a system with server-side duplicate checking.", + "title": "Drop Duplicate Schema Fields", + "type": "boolean" + }, + "query_timeout": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Timeout for each query in seconds. ", + "title": "Query Timeout" + }, + "preserve_system_metadata": { + "default": true, + "description": "Copy system metadata from the source system", + "title": "Preserve System Metadata", + "type": "boolean" + } + }, + "title": "DataHubSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Stateful Ingestion + +The source checkpoints by database `createdon` and Kafka offsets so interrupted runs can resume without restarting from scratch. Use `stateful_ingestion.ignore_old_state` or a distinct `pipeline_name` when you want a full replay. + +On first run, the source will read from the earliest data in the database and the earliest kafka offsets. Every `commit_state_interval` (default 1000) records, the source will store a checkpoint to remember its place, i.e. the last `createdon` timestamp and kafka offsets. This allows you to stop and restart the source without losing much progress, but note that you will re-ingest some data at the start of the new run. + +If any errors are encountered in the ingestion process, e.g. we are unable to emit an aspect due to network errors, the source will keep running, but will stop committing checkpoints, unless `commit_with_parse_errors` (default false) is set. Thus, if you re-run the ingestion, you can re-ingest the data that was missed, but note it will all re-ingest all subsequent data. + +If you want to re-ingest all data, you can set a different `pipeline_name` in your recipe, or set `stateful_ingestion.ignore_old_state: true` + +##### Limitations + +- Can only pull timeseries aspects retained by Kafka, which by default lasts 90 days. +- Does not detect hard timeseries deletions, e.g. if via a datahub delete command using the CLI. Therefore, if you deleted data in this way, it will still exist in the destination instance. +- If you have a significant amount of aspects with the exact same createdon timestamp, stateful ingestion will not be able to save checkpoints partially through that timestamp. On a subsequent run, all aspects for that timestamp will be ingested. + +#### Performance + +For large migrations, ensure `metadata_aspect_v2.createdon` is indexed (`timeIndex`), enable async ingestion on the destination, and scale consumers/GMS/Elasticsearch workers as needed. + +- Enable [async ingestion](/docs/deploy/environment-vars#ingestion) +- Use standalone consumers ([mae-consumer](/docs/metadata-jobs/mae-consumer-job) and [mce-consumer](/docs/metadata-jobs/mce-consumer-job)) + - If you are migrating large amounts of data, consider scaling consumer replicas. +- Increase the number of gms pods to add redundancy and increase resilience to node evictions + - If you are migrating large amounts of data, consider increasing elasticsearch's thread count via the `ELASTICSEARCH_THREAD_COUNT` environment variable. + +#### Exclusions + +You will likely want to exclude some urn types from your ingestion, as they contain instance-specific metadata, such as settings, roles, policies, ingestion sources, and ingestion runs. For example, you will likely want to start with this: + +```yaml +source: + config: + urn_pattern: # URN pattern to ignore/include in the ingestion + deny: + # Ignores all datahub metadata where the urn matches the regex + - ^urn:li:role.* # Only exclude if you do not want to ingest roles + - ^urn:li:dataHubRole.* # Only exclude if you do not want to ingest roles + - ^urn:li:dataHubPolicy.* # Only exclude if you do not want to ingest policies + - ^urn:li:dataHubIngestionSource.* # Only exclude if you do not want to ingest ingestion sources + - ^urn:li:dataHubSecret.* + - ^urn:li:dataHubExecutionRequest.* + - ^urn:li:dataHubAccessToken.* + - ^urn:li:dataHubUpgrade.* + - ^urn:li:inviteToken.* + - ^urn:li:globalSettings.* + - ^urn:li:dataHubStepState.* +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.datahub.datahub_source.DataHubSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/datahub/datahub_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for DataHub, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubapply.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubapply.md new file mode 100644 index 00000000..7cdbbf63 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubapply.md @@ -0,0 +1,303 @@ +--- +sidebar_position: 15 +title: DataHubApply +slug: /generated/ingestion/sources/datahubapply +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHubApply + +## Overview + +DataHub Apply is a DataHub utility or metadata-focused integration. Learn more in the [official DataHub Apply documentation](https://datahub.com/docs/). + +The DataHub integration for DataHub Apply covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| ------------------------------------------ | ---------------------------------------------------- | -------------------------------------------------------- | +| Apply operation input | Metadata Change Proposal (MCP) updates | Input drives metadata updates rather than discovery. | +| Asset target list | Dataset / Container (and other supported entities) | Targets are selected explicitly in recipe configuration. | +| Ownership / domain / tag / term assignment | Ownership, Domain, GlobalTags, GlossaryTerms aspects | Applied directly to existing DataHub entities. | + + +## Module `datahub-apply` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +Capability metadata is not explicitly declared for this module. Refer to module documentation and configuration sections below. + +### Overview + +The `datahub-apply` module applies metadata changes directly to existing DataHub entities. It is useful for programmatic curation tasks such as bulk ownership, domain, tag, and glossary-term updates. + +### Prerequisites + +- Access to a DataHub instance with permissions to update target entities. +- Valid authentication configuration for the ingestion run. +- Existing target entities in DataHub for each configured apply operation. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[datahub-apply]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: datahub-apply + config: + owner_apply: + - owner_urn: "urn:li:corpuser:datahub" + assets: + - "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)" + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
domain_apply
One of array, null
| List to apply domains to assets
Default: None
| +|
domain_apply.DomainApplyConfig
DomainApplyConfig
| | +|
domain_apply.DomainApplyConfig.domain_urn
string
|
Default:
| +|
domain_apply.DomainApplyConfig.assets
array
| List of assets to apply domain hierarchically. Currently only containers and datasets are supported | +|
domain_apply.DomainApplyConfig.assets.string
string
| | +|
owner_apply
One of array, null
| List to apply owners to assets
Default: None
| +|
owner_apply.OwnerApplyConfig
OwnerApplyConfig
| | +|
owner_apply.OwnerApplyConfig.owner_urn
string
|
Default:
| +|
owner_apply.OwnerApplyConfig.assets
array
| List of assets to apply owner hierarchically. Currently only containers and datasets are supported | +|
owner_apply.OwnerApplyConfig.assets.string
string
| | +|
tag_apply
One of array, null
| List to apply tags to assets
Default: None
| +|
tag_apply.TagApplyConfig
TagApplyConfig
| | +|
tag_apply.TagApplyConfig.tag_urn
string
|
Default:
| +|
tag_apply.TagApplyConfig.assets
array
| List of assets to apply tag hierarchically. Currently only containers and datasets are supported | +|
tag_apply.TagApplyConfig.assets.string
string
| | +|
term_apply
One of array, null
| List to apply terms to assets
Default: None
| +|
term_apply.TermApplyConfig
TermApplyConfig
| | +|
term_apply.TermApplyConfig.term_urn
string
|
Default:
| +|
term_apply.TermApplyConfig.assets
array
| List of assets to apply term hierarchically. Currently only containers and datasets are supported | +|
term_apply.TermApplyConfig.assets.string
string
| | + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "DomainApplyConfig": { + "additionalProperties": false, + "properties": { + "assets": { + "description": "List of assets to apply domain hierarchically. Currently only containers and datasets are supported", + "items": { + "type": "string" + }, + "title": "Assets", + "type": "array" + }, + "domain_urn": { + "default": "", + "title": "Domain Urn", + "type": "string" + } + }, + "title": "DomainApplyConfig", + "type": "object" + }, + "OwnerApplyConfig": { + "additionalProperties": false, + "properties": { + "assets": { + "description": "List of assets to apply owner hierarchically. Currently only containers and datasets are supported", + "items": { + "type": "string" + }, + "title": "Assets", + "type": "array" + }, + "owner_urn": { + "default": "", + "title": "Owner Urn", + "type": "string" + } + }, + "title": "OwnerApplyConfig", + "type": "object" + }, + "TagApplyConfig": { + "additionalProperties": false, + "properties": { + "assets": { + "description": "List of assets to apply tag hierarchically. Currently only containers and datasets are supported", + "items": { + "type": "string" + }, + "title": "Assets", + "type": "array" + }, + "tag_urn": { + "default": "", + "title": "Tag Urn", + "type": "string" + } + }, + "title": "TagApplyConfig", + "type": "object" + }, + "TermApplyConfig": { + "additionalProperties": false, + "properties": { + "assets": { + "description": "List of assets to apply term hierarchically. Currently only containers and datasets are supported", + "items": { + "type": "string" + }, + "title": "Assets", + "type": "array" + }, + "term_urn": { + "default": "", + "title": "Term Urn", + "type": "string" + } + }, + "title": "TermApplyConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "domain_apply": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/DomainApplyConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List to apply domains to assets", + "title": "Domain Apply" + }, + "tag_apply": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/TagApplyConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List to apply tags to assets", + "title": "Tag Apply" + }, + "term_apply": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/TermApplyConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List to apply terms to assets", + "title": "Term Apply" + }, + "owner_apply": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/OwnerApplyConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List to apply owners to assets", + "title": "Owner Apply" + } + }, + "title": "DataHubApplyConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features. This module focuses on applying metadata updates rather than extracting metadata from external systems. + +### Limitations + +- This module does not discover source metadata; it only applies configured updates to existing DataHub entities. +- Incorrect URNs or selectors can lead to partial updates or no-ops. + +### Troubleshooting + +- Validate target URNs and entity existence before running large apply jobs. +- Start with a small scoped recipe to verify permissions and expected update behavior. +- Review ingestion logs for validation or authorization errors returned by DataHub APIs. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.apply.datahub_apply.DataHubApplySource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/apply/datahub_apply.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for DataHubApply, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubdebug.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubdebug.md new file mode 100644 index 00000000..af25a311 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubdebug.md @@ -0,0 +1,144 @@ +--- +sidebar_position: 16 +title: DataHubDebug +slug: /generated/ingestion/sources/datahubdebug +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHubDebug + +## Overview + +DataHub Debug is a DataHub utility or metadata-focused integration. Learn more in the [official DataHub Debug documentation](https://datahub.com/docs/). + +The DataHub integration for DataHub Debug covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| ----------------- | -------------------------------- | ---------------------------------------------------------------- | +| Debug query/input | DataHub entity/aspect inspection | Reads metadata for diagnostics. | +| Diagnostic output | Operational debugging signal | Used for troubleshooting and validation, not catalog enrichment. | + + +## Module `datahub-debug` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +Capability metadata is not explicitly declared for this module. Refer to module documentation and configuration sections below. + +### Overview + +The `datahub-debug` module provides targeted debugging and inspection capabilities for DataHub metadata operations. + +### Prerequisites + +- Access to the DataHub instance being inspected. +- Authentication with sufficient read permissions on entities/aspects involved in debugging. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[datahub-debug]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: datahub-debug + config: {} + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
dns_probe_url
One of string, null
|
Default: None
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "additionalProperties": false, + "properties": { + "dns_probe_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dns Probe Url" + } + }, + "title": "DataHubDebugSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth. This module is intended for diagnostics and validation workflows. + +### Limitations + +- This module is not a general external-source ingestion connector. +- Output is intended for debugging workflows and may require interpretation by platform operators. + +### Troubleshooting + +- Confirm authentication and API connectivity to the target DataHub environment. +- Scope debug requests narrowly first, then expand once the expected output is validated. +- Use ingestion logs to identify permission or query errors. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.debug.datahub_debug.DataHubDebugSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/debug/datahub_debug.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for DataHubDebug, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubgc.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubgc.md new file mode 100644 index 00000000..2c8ed890 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/datahubgc.md @@ -0,0 +1,696 @@ +--- +sidebar_position: 18 +title: DataHubGc +slug: /generated/ingestion/sources/datahubgc +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHubGc + +## Overview + +DataHub GC is a DataHub utility or metadata-focused integration. Learn more in the [official DataHub GC documentation](https://datahub.com/docs/). + +The DataHub integration for DataHub GC covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `datahub-gc` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +Capability metadata is not explicitly declared for this module. Refer to module documentation and configuration sections below. + +### Overview + +The DataHub Garbage Collection (GC) source is a maintenance component responsible for cleaning up various types of metadata to maintain system performance and data quality. It performs multiple cleanup tasks, each focusing on different aspects of DataHub's metadata. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[datahub-gc]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: datahub-gc + config: + dry_run: false + cleanup_expired_tokens: true + truncate_indices: true + dataprocess_cleanup: + retention_days: 10 + delete_empty_data_jobs: true + delete_empty_data_flows: true + hard_delete_entities: false + keep_last_n: 5 + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
cleanup_expired_tokens
boolean
| Whether to clean up expired tokens or not
Default: True
| +|
dry_run
boolean
| Whether to perform a dry run or not. This is only supported for dataprocess cleanup and soft deleted entities cleanup.
Default: False
| +|
truncate_index_older_than_days
integer
| Indices older than this number of days will be truncated
Default: 30
| +|
truncate_indices
boolean
| Whether to truncate elasticsearch indices or not which can be safely truncated
Default: True
| +|
truncation_sleep_between_seconds
integer
| Sleep between truncation monitoring.
Default: 30
| +|
truncation_watch_until
integer
| Wait for truncation of indices until this number of documents are left
Default: 10000
| +|
dataprocess_cleanup
DataProcessCleanupConfig
| | +|
dataprocess_cleanup.batch_size
integer
| The number of entities to get in a batch from API
Default: 500
| +|
dataprocess_cleanup.delay
One of number, null
| Delay between each batch
Default: 0.25
| +|
dataprocess_cleanup.delete_empty_data_flows
boolean
| Whether to delete Data Flows without runs
Default: False
| +|
dataprocess_cleanup.delete_empty_data_jobs
boolean
| Whether to delete Data Jobs without runs
Default: False
| +|
dataprocess_cleanup.enabled
boolean
| Whether to do data process cleanup.
Default: True
| +|
dataprocess_cleanup.hard_delete_entities
boolean
| Whether to hard delete entities
Default: False
| +|
dataprocess_cleanup.keep_last_n
One of integer, null
| Number of latest aspects to keep
Default: 5
| +|
dataprocess_cleanup.max_workers
integer
| The number of workers to use for deletion
Default: 10
| +|
dataprocess_cleanup.retention_days
One of integer, null
| Number of days to retain metadata in DataHub
Default: 10
| +|
dataprocess_cleanup.aspects_to_clean
array
| List of aspect names to clean up
Default: ['DataprocessInstance']
| +|
dataprocess_cleanup.aspects_to_clean.string
string
| | +|
execution_request_cleanup
DatahubExecutionRequestCleanupConfig
| | +|
execution_request_cleanup.batch_read_size
integer
| Number of records per read operation
Default: 100
| +|
execution_request_cleanup.enabled
boolean
| Global switch for this cleanup task
Default: True
| +|
execution_request_cleanup.keep_history_max_count
integer
| Maximum number of execution requests to keep, per ingestion source
Default: 1000
| +|
execution_request_cleanup.keep_history_max_days
integer
| Maximum number of days to keep execution requests for, per ingestion source
Default: 90
| +|
execution_request_cleanup.keep_history_min_count
integer
| Minimum number of execution requests to keep, per ingestion source
Default: 10
| +|
execution_request_cleanup.limit_entities_delete
One of integer, null
| Max number of execution requests to hard delete.
Default: 10000
| +|
execution_request_cleanup.max_read_errors
integer
| Maximum number of read errors before aborting
Default: 10
| +|
execution_request_cleanup.runtime_limit_seconds
integer
| Maximum runtime in seconds for the cleanup task
Default: 3600
| +|
soft_deleted_entities_cleanup
SoftDeletedEntitiesCleanupConfig
| | +|
soft_deleted_entities_cleanup.batch_size
integer
| The number of entities to get in a batch from GraphQL
Default: 500
| +|
soft_deleted_entities_cleanup.delay
One of number, null
| Delay between each batch
Default: 0.25
| +|
soft_deleted_entities_cleanup.enabled
boolean
| Whether to do soft deletion cleanup.
Default: True
| +|
soft_deleted_entities_cleanup.futures_max_at_time
integer
| Max number of futures to have at a time.
Default: 1000
| +|
soft_deleted_entities_cleanup.limit_entities_delete
One of integer, null
| Max number of entities to delete.
Default: 25000
| +|
soft_deleted_entities_cleanup.max_workers
integer
| The number of workers to use for deletion
Default: 10
| +|
soft_deleted_entities_cleanup.platform
One of string, null
| Platform to cleanup
Default: None
| +|
soft_deleted_entities_cleanup.query
One of string, null
| Query to filter entities
Default: None
| +|
soft_deleted_entities_cleanup.retention_days
integer
| Number of days to retain metadata in DataHub
Default: 10
| +|
soft_deleted_entities_cleanup.runtime_limit_seconds
integer
| Runtime limit in seconds
Default: 7200
| +|
soft_deleted_entities_cleanup.env
One of string, null
| Environment to cleanup
Default: None
| +|
soft_deleted_entities_cleanup.entity_types
One of array, null
| List of entity types to cleanup
Default: ['dataset', 'dashboard', 'chart', 'mlmodel', 'mlmo...
| +|
soft_deleted_entities_cleanup.entity_types.string
string
| | + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "DataProcessCleanupConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": true, + "description": "Whether to do data process cleanup.", + "title": "Enabled", + "type": "boolean" + }, + "retention_days": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 10, + "description": "Number of days to retain metadata in DataHub", + "title": "Retention Days" + }, + "aspects_to_clean": { + "default": [ + "DataprocessInstance" + ], + "description": "List of aspect names to clean up", + "items": { + "type": "string" + }, + "title": "Aspects To Clean", + "type": "array" + }, + "keep_last_n": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Number of latest aspects to keep", + "title": "Keep Last N" + }, + "delete_empty_data_jobs": { + "default": false, + "description": "Whether to delete Data Jobs without runs", + "title": "Delete Empty Data Jobs", + "type": "boolean" + }, + "delete_empty_data_flows": { + "default": false, + "description": "Whether to delete Data Flows without runs", + "title": "Delete Empty Data Flows", + "type": "boolean" + }, + "hard_delete_entities": { + "default": false, + "description": "Whether to hard delete entities", + "title": "Hard Delete Entities", + "type": "boolean" + }, + "batch_size": { + "default": 500, + "description": "The number of entities to get in a batch from API", + "title": "Batch Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "The number of workers to use for deletion", + "title": "Max Workers", + "type": "integer" + }, + "delay": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": 0.25, + "description": "Delay between each batch", + "title": "Delay" + } + }, + "title": "DataProcessCleanupConfig", + "type": "object" + }, + "DatahubExecutionRequestCleanupConfig": { + "additionalProperties": false, + "properties": { + "keep_history_min_count": { + "default": 10, + "description": "Minimum number of execution requests to keep, per ingestion source", + "title": "Keep History Min Count", + "type": "integer" + }, + "keep_history_max_count": { + "default": 1000, + "description": "Maximum number of execution requests to keep, per ingestion source", + "title": "Keep History Max Count", + "type": "integer" + }, + "keep_history_max_days": { + "default": 90, + "description": "Maximum number of days to keep execution requests for, per ingestion source", + "title": "Keep History Max Days", + "type": "integer" + }, + "batch_read_size": { + "default": 100, + "description": "Number of records per read operation", + "title": "Batch Read Size", + "type": "integer" + }, + "enabled": { + "default": true, + "description": "Global switch for this cleanup task", + "title": "Enabled", + "type": "boolean" + }, + "runtime_limit_seconds": { + "default": 3600, + "description": "Maximum runtime in seconds for the cleanup task", + "title": "Runtime Limit Seconds", + "type": "integer" + }, + "limit_entities_delete": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 10000, + "description": "Max number of execution requests to hard delete.", + "title": "Limit Entities Delete" + }, + "max_read_errors": { + "default": 10, + "description": "Maximum number of read errors before aborting", + "title": "Max Read Errors", + "type": "integer" + } + }, + "title": "DatahubExecutionRequestCleanupConfig", + "type": "object" + }, + "SoftDeletedEntitiesCleanupConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": true, + "description": "Whether to do soft deletion cleanup.", + "title": "Enabled", + "type": "boolean" + }, + "retention_days": { + "default": 10, + "description": "Number of days to retain metadata in DataHub", + "title": "Retention Days", + "type": "integer" + }, + "batch_size": { + "default": 500, + "description": "The number of entities to get in a batch from GraphQL", + "title": "Batch Size", + "type": "integer" + }, + "delay": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": 0.25, + "description": "Delay between each batch", + "title": "Delay" + }, + "max_workers": { + "default": 10, + "description": "The number of workers to use for deletion", + "title": "Max Workers", + "type": "integer" + }, + "entity_types": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": [ + "dataset", + "dashboard", + "chart", + "mlmodel", + "mlmodelGroup", + "mlfeatureTable", + "mlfeature", + "mlprimaryKey", + "dataFlow", + "dataJob", + "glossaryTerm", + "glossaryNode", + "tag", + "role", + "corpuser", + "corpGroup", + "container", + "domain", + "dataProduct", + "notebook", + "businessAttribute", + "schemaField", + "query", + "dataProcessInstance" + ], + "description": "List of entity types to cleanup", + "title": "Entity Types" + }, + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Platform to cleanup", + "title": "Platform" + }, + "env": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Environment to cleanup", + "title": "Env" + }, + "query": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Query to filter entities", + "title": "Query" + }, + "limit_entities_delete": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 25000, + "description": "Max number of entities to delete.", + "title": "Limit Entities Delete" + }, + "futures_max_at_time": { + "default": 1000, + "description": "Max number of futures to have at a time.", + "title": "Futures Max At Time", + "type": "integer" + }, + "runtime_limit_seconds": { + "default": 7200, + "description": "Runtime limit in seconds", + "title": "Runtime Limit Seconds", + "type": "integer" + } + }, + "title": "SoftDeletedEntitiesCleanupConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "dry_run": { + "default": false, + "description": "Whether to perform a dry run or not. This is only supported for dataprocess cleanup and soft deleted entities cleanup.", + "title": "Dry Run", + "type": "boolean" + }, + "cleanup_expired_tokens": { + "default": true, + "description": "Whether to clean up expired tokens or not", + "title": "Cleanup Expired Tokens", + "type": "boolean" + }, + "truncate_indices": { + "default": true, + "description": "Whether to truncate elasticsearch indices or not which can be safely truncated", + "title": "Truncate Indices", + "type": "boolean" + }, + "truncate_index_older_than_days": { + "default": 30, + "description": "Indices older than this number of days will be truncated", + "title": "Truncate Index Older Than Days", + "type": "integer" + }, + "truncation_watch_until": { + "default": 10000, + "description": "Wait for truncation of indices until this number of documents are left", + "title": "Truncation Watch Until", + "type": "integer" + }, + "truncation_sleep_between_seconds": { + "default": 30, + "description": "Sleep between truncation monitoring.", + "title": "Truncation Sleep Between Seconds", + "type": "integer" + }, + "dataprocess_cleanup": { + "$ref": "#/$defs/DataProcessCleanupConfig", + "description": "Configuration for data process cleanup" + }, + "soft_deleted_entities_cleanup": { + "$ref": "#/$defs/SoftDeletedEntitiesCleanupConfig", + "description": "Configuration for soft deleted entities cleanup" + }, + "execution_request_cleanup": { + "$ref": "#/$defs/DatahubExecutionRequestCleanupConfig", + "description": "Configuration for execution request cleanup" + } + }, + "title": "DataHubGcSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Index Cleanup + +Manages Elasticsearch indices in DataHub, particularly focusing on time-series data. + +##### Configuration + +```yaml +source: + type: datahub-gc + config: + truncate_indices: true + truncate_index_older_than_days: 30 + truncation_watch_until: 10000 + truncation_sleep_between_seconds: 30 +``` + +##### Features + +- Truncates old Elasticsearch indices for the following timeseries aspects: + - DatasetOperations + - DatasetUsageStatistics + - ChartUsageStatistics + - DashboardUsageStatistics + - QueryUsageStatistics + - Timeseries Aspects +- Monitors truncation progress +- Implements safe deletion with monitoring thresholds +- Supports gradual truncation with sleep intervals + +#### Expired Token Cleanup + +Manages access tokens in DataHub to maintain security and prevent token accumulation. + +##### Configuration + +```yaml +source: + type: datahub-gc + config: + cleanup_expired_tokens: true +``` + +##### Features + +- Automatically identifies and revokes expired access tokens +- Processes tokens in batches for efficiency +- Maintains system security by removing outdated credentials +- Reports number of tokens revoked +- Uses GraphQL API for token management + +#### Data Process Cleanup + +Manages the lifecycle of data processes, jobs, and their instances (DPIs) within DataHub. + +##### Features + +- Cleans up Data Process Instances (DPIs) based on age and count +- Can remove empty DataJobs and DataFlows +- Supports both soft and hard deletion +- Uses parallel processing for efficient cleanup +- Maintains configurable retention policies + +##### Configuration + +```yaml +source: + type: datahub-gc + config: + dataprocess_cleanup: + enabled: true + retention_days: 10 + keep_last_n: 5 + delete_empty_data_jobs: false + delete_empty_data_flows: false + hard_delete_entities: false + batch_size: 500 + max_workers: 10 + delay: 0.25 +``` + +##### Limitations + +- Maximum 9000 DPIs per job for performance + +#### Execution Request Cleanup + +Manages DataHub execution request records to prevent accumulation of historical execution data. + +##### Features + +- Maintains execution history per ingestion source +- Preserves minimum number of recent requests +- Removes old requests beyond retention period +- Special handling for running/pending requests +- Automatic cleanup of corrupted records + +##### Configuration + +```yaml +source: + type: datahub-gc + config: + execution_request_cleanup: + enabled: true + keep_history_min_count: 10 + keep_history_max_count: 1000 + keep_history_max_days: 30 + batch_read_size: 100 + runtime_limit_seconds: 3600 + max_read_errors: 10 +``` + +#### Soft-Deleted Entities Cleanup + +Manages the permanent removal of soft-deleted entities after a retention period. + +##### Features + +- Permanently removes soft-deleted entities after retention period +- Handles entity references cleanup +- Special handling for query entities +- Supports filtering by entity type, platform, or environment +- Concurrent processing with safety limits + +##### Configuration + +```yaml +source: + type: datahub-gc + config: + soft_deleted_entities_cleanup: + enabled: true + retention_days: 10 + batch_size: 500 + max_workers: 10 + delay: 0.25 + entity_types: null # Optional list of entity types to clean + platform: null # Optional platform filter + env: null # Optional environment filter + query: null # Optional custom query filter + limit_entities_delete: 25000 + futures_max_at_time: 1000 + runtime_limit_seconds: 7200 +``` + +##### Performance Considerations + +- Concurrent processing using thread pools +- Configurable batch sizes for optimal performance +- Rate limiting through configurable delays +- Maximum limits on concurrent operations + +#### Reporting + +Each cleanup task maintains detailed reports including: + +- Number of entities processed +- Number of entities removed +- Errors encountered +- Sample of affected entities +- Runtime statistics +- Task-specific metrics + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.gc.datahub_gc.DataHubGcSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for DataHubGc, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dataplex.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dataplex.md new file mode 100644 index 00000000..32444913 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dataplex.md @@ -0,0 +1,741 @@ +--- +sidebar_position: 19 +title: Dataplex +slug: /generated/ingestion/sources/dataplex +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Dataplex + +## Overview + +Dataplex is a DataHub utility or metadata-focused integration. Learn more in the [official Dataplex documentation](https://cloud.google.com/dataplex). + +The DataHub integration for Dataplex covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Dataplex Concept | DataHub Concept | Notes | +| :------------------------ | :---------------------------------------------------------------------------------- | :--------------------------------------------------------------------------- | +| Entry (Universal Catalog) | [`Dataset`](/docs/generated/metamodel/entities/dataset) | From Universal Catalog. Uses source platform URNs (e.g., `bigquery`, `gcs`). | +| BigQuery Project/Dataset | [`Container`](/docs/generated/metamodel/entities/container) | Created as containers to align with native BigQuery connector. | + + +## Module `dataplex` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Schema Metadata | ✅ | Enabled by default, can be disabled via configuration `include_schema`. | +| Table-Level Lineage | ✅ | Optionally enabled via configuration `include_lineage`. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `dataplex` module ingests metadata from Dataplex into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +The Dataplex connector extracts metadata from Google Dataplex using the **Universal Catalog Entries API**. This API extracts entries from system-managed entry groups for Google Cloud services and is the recommended approach for discovering resources across your GCP organization. + +#### Supported services + +- **BigQuery**: datasets, tables, models, routines, connections, and linked datasets +- **Cloud SQL**: instances +- **AlloyDB**: instances, databases, schemas, tables, and views +- **Spanner**: instances, databases, and tables +- **Pub/Sub**: topics and subscriptions +- **Cloud Storage**: buckets +- **Bigtable**: instances, clusters, and tables +- **Vertex AI**: models, datasets, and feature stores +- **Dataform**: repositories and workflows +- **Dataproc Metastore**: services and databases + +:::note +Only **BigQuery** and **Cloud Storage (GCS)** have been thoroughly tested with this connector. Other services may work but have not been validated. +::: + +### Prerequisites + +Refer to [Dataplex documentation](https://cloud.google.com/dataplex/docs) for Dataplex basics. + +#### Authentication + +Supports Application Default Credentials (ADC). See [GCP documentation](https://cloud.google.com/docs/authentication/provide-credentials-adc) for ADC setup. + +For service account authentication, follow these instructions: + +#### Create a service account and assign roles + +1. Create a service account following [GCP docs](https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-console) and assign the required roles + +2. Download the service account JSON keyfile + + Example credential file: + + ```json + { + "type": "service_account", + "project_id": "project-id-1234567", + "private_key_id": "d0121d0000882411234e11166c6aaa23ed5d74e0", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-----", + "client_email": "test@suppproject-id-1234567.iam.gserviceaccount.com", + "client_id": "113545814931671546333", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%suppproject-id-1234567.iam.gserviceaccount.com" + } + ``` + +3. To provide credentials to the source, you can either: + + Set an environment variable: + + ```sh + $ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/keyfile.json" + ``` + + _or_ + + Set credential config in your source based on the credential json file. For example: + + ```yml + credential: + project_id: "project-id-1234567" + private_key_id: "d0121d0000882411234e11166c6aaa23ed5d74e0" + private_key: "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-----\n" + client_email: "test@suppproject-id-1234567.iam.gserviceaccount.com" + client_id: "123456678890" + ``` + +#### Permissions + +Grant the following permissions to the service account on all target projects. + +**Universal Catalog Entries API:** + +**Default GCP Role:** [roles/dataplex.catalogViewer](https://cloud.google.com/dataplex/docs/iam-roles#dataplex.catalogViewer) + +| Permission | Description | +| --------------------------- | ------------------------------------- | +| `dataplex.entryGroups.get` | Retrieve specific entry group details | +| `dataplex.entryGroups.list` | View all entry groups in a location | +| `dataplex.entries.get` | Access entry metadata and details | +| `dataplex.entries.getData` | View data aspects within entries | +| `dataplex.entries.list` | Enumerate entries within groups | + +**Lineage extraction** (optional, `include_lineage: true`): + +**Default GCP Role:** [roles/datalineage.viewer](https://docs.cloud.google.com/iam/docs/roles-permissions/datalineage#datalineage.viewer) + +| Permission | Description | +| -------------------------- | ----------------------------------------- | +| `datalineage.links.get` | Allows a user to view lineage links | +| `datalineage.links.search` | Allows a user to search for lineage links | + + +### Install the Plugin +```shell +pip install 'acryl-datahub[dataplex]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: dataplex + config: + # Required: GCP project ID(s) where Dataplex resources are located + project_ids: + - "my-gcp-project" + + # Optional: GCP location for entries (Universal Catalog) + # Use multi-region locations (us, eu, asia) to access system entry groups like @bigquery + # Default: "us" + entries_location: "us" + + # Optional: Environment (default: PROD) + env: "PROD" + + # Optional: GCP credentials (if not using Application Default Credentials) + # credential: + # project_id: "my-gcp-project" + # private_key_id: "d0121d0000882411234e11166c6aaa23ed5d74e0" + # private_key: "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-----\n" + # client_email: "test@suppproject-id-1234567.iam.gserviceaccount.com" + # client_id: "123456678890" + + # Optional: Metadata extraction + # include_lineage: true # Extract lineage (default: true) + # include_schema: true # Extract schema metadata (default: true) + + # Optional: Lineage retry settings + # lineage_max_retries: 3 # Max retry attempts (range: 1-10, default: 3) + # lineage_retry_backoff_multiplier: 1.0 # Backoff delay multiplier (range: 0.1-10.0, default: 1.0) + + # Optional: Filtering patterns for entries + # filter_config: + # entries: + # dataset_pattern: + # allow: + # - "bq_.*" # Allow BigQuery entries + # - "pubsub_.*" # Allow Pub/Sub entries + # deny: + # - ".*_test" # Deny test entries + # - ".*_temp" # Deny temporary entries + + # Optional: Performance tuning + # batch_size: 1000 # Entries per batch for memory optimization (default: 1000) + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
batch_size
One of integer, null
| Batch size for metadata emission and lineage extraction. Entries are emitted in batches to prevent memory issues in large deployments. Lower values reduce memory usage but may increase processing time. Set to None to disable batching (process all entries at once). Recommended: 1000 for large deployments (>10k entries), None for small deployments (<1k entries). Default: 1000.
Default: 1000
| +|
dataplex_url
string
| Base URL for Dataplex console (for generating external links).
Default: https://console.cloud.google.com/dataplex
| +|
enable_stateful_lineage_ingestion
boolean
| Enable stateful lineage ingestion. This will store lineage window timestamps after successful lineage ingestion. and will not run lineage ingestion for same timestamps in subsequent run. NOTE: This only works with use_queries_v2=False (legacy extraction path). For queries v2, use enable_stateful_time_window instead.
Default: True
| +|
entries_location
string
| GCP location for Universal Catalog entries extraction. Must be a multi-region location (us, eu, asia) to access system-managed entry groups like @bigquery. Regional locations (us-central1, etc.) only contain placeholder entries and will miss BigQuery tables. Default: 'us' (recommended for most users).
Default: us
| +|
include_lineage
boolean
| Whether to extract lineage information using Dataplex Lineage API. Extracts table-level lineage relationships between entries. Lineage API calls automatically retry transient errors (timeouts, rate limits) with exponential backoff.
Default: True
| +|
include_schema
boolean
| Whether to extract and ingest schema metadata (columns, types, descriptions). Set to False to skip schema extraction for faster ingestion when only basic dataset metadata is needed. Disabling schema extraction can improve performance for large deployments. Default: True.
Default: True
| +|
lineage_max_retries
integer
| Maximum number of retry attempts for lineage API calls when encountering transient errors (timeouts, rate limits, service unavailable). Each attempt uses exponential backoff. Higher values increase resilience but may slow down ingestion. Default: 3.
Default: 3
| +|
lineage_retry_backoff_multiplier
number
| Multiplier for exponential backoff between lineage API retry attempts (in seconds). Wait time formula: multiplier * (2 ^ attempt_number), capped between 2-10 seconds. Higher values reduce API load but increase ingestion time. Default: 1.0.
Default: 1.0
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
credential
One of GCPCredential, null
| GCP credential information. If not specified, uses Application Default Credentials.
Default: None
| +|
credential.client_email ❓
string
| Client email | +|
credential.client_id ❓
string
| Client Id | +|
credential.private_key ❓
string(password)
| Private key in a form of '-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n' | +|
credential.private_key_id ❓
string
| Private key id | +|
credential.auth_provider_x509_cert_url
string
| Auth provider x509 certificate url
Default: https://www.googleapis.com/oauth2/v1/certs
| +|
credential.auth_uri
string
| Authentication uri
Default: https://accounts.google.com/o/oauth2/auth
| +|
credential.client_x509_cert_url
One of string, null
| If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email
Default: None
| +|
credential.project_id
One of string, null
| Project id to set the credentials
Default: None
| +|
credential.token_uri
string
| Token uri
Default: https://oauth2.googleapis.com/token
| +|
credential.type
string
| Authentication type
Default: service_account
| +|
filter_config
DataplexFilterConfig
| Filter configuration for Dataplex ingestion. | +|
filter_config.entries
EntriesFilterConfig
| Filter configuration specific to Dataplex Entries API (Universal Catalog). | +|
filter_config.entries.dataset_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
filter_config.entries.dataset_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
project_ids
array
| List of Google Cloud Project IDs to ingest Dataplex resources from. If not specified, uses project_id or attempts to detect from credentials. | +|
project_ids.string
string
| | +|
stateful_ingestion
One of StatefulIngestionConfig, null
| Stateful Ingestion Config
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "DataplexFilterConfig": { + "additionalProperties": false, + "description": "Filter configuration for Dataplex ingestion.", + "properties": { + "entries": { + "$ref": "#/$defs/EntriesFilterConfig", + "description": "Filters specific to Dataplex Entries API (Universal Catalog)." + } + }, + "title": "DataplexFilterConfig", + "type": "object" + }, + "EntriesFilterConfig": { + "additionalProperties": false, + "description": "Filter configuration specific to Dataplex Entries API (Universal Catalog).", + "properties": { + "dataset_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for entry IDs to filter in ingestion." + } + }, + "title": "EntriesFilterConfig", + "type": "object" + }, + "GCPCredential": { + "additionalProperties": false, + "properties": { + "project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Project id to set the credentials", + "title": "Project Id" + }, + "private_key_id": { + "description": "Private key id", + "title": "Private Key Id", + "type": "string" + }, + "private_key": { + "description": "Private key in a form of '-----BEGIN PRIVATE KEY-----\\nprivate-key\\n-----END PRIVATE KEY-----\\n'", + "format": "password", + "title": "Private Key", + "type": "string", + "writeOnly": true + }, + "client_email": { + "description": "Client email", + "title": "Client Email", + "type": "string" + }, + "client_id": { + "description": "Client Id", + "title": "Client Id", + "type": "string" + }, + "auth_uri": { + "default": "https://accounts.google.com/o/oauth2/auth", + "description": "Authentication uri", + "title": "Auth Uri", + "type": "string" + }, + "token_uri": { + "default": "https://oauth2.googleapis.com/token", + "description": "Token uri", + "title": "Token Uri", + "type": "string" + }, + "auth_provider_x509_cert_url": { + "default": "https://www.googleapis.com/oauth2/v1/certs", + "description": "Auth provider x509 certificate url", + "title": "Auth Provider X509 Cert Url", + "type": "string" + }, + "type": { + "default": "service_account", + "description": "Authentication type", + "title": "Type", + "type": "string" + }, + "client_x509_cert_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email", + "title": "Client X509 Cert Url" + } + }, + "required": [ + "private_key_id", + "private_key", + "client_email", + "client_id" + ], + "title": "GCPCredential", + "type": "object" + }, + "StatefulIngestionConfig": { + "additionalProperties": false, + "description": "Basic Stateful Ingestion Specific Configuration for any source.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + } + }, + "title": "StatefulIngestionConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Configuration for Google Dataplex source.", + "properties": { + "enable_stateful_lineage_ingestion": { + "default": true, + "description": "Enable stateful lineage ingestion. This will store lineage window timestamps after successful lineage ingestion. and will not run lineage ingestion for same timestamps in subsequent run. NOTE: This only works with use_queries_v2=False (legacy extraction path). For queries v2, use enable_stateful_time_window instead.", + "title": "Enable Stateful Lineage Ingestion", + "type": "boolean" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulIngestionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Stateful Ingestion Config" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "credential": { + "anyOf": [ + { + "$ref": "#/$defs/GCPCredential" + }, + { + "type": "null" + } + ], + "default": null, + "description": "GCP credential information. If not specified, uses Application Default Credentials." + }, + "project_ids": { + "description": "List of Google Cloud Project IDs to ingest Dataplex resources from. If not specified, uses project_id or attempts to detect from credentials.", + "items": { + "type": "string" + }, + "title": "Project Ids", + "type": "array" + }, + "entries_location": { + "default": "us", + "description": "GCP location for Universal Catalog entries extraction. Must be a multi-region location (us, eu, asia) to access system-managed entry groups like @bigquery. Regional locations (us-central1, etc.) only contain placeholder entries and will miss BigQuery tables. Default: 'us' (recommended for most users).", + "title": "Entries Location", + "type": "string" + }, + "filter_config": { + "$ref": "#/$defs/DataplexFilterConfig", + "description": "Filters to control which Dataplex resources are ingested." + }, + "include_schema": { + "default": true, + "description": "Whether to extract and ingest schema metadata (columns, types, descriptions). Set to False to skip schema extraction for faster ingestion when only basic dataset metadata is needed. Disabling schema extraction can improve performance for large deployments. Default: True.", + "title": "Include Schema", + "type": "boolean" + }, + "include_lineage": { + "default": true, + "description": "Whether to extract lineage information using Dataplex Lineage API. Extracts table-level lineage relationships between entries. Lineage API calls automatically retry transient errors (timeouts, rate limits) with exponential backoff.", + "title": "Include Lineage", + "type": "boolean" + }, + "lineage_max_retries": { + "default": 3, + "description": "Maximum number of retry attempts for lineage API calls when encountering transient errors (timeouts, rate limits, service unavailable). Each attempt uses exponential backoff. Higher values increase resilience but may slow down ingestion. Default: 3.", + "maximum": 10, + "minimum": 1, + "title": "Lineage Max Retries", + "type": "integer" + }, + "lineage_retry_backoff_multiplier": { + "default": 1.0, + "description": "Multiplier for exponential backoff between lineage API retry attempts (in seconds). Wait time formula: multiplier * (2 ^ attempt_number), capped between 2-10 seconds. Higher values reduce API load but increase ingestion time. Default: 1.0.", + "maximum": 10.0, + "minimum": 0.1, + "title": "Lineage Retry Backoff Multiplier", + "type": "number" + }, + "batch_size": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1000, + "description": "Batch size for metadata emission and lineage extraction. Entries are emitted in batches to prevent memory issues in large deployments. Lower values reduce memory usage but may increase processing time. Set to None to disable batching (process all entries at once). Recommended: 1000 for large deployments (>10k entries), None for small deployments (<1k entries). Default: 1000.", + "title": "Batch Size" + }, + "dataplex_url": { + "default": "https://console.cloud.google.com/dataplex", + "description": "Base URL for Dataplex console (for generating external links).", + "title": "Dataplex Url", + "type": "string" + } + }, + "title": "DataplexConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +:::caution +The Dataplex connector will overwrite metadata from other Google Cloud source connectors (BigQuery, GCS, etc.) if they extract the same entities. If you're running multiple Google Cloud connectors, be aware that the last connector to run will determine the final metadata state for overlapping entities. +::: + +#### Platform Alignment + +Datasets discovered by Dataplex use the same URNs as native connectors (e.g., `bigquery`, `gcs`). This means: + +- **No Duplication**: Dataplex and native BigQuery/GCS connectors can run together - entities discovered by both will merge +- **Native Containers**: BigQuery tables appear in their native dataset containers +- **Unified View**: Users see a single view of all datasets regardless of discovery method + +#### Custom Properties + +The connector adds the following custom properties to datasets: + +- `dataplex_entry_id`: The entry identifier in Dataplex +- `dataplex_entry_group`: The entry group containing this entry +- `dataplex_fully_qualified_name`: The fully qualified name of the entry +- `dataplex_ingested`: Marker indicating the dataset was ingested via Dataplex + +:::note +To access system-managed entry groups like `@bigquery`, use multi-region locations (`us`, `eu`, `asia`) via the `entries_location` config parameter. Regional locations (`us-central1`, etc.) only contain placeholder entries. +::: + +#### Filtering Configuration + +Filter which datasets to ingest using regex patterns with allow/deny lists: + +**Example:** + +```yaml +source: + type: dataplex + config: + project_ids: + - "my-gcp-project" + + filter_config: + entries: + dataset_pattern: + allow: + - "production_.*" # Only production datasets + deny: + - ".*_test" # Exclude test datasets + - ".*_temp" # Exclude temporary datasets +``` + +#### Lineage + +When `include_lineage` is enabled and proper permissions are granted, the connector extracts **table-level lineage** using the Dataplex Lineage API. Dataplex automatically tracks lineage from these Google Cloud systems: + +**Supported Systems:** + +- **BigQuery**: DDL (CREATE TABLE, CREATE TABLE AS SELECT, views, materialized views) and DML (SELECT, INSERT, MERGE, UPDATE, DELETE) operations +- **Cloud Data Fusion**: Pipeline executions +- **Cloud Composer**: Workflow orchestration +- **Dataflow**: Streaming and batch jobs +- **Dataproc**: Apache Spark and Apache Hive jobs (including Dataproc Serverless) +- **Vertex AI**: Models, datasets, feature store views, and feature groups + +:::note +Only **BigQuery** lineage has been thoroughly tested with this connector. Lineage from other systems may work but has not been validated. +::: + +**Not Supported:** + +- **Column-level lineage**: The connector extracts only table-level lineage (column-level lineage is available in Dataplex but not exposed through this connector) +- **Custom sources**: Only Google Cloud systems with automatic lineage tracking are supported +- **BigQuery Data Transfer Service**: Recurring loads are not automatically tracked + +**Lineage Limitations:** + +- Lineage data is retained for 30 days in Dataplex +- Lineage may take up to 24 hours to appear after job completion +- Cross-region lineage is not supported by Dataplex +- Lineage is only available for entries with active lineage tracking enabled + +For more details, see [Dataplex Lineage Documentation](https://docs.cloud.google.com/dataplex/docs/about-data-lineage). + +#### Configuration Options + +**Metadata Extraction:** + +- **`include_schema`** (default: `true`): Extract column metadata and types +- **`include_lineage`** (default: `true`): Extract table-level lineage (automatically retries transient errors) + +**Performance Tuning:** + +- **`batch_size`** (default: `1000`): Entries per batch for memory optimization. Set to `None` to disable batching (small deployments only) + +**Lineage Retry Settings** (optional): + +- **`lineage_max_retries`** (default: `3`, range: `1-10`): Retry attempts for transient errors +- **`lineage_retry_backoff_multiplier`** (default: `1.0`, range: `0.1-10.0`): Backoff delay multiplier + +**Example Configuration:** + +```yaml +source: + type: dataplex + config: + project_ids: + - "my-gcp-project" + + # Location for entries (Universal Catalog) - defaults to "us" + # Must be multi-region (us, eu, asia) for system entry groups like @bigquery + entries_location: "us" + + # Metadata extraction settings + include_schema: true # Enable schema metadata extraction (default: true) + include_lineage: true # Enable lineage extraction with automatic retries + + # Lineage retry settings (optional, defaults shown) + lineage_max_retries: 3 # Max retry attempts (range: 1-10) + lineage_retry_backoff_multiplier: 1.0 # Exponential backoff multiplier (range: 0.1-10.0) +``` + +**Configuration for Large Deployments:** + +For deployments with thousands of entries, memory optimization is important. The connector uses batched emission to keep memory bounded: + +```yaml +source: + type: dataplex + config: + project_ids: + - "my-gcp-project" + entries_location: "us" + + # Performance tuning + batch_size: 1000 # Process and emit 1000 entries at a time to optimize memory usage +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +#### Lineage Extraction Issues + +**Automatic Retry Behavior:** + +The connector automatically retries transient errors when extracting lineage: + +- **Retried errors** (with exponential backoff): Timeouts (DeadlineExceeded), rate limiting (HTTP 429), service issues (HTTP 503, 500) +- **Non-retried errors** (logs warning and continues): Permission denied (HTTP 403), not found (HTTP 404), invalid argument (HTTP 400) + +After exhausting retries, the connector logs a warning and continues processing other entries. You'll still get metadata even if lineage extraction fails for some entries. + +**Common Issues:** + +1. **Regional restrictions**: Lineage API requires multi-region location (`us`, `eu`, `asia`) rather than specific regions (`us-central1`). The connector automatically uses the `entries_location` config. +2. **Missing permissions**: Ensure service account has `roles/datalineage.viewer` role on all projects. +3. **No lineage data**: Some entries may not have lineage if they weren't created through supported systems (BigQuery DDL/DML, Cloud Data Fusion, etc.). +4. **Rate limiting**: If you encounter persistent rate limiting, increase `lineage_retry_backoff_multiplier` to add more delay between retries, or decrease `lineage_max_retries` if you prefer faster failure. + +#### Others + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.dataplex.dataplex.DataplexSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/dataplex/dataplex.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Dataplex, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/db2.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/db2.md new file mode 100644 index 00000000..1efe671b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/db2.md @@ -0,0 +1,1177 @@ +--- +sidebar_position: 38 +title: IBM Db2 +slug: /generated/ingestion/sources/db2 +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# IBM Db2 + +## Overview + +Db2 is a data platform used to store and query analytical or operational data. Learn more in the [official Db2 documentation](https://www.ibm.com/products/db2). + +The DataHub integration for Db2 covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `db2` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_column_lineage`. Supported for types - View. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_lineage`. Supported for types - View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `db2` module ingests metadata from Db2 into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +#### Permissions + +The user requires `SELECT` privileges on the system schema and tables: + +- Db2 for LUW: `SYSCAT.*` +- Db2 for z/OS: `SYSIBM.*` +- Db2 for IBM i (AS/400): `QSYS2.*` + +Additionally, when profiling is enabled, the user will need `SELECT` privileges on the tables to be profiled. + +#### Authentication and TLS/SSL + +Authentication is done by default via a Db2 username and password over a non-encrypted connection. + +Other authentication methods as well as TLS/SSL may be configured using the `uri_args` config option, e.g. + +```yaml +uri_args: + Security: SSL +``` + + +### Install the Plugin +```shell +pip install 'acryl-datahub[db2]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: db2 + config: + # Coordinates + host_port: localhost:50000 + database: dbname + + # Credentials + username: db2inst1 + password: password + + # Optional TLS + # uri_args: + # Security: SSL + + # Use the IBM i Access ODBC Driver to connect to Db2 for IBM i (AS/400) + # scheme: db2+pyodbc400 +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
database ✅
string
| The Db2 database to ingest from. | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
host_port
string
| Db2 host URL.
Default: localhost:50000
| +|
include_stored_procedures
boolean
| Ingest stored procedures.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
scheme
string
| ibm_db_sa scheme to use (db2, db2+pyodbc, db2+pyodbc400).
Default: db2
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
uri_args
map(str,string)
| | +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
procedure_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
procedure_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schemas to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "default": "localhost:50000", + "description": "Db2 host URL.", + "title": "Host Port", + "type": "string" + }, + "database": { + "description": "The Db2 database to ingest from.", + "title": "Database", + "type": "string" + }, + "scheme": { + "default": "db2", + "description": "ibm_db_sa scheme to use (db2, db2+pyodbc, db2+pyodbc400).", + "title": "Scheme", + "type": "string" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "include_stored_procedures": { + "default": true, + "description": "Ingest stored procedures.", + "title": "Include Stored Procedures", + "type": "boolean" + }, + "procedure_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for stored procedures to filter in ingestion.Specify regex to match the entire procedure name in schema.procedure_name format." + }, + "uri_args": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Arguments to add to the URL when connecting.", + "title": "Uri Args", + "type": "object" + } + }, + "required": [ + "database" + ], + "title": "Db2Config", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Using the IBM i Access ODBC Driver + +This source can connect to Db2 for IBM i (AS/400) directly using the IBM i Access ODBC Driver rather than using the default CLI driver that requires Db2 Connect. + +This requires you to [install the IBM i Access ODBC Driver](https://ibmi-oss-docs.readthedocs.io/en/latest/odbc/installation.html) along with an ODBC driver manager such as `unixodbc`. + +To use the IBM i Access ODBC Driver rather than the default driver, specify the `db2+pyodbc400` scheme in your ingestion configuration: + +```yaml +scheme: "db2+pyodbc400" +``` + +If you're using [DataHub Cloud's Remote Executor](/docs/managed-datahub/remote-executor/about), you can either build a custom image with the ODBC driver included, or sideload the driver. For instance, to sideload the driver and when using the Remote Executor Helm chart, you might use the following Helm values file: + +```yaml +extraInitContainers: + - name: init-db2-as400-driver + image: debian:stable-slim + command: + - bash + - -c + - | + set -euxo pipefail + apt-get update + apt-get install -y wget + wget -P /etc/apt/sources.list.d/ https://public.dhe.ibm.com/software/ibmi/products/odbc/debs/dists/1.1.0/ibmi-acs-1.1.0.list + apt-get update + apt-get install -y ibm-iaccess + cp /etc/odbcinst.ini /opt/ibm + volumeMounts: + - name: ibmdriver + mountPath: "/opt/ibm" + +extraVolumes: + - name: ibmdriver + emptyDir: {} + +extraVolumeMounts: + - name: ibmdriver + mountPath: "/opt/ibm" + readOnly: true + +nodeSelector: + kubernetes.io/arch: amd64 +``` + +Then configure your source to set the following environment variables: + +```bash +LD_LIBRARY_PATH="/opt/ibm/iaccess/lib64/" +ODBCSYSINI="/opt/ibm" +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +#### Architecture Support + +The Db2 source is available on Linux x86_64, macOS x86_64, and macOS ARM. It is not available on Linux ARM, and attempting to execute the source will result in an error. + +#### Db2 Platform Support + +This source has been tested against: + +- Db2 for LUW using the default CLI driver (already included when installing the `acryl-datahub[db2]` package) +- Db2 for IBM i (AS/400) using the IBM i Access ODBC Driver (not included and so must be installed separately by the end user; see [Using the IBM i Access ODBC Driver](#using-the-ibm-i-access-odbc-driver) below) + +This source also includes nominal support for: + +- Db2 for IBM i (AS/400) using the CLI driver ([requires Db2 Connect](https://www.ibm.com/support/pages/db2-odbc-cli-driver-download-and-installation-information)) +- Db2 for z/OS using the CLI driver ([requires Db2 Connect](https://www.ibm.com/support/pages/db2-odbc-cli-driver-download-and-installation-information)) + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.db2.Db2Source` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/db2.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for IBM Db2, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dbt.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dbt.md new file mode 100644 index 00000000..cf97ca3f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dbt.md @@ -0,0 +1,2574 @@ +--- +sidebar_position: 20 +title: dbt +slug: /generated/ingestion/sources/dbt +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# dbt +There are 2 sources that provide integration with dbt + + + + + + + + + + +
Source ModuleDocumentation
+ +`dbt` + + + + + [Read more...](#module-dbt) + + +
+ +`dbt-cloud` + + + + + [Read more...](#module-dbt-cloud) + + +
+ + +## Overview + +dbt is a data platform used to store and query analytical or operational data. Learn more in the [official dbt documentation](https://www.getdbt.com/). + +The DataHub integration for dbt covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +:::info Run both dbt and data warehouse ingestion for lineage + +1. You must **run ingestion for both dbt and your data warehouse** (target platform). They can be run in any order. +2. It generates column lineage between the `dbt` nodes (e.g. when a model/snapshot depends on a dbt source or ephemeral model) as well as lineage between the `dbt` nodes and the underlying target platform nodes (e.g. BigQuery Table -> dbt source, dbt model -> BigQuery table/view). +3. It automatically generates "sibling" relationships between the dbt nodes and the target / data warehouse nodes. These nodes will show up in the UI with both platform logos. +4. We also support automated actions (like add a tag, term or owner) based on properties defined in dbt meta. + ::: + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| -------------- | ---------------------------------------------------------------------- | ----------------------- | +| Source | [Dataset](../../metamodel/entities/dataset.md) | Subtype `Source` | +| Seed | [Dataset](../../metamodel/entities/dataset.md) | Subtype `Seed` | +| Model | [Dataset](../../metamodel/entities/dataset.md) | Subtype `Model` | +| Snapshot | [Dataset](../../metamodel/entities/dataset.md) | Subtype `Snapshot` | +| Semantic View | [Dataset](../../metamodel/entities/dataset.md) | Subtype `Semantic View` | +| Test | [Assertion](../../metamodel/entities/assertion.md) | | +| Test Result | [Assertion Run Result](../../metamodel/entities/assertion.md) | | +| Model Runs | [DataProcessInstance](../../metamodel/entities/dataProcessInstance.md) | | + + +## Module `dbt` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Column-level Lineage | ✅ | Enabled by default, configure using `include_column_lineage`. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Table-Level Lineage | ✅ | Enabled by default. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `dbt` module ingests metadata from Dbt into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +#### Query Entities from dbt Meta + +DataHub can ingest Query entities from the `meta.queries` field in your dbt models. This allows you to document "blessed" or commonly-used query patterns directly in dbt and surface them in DataHub's Queries tab for easy discovery and reuse by your team. + +**Config Options:** + +```yaml +source: + type: dbt + config: + manifest_path: target/manifest.json + # Control Query entity emission (default: YES) + entities_enabled: + queries: "NO" # or "YES" (default), "ONLY" + # Limit queries per model (default: 100, set 0 for unlimited) + max_queries_per_model: 100 +``` + +:::note Integration with Warehouse Query Ingestion + +If you're also using warehouse query ingestion (e.g., Snowflake usage, BigQuery audit logs), dbt-emitted queries will coexist with warehouse-discovered queries in the Queries tab. They're differentiated by source: dbt queries have `source: MANUAL` while warehouse queries typically have `source: SYSTEM`. + +::: + +##### How to Configure + +The `meta.queries` field is defined in your dbt model's properties file (e.g., `schema.yml`, `models.yml`, or any `.yml` file in your dbt project). When you run `dbt docs generate` or `dbt compile`, this metadata is included in the `manifest.json` file, which DataHub then ingests. + +**Add queries to your model's `meta` field in your dbt properties file:** + +```yaml +# models/schema.yml or models/customers.yml +version: 2 + +models: + - name: customers + description: "Customer dimension table" + meta: + queries: + - name: "Active customers (30d)" + description: "Customers active in the last 30 days" + sql: | + SELECT * + FROM {{ ref('customers') }} + WHERE active = true + AND last_seen > CURRENT_DATE - INTERVAL '30 days' + tags: ["production", "analytics"] + terms: ["CustomerData", "Engagement"] + + - name: "Revenue by customer" + description: "Total revenue aggregated by customer" + sql: | + SELECT + customer_id, + SUM(amount) as total_revenue + FROM {{ ref('customers') }} + GROUP BY customer_id + tags: ["finance", "reporting"] +``` + +**Then generate your dbt artifacts:** + +```sh +dbt docs generate +# This creates/updates target/manifest.json with the meta.queries data +``` + +**Finally, run DataHub ingestion:** + +```sh +datahub ingest -c your_dbt_recipe.yml +# DataHub reads manifest.json and creates Query entities +``` + +##### Field Reference + +Each query in the `queries` list supports the following fields: + +| Field | Required | Type | Description | +| ------------- | -------- | --------------- | -------------------------------------------------------------- | +| `name` | ✅ Yes | string | Unique name for the query | +| `sql` | ✅ Yes | string | SQL statement for the query | +| `description` | ❌ No | string | Human-readable description | +| `tags` | ❌ No | list of strings | Tags for categorization (stored in customProperties) | +| `terms` | ❌ No | list of strings | Glossary terms for classification (stored in customProperties) | + +##### How It Works + +1. **dbt Configuration**: You define `queries` in the `meta` field of your dbt model properties +2. **Manifest Generation**: When you run `dbt docs generate`, the `meta.queries` data is included in `manifest.json` +3. **DataHub Ingestion**: DataHub reads the manifest.json and extracts the `meta.queries` field +4. **Query Entity Creation**: Each query in `meta.queries` becomes a Query entity in DataHub +5. **URN Generation**: Query URN is generated as `urn:li:query:{dbt_unique_id}_{sanitized_query_name}` (e.g., `urn:li:query:model.my_project.customers_Active_customers_30d_`) +6. **Dataset Linking**: Queries are linked to the **target platform** dataset (e.g., Snowflake, Postgres) via QuerySubjects aspect, so they appear where analysts actually query +7. **UI Visibility**: Queries appear in the "Queries" tab of the target platform dataset in DataHub UI + +##### Technical Details + +- **Actor**: All queries are attributed to the `dbt_executor` actor +- **Timestamps**: Uses manifest `generated_at` for reproducibility; falls back to current time if unavailable +- **Custom Properties**: Tags/terms stored in `customProperties` (see [Known Limitations](#known-limitations) below) +- **SQL Truncation**: SQL exceeding 1MB is truncated with "..." suffix +- **URN Sanitization**: Consecutive special characters collapsed into single underscore (`[^a-zA-Z0-9_\-\.]+` → `_`) + +##### Error Handling + +| Scenario | Behavior | +| ---------------------------------- | ------------------------------------------------------------------------------------ | +| `meta.queries` not a list | Skipped with WARNING log | +| Query missing `name` or `sql` | Skipped, all validation errors shown in log and `queries_failed_list` | +| Duplicate query names | Duplicate skipped, first definition wins (WARNING) | +| Invalid `description` (not string) | Field ignored with WARNING log | +| Invalid `tags`/`terms` (not list) | Field ignored with WARNING log | +| Empty values in tags/terms list | Filtered out automatically | +| Manifest timestamp unparseable | Falls back to current time with WARNING; tracked in `query_timestamps_fallback_used` | +| Queries on ephemeral model | Skipped with WARNING (ephemeral models don't exist in target platform) | +| Exceeds `max_queries_per_model` | Only first N processed (configurable, default 100), WARNING logged | + +All validation errors are logged at WARNING level and tracked in the ingestion report. + +##### Example Output in DataHub + +After ingestion, you'll see: + +- Query entities in DataHub with name, description, and SQL statement +- Queries linked to the target platform dataset (visible in the dataset's "Queries" tab) +- Tags and terms visible in custom properties +- Creation/modification timestamps from dbt manifest +- Queries attributed to `dbt_executor` actor + +##### Use Cases + +- **Blessed Query Patterns**: Document approved query patterns for common analytics use cases +- **Query Templates**: Provide reusable query templates for team members +- **Best Practices**: Share optimized queries that follow your organization's standards +- **Self-Service Analytics**: Enable analysts to discover and reuse proven queries + +##### Integration with Other dbt Features + +The `meta.queries` feature works alongside other dbt metadata capabilities in DataHub: + +| Feature | Purpose | Config Key | +| ---------------------------- | ------------------------------------------------ | -------------------------- | +| **meta.queries** | Define Query entities for discovery | `entities_enabled.queries` | +| **meta_mapping** | Map dbt meta fields to DataHub tags/terms/owners | `meta_mapping` | +| **column_meta_mapping** | Map column-level meta to DataHub aspects | `column_meta_mapping` | +| **owner_extraction_pattern** | Extract owners from meta fields | `owner_extraction_pattern` | +| **tag_prefix** | Prefix for auto-generated tags | `tag_prefix` | + +**Example combining features:** + +```yaml +source: + type: dbt + config: + manifest_path: target/manifest.json + catalog_path: target/catalog.json + target_platform: snowflake + + # Enable query entities from meta.queries + entities_enabled: + queries: "YES" + max_queries_per_model: 100 + + # Map other meta fields to DataHub aspects + meta_mapping: + business_owner: + match: ".*" + operation: "add_owner" + config: + owner_type: user + data_tier: + match: ".*" + operation: "add_tag" + + # Extract owners from specific meta patterns + enable_meta_mapping: true +``` + +##### Known Limitations + +1. **Tags/Terms in Custom Properties**: Query entities don't currently support native `GlobalTags` or `GlossaryTerms` aspects. Tags and terms are stored as comma-separated strings in `customProperties`. This means: + + - Cannot filter queries by tags in the DataHub UI search + - Cannot apply tag-based governance policies to queries + - Tags/terms appear as plain text in the Properties tab, not as clickable links + +2. **No SQL Validation Against Model**: The `sql` field in `meta.queries` is not validated against the model it's defined on. You could define `sql: "SELECT * FROM products"` under the `customers` model. DataHub trusts that users define meaningful queries. Consider documenting your team's conventions for query definitions. + +3. **URN Collision on Similar Names**: Query names are sanitized for URN generation. Names like `"Revenue (USD)"` and `"Revenue [USD]"` both become `Revenue_USD_`, causing a collision (second one is skipped with a warning). Use distinct, alphanumeric query names to avoid this. + +4. **Ephemeral Models Not Supported**: Queries defined on ephemeral models (`materialized: ephemeral`) are skipped because ephemeral models don't exist as physical tables in the target platform. Queries are linked to target platform datasets, so there's no dataset to link to. + +:::tip Choosing Between meta.queries and meta_mapping + +- Use **meta.queries** for defining reusable SQL query patterns that should appear in the Queries tab +- Use **meta_mapping** for mapping arbitrary meta fields to DataHub tags, terms, and owners +- Both features can be used together - they operate on different parts of the meta object + +::: + +### Prerequisites + +The artifacts used by this source are: + +- [dbt manifest file](https://docs.getdbt.com/reference/artifacts/manifest-json) + - This file contains model, source, tests and lineage data. +- [dbt catalog file](https://docs.getdbt.com/reference/artifacts/catalog-json) + - This file contains schema data. + - dbt does not record schema data for Ephemeral models, as such datahub will show Ephemeral models in the lineage, however there will be no associated schema for Ephemeral models +- [dbt sources file](https://docs.getdbt.com/reference/artifacts/sources-json) + - This file contains metadata for sources with freshness checks. + - We transfer dbt's freshness checks to DataHub's last-modified fields. + - Note that this file is optional – if not specified, we'll use time of ingestion instead as a proxy for time last-modified. +- [dbt run_results file](https://docs.getdbt.com/reference/artifacts/run-results-json) + - This file contains metadata from the result of a dbt run, e.g. dbt test + - When provided, we transfer dbt test run results into assertion run events to see a timeline of test runs on the dataset + +**Recommended workflow for dbt build and DataHub ingestion:** + +```sh +dbt source snapshot-freshness +dbt build +cp target/run_results.json target/run_results_backup.json +dbt docs generate +cp target/run_results_backup.json target/run_results.json + +# Run datahub ingestion, pointing at the files in the target/ directory +``` + +The necessary artifact files will then appear in the `target/` directory of your dbt project. + +We also have guides on handling more complex dbt orchestration techniques and multi-project setups below. + +:::note Entity is in manifest but missing from catalog + +This warning usually appears when the catalog.json file was not generated by a `dbt docs generate` command. +Most other dbt commands generate a partial catalog file, which may impact the completeness of the metadata in ingested into DataHub. + +Following the above workflow should ensure that the catalog file is generated correctly. + +::: + + +### Install the Plugin +```shell +pip install 'acryl-datahub[dbt]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "dbt" + config: + # Coordinates + # To use this as-is, set the environment variable DBT_PROJECT_ROOT to the root folder of your dbt project + manifest_path: "${DBT_PROJECT_ROOT}/target/manifest_file.json" + catalog_path: "${DBT_PROJECT_ROOT}/target/catalog_file.json" + sources_path: "${DBT_PROJECT_ROOT}/target/sources_file.json" # optional for freshness + run_results_paths: + - "${DBT_PROJECT_ROOT}/target/run_results.json" # optional for recording dbt test results after running dbt test + + # Options + target_platform: "my_target_platform_id" # e.g. bigquery/postgres/etc. + # convert_urns_to_lowercase: true # optional: lowercase dbt URNs to avoid duplicates from casing differences (default: true for Snowflake) + +# sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
manifest_path ✅
string
| Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json. This can be a local file or a URI. | +|
target_platform ✅
string
| The platform that dbt is loading onto. (e.g. bigquery / redshift / postgres etc.) | +|
catalog_path
One of string, null
| Path to dbt catalog JSON. See https://docs.getdbt.com/reference/artifacts/catalog-json. This file is optional, but highly recommended. Without it, some metadata like column info will be incomplete or missing. This can be a local file or a URI.
Default: None
| +|
column_meta_mapping
object
| mapping rules that will be executed against dbt column meta properties. Refer to the section below on dbt meta automated mappings.
Default: {}
| +|
convert_column_urns_to_lowercase
boolean
| When enabled, converts column URNs to lowercase to ensure cross-platform compatibility. If `target_platform` is Snowflake, the default is True.
Default: False
| +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
dbt_is_primary_sibling
boolean
| Experimental: Controls sibling relationship primary designation between dbt entities and target platform entities. When True (default), dbt entities are primary and target platform entities are secondary. When False, target platform entities are primary and dbt entities are secondary. Uses aspect patches for precise control. Requires DataHub server 1.3.0+.
Default: True
| +|
drop_duplicate_sources
boolean
| When enabled, drops sources that have the same name in the target platform as a model. This ensures that lineage is generated reliably, but will lose any documentation associated only with the source.
Default: True
| +|
enable_meta_mapping
boolean
| When enabled, applies the mappings that are defined through the meta_mapping directives.
Default: True
| +|
enable_owner_extraction
boolean
| When enabled, ownership info will be extracted from the dbt meta
Default: True
| +|
enable_query_tag_mapping
boolean
| When enabled, applies the mappings that are defined through the `query_tag_mapping` directives.
Default: True
| +|
include_column_lineage
boolean
| When enabled, column-level lineage will be extracted from the dbt node definition. Requires `infer_dbt_schemas` to be enabled. If you run into issues where the column name casing does not match up with properly, providing a datahub_api or using the rest sink will improve accuracy.
Default: True
| +|
include_compiled_code
boolean
| When enabled, includes the compiled code in the emitted metadata.
Default: True
| +|
include_database_name
boolean
| Whether to add database name to the table urn. Set to False to skip it for engines like AWS Athena where it's not required.
Default: True
| +|
include_env_in_assertion_guid
boolean
| Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion that are only distinguished by env, then you should set this flag to True.
Default: False
| +|
incremental_lineage
boolean
| When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run. This would also require enabling 'incremental_lineage' in the counterpart warehouse ingestion (_e.g._ BigQuery, Redshift, etc).
Default: True
| +|
infer_dbt_schemas
boolean
| When enabled, schemas will be inferred from the dbt node definition.
Default: True
| +|
max_queries_per_model
integer
| Maximum number of Query entities to emit per dbt model. Prevents metadata explosion from malformed manifests. Set to 0 for unlimited.
Default: 100
| +|
meta_mapping
object
| mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.
Default: {}
| +|
only_include_if_in_catalog
boolean
| [experimental] If true, only include nodes that are also present in the catalog file. This is useful if you only want to include models that have been built by the associated run.
Default: False
| +|
owner_extraction_pattern
One of string, null
| Regex string to extract owner from the dbt node using the `(?P...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r"(?P(.*)): (\w+) (\w+)"` will extract `jdoe` as the owner from `"jdoe: John Doe"` (2) `r"@(?P(.*))"` will extract `alice` as the owner from `"@alice"`.
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
prefer_sql_parser_lineage
boolean
| Normally we use dbt's metadata to generate table lineage. When enabled, we prefer results from the SQL parser when generating lineage instead. This can be useful when dbt models reference tables directly, instead of using the ref() macro. This requires that `skip_sources_in_lineage` is enabled.
Default: False
| +|
query_tag_mapping
object
| mapping rules that will be executed against dbt query_tag meta properties. Refer to the section below on dbt meta automated mappings.
Default: {}
| +|
skip_sources_in_lineage
boolean
| [Experimental] When enabled, dbt sources will not be included in the lineage graph. Requires that `entities_enabled.sources` is set to `NO`. This is mainly useful when you have multiple, interdependent dbt projects.
Default: False
| +|
sources_path
One of string, null
| Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. If not specified, last-modified fields will not be populated. This can be a local file or a URI.
Default: None
| +|
strip_user_ids_from_email
boolean
| Whether or not to strip email id while adding owners using dbt meta actions.
Default: False
| +|
tag_prefix
string
| Prefix added to tags during ingestion.
Default: dbt:
| +|
target_platform_instance
One of string, null
| The platform instance for the platform that dbt is operating on. Use this if you have multiple instances of the same platform (e.g. redshift) and need to distinguish between them.
Default: None
| +|
test_warnings_are_errors
boolean
| When enabled, dbt test warnings will be treated as failures.
Default: False
| +|
use_identifiers
boolean
| Use model identifier instead of model name if defined (if not, default to model name).
Default: False
| +|
write_semantics
string
| Whether the new tags, terms and owners to be added will override the existing ones added only by this source or not. Value for this config can be "PATCH" or "OVERRIDE"
Default: PATCH
| +|
env
string
| Environment to use in namespace when constructing URNs.
Default: PROD
| +|
aws_connection
One of AwsConnectionConfig, null
| When fetching manifest files from s3, configuration for aws connection details
Default: None
| +|
aws_connection.aws_access_key_id
One of string, null
| AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_connection.aws_advanced_config
object
| Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html). | +|
aws_connection.aws_endpoint_url
One of string, null
| The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.
Default: None
| +|
aws_connection.aws_profile
One of string, null
| The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.
Default: None
| +|
aws_connection.aws_proxy
One of string, null
| A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.
Default: None
| +|
aws_connection.aws_region
One of string, null
| AWS region code.
Default: None
| +|
aws_connection.aws_retry_mode
Enum
| One of: "legacy", "standard", "adaptive"
Default: standard
| +|
aws_connection.aws_retry_num
integer
| Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.
Default: 5
| +|
aws_connection.aws_secret_access_key
One of string(password), null
| AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_connection.aws_session_token
One of string(password), null
| AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_connection.read_timeout
number
| The timeout for reading from the connection (in seconds).
Default: 60
| +|
aws_connection.aws_role
One of string, array, null
| AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).
Default: None
| +|
aws_connection.aws_role.union
One of string, AwsAssumeRoleConfig
| | +|
aws_connection.aws_role.union.RoleArn ❓
string
| ARN of the role to assume. | +|
aws_connection.aws_role.union.ExternalId
One of string, null
| External ID to use when assuming the role.
Default: None
| +|
entities_enabled
DBTEntitiesEnabled
| Controls which dbt entities are going to be emitted by this source | +|
entities_enabled.exposures
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.model_performance
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.models
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.queries
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.seeds
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.semantic_models
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.snapshots
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.sources
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.test_definitions
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.test_results
Enum
| One of: "YES", "NO", "ONLY" | +|
git_info
One of GitReference, null
| Reference to your git location to enable easy navigation from DataHub to your dbt files.
Default: None
| +|
git_info.repo ❓
string
| Name of your Git repo e.g. https://github.com/datahub-project/datahub or https://gitlab.com/gitlab-org/gitlab. If organization/repo is provided, we assume it is a GitHub repo. | +|
git_info.branch
string
| Branch on which your files live by default. Typically main or master. This can also be a commit hash.
Default: main
| +|
git_info.url_subdir
One of string, null
| Prefix to prepend when generating URLs for files - useful when files are in a subdirectory. Only affects URL generation, not git operations.
Default: None
| +|
git_info.url_template
One of string, null
| Template for generating a URL to a file in the repo e.g. '{repo_url}/blob/{branch}/{file_path}'. We can infer this for GitHub and GitLab repos, and it is otherwise required.It supports the following variables: {repo_url}, {branch}, {file_path}
Default: None
| +|
materialized_node_pattern
MaterializedNodePatternConfig
| Configuration for filtering materialized nodes based on their physical location | +|
materialized_node_pattern.database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
materialized_node_pattern.database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
materialized_node_pattern.schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
materialized_node_pattern.schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
materialized_node_pattern.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
materialized_node_pattern.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
node_name_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
node_name_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
run_results_paths
array
| Path to output of dbt test run as run_results files in JSON format. If not specified, test execution results and model performance metadata will not be populated in DataHub.If invoking dbt multiple times, you can provide paths to multiple run result files. See https://docs.getdbt.com/reference/artifacts/run-results-json.
Default: []
| +|
run_results_paths.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| DBT Stateful Ingestion Config.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AwsAssumeRoleConfig": { + "additionalProperties": true, + "properties": { + "RoleArn": { + "description": "ARN of the role to assume.", + "title": "Rolearn", + "type": "string" + }, + "ExternalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "External ID to use when assuming the role.", + "title": "Externalid" + } + }, + "required": [ + "RoleArn" + ], + "title": "AwsAssumeRoleConfig", + "type": "object" + }, + "AwsConnectionConfig": { + "additionalProperties": false, + "description": "Common AWS credentials config.\n\nCurrently used by:\n - Glue source\n - SageMaker source\n - dbt source", + "properties": { + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Session Token" + }, + "aws_role": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/AwsAssumeRoleConfig" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).", + "title": "Aws Role" + }, + "aws_profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.", + "title": "Aws Profile" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region code.", + "title": "Aws Region" + }, + "aws_endpoint_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.", + "title": "Aws Endpoint Url" + }, + "aws_proxy": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", + "title": "Aws Proxy" + }, + "aws_retry_num": { + "default": 5, + "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "title": "Aws Retry Num", + "type": "integer" + }, + "aws_retry_mode": { + "default": "standard", + "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "enum": [ + "legacy", + "standard", + "adaptive" + ], + "title": "Aws Retry Mode", + "type": "string" + }, + "read_timeout": { + "default": 60, + "description": "The timeout for reading from the connection (in seconds).", + "title": "Read Timeout", + "type": "number" + }, + "aws_advanced_config": { + "additionalProperties": true, + "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).", + "title": "Aws Advanced Config", + "type": "object" + } + }, + "title": "AwsConnectionConfig", + "type": "object" + }, + "DBTEntitiesEnabled": { + "additionalProperties": false, + "description": "Controls which dbt entities are going to be emitted by this source", + "properties": { + "models": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt models when set to Yes or Only" + }, + "sources": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt sources when set to Yes or Only" + }, + "seeds": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt seeds when set to Yes or Only" + }, + "snapshots": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt snapshots when set to Yes or Only" + }, + "test_definitions": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for test definitions when enabled when set to Yes or Only" + }, + "test_results": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for test results when set to Yes or Only" + }, + "model_performance": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit model performance metadata when set to Yes or Only. Only supported with dbt core." + }, + "exposures": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt exposures when set to Yes or Only. Exposures represent downstream consumers like dashboards, notebooks, or applications." + }, + "semantic_models": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt semantic models when set to Yes or Only. Semantic models define entities, dimensions, and measures for the dbt semantic layer (dbt 1.6+)." + }, + "queries": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit Query entities from meta.queries field when set to Yes or Only." + } + }, + "title": "DBTEntitiesEnabled", + "type": "object" + }, + "EmitDirective": { + "description": "A holder for directives for emission for specific types of entities", + "enum": [ + "YES", + "NO", + "ONLY" + ], + "title": "EmitDirective", + "type": "string" + }, + "GitReference": { + "additionalProperties": false, + "description": "Reference to a hosted Git repository. Used to generate \"view source\" links.", + "properties": { + "repo": { + "description": "Name of your Git repo e.g. https://github.com/datahub-project/datahub or https://gitlab.com/gitlab-org/gitlab. If organization/repo is provided, we assume it is a GitHub repo.", + "title": "Repo", + "type": "string" + }, + "branch": { + "default": "main", + "description": "Branch on which your files live by default. Typically main or master. This can also be a commit hash.", + "title": "Branch", + "type": "string" + }, + "url_subdir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Prefix to prepend when generating URLs for files - useful when files are in a subdirectory. Only affects URL generation, not git operations.", + "title": "Url Subdir" + }, + "url_template": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Template for generating a URL to a file in the repo e.g. '{repo_url}/blob/{branch}/{file_path}'. We can infer this for GitHub and GitLab repos, and it is otherwise required.It supports the following variables: {repo_url}, {branch}, {file_path}", + "title": "Url Template" + } + }, + "required": [ + "repo" + ], + "title": "GitReference", + "type": "object" + }, + "MaterializedNodePatternConfig": { + "additionalProperties": false, + "description": "Configuration for filtering materialized nodes based on their physical location", + "properties": { + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for database names to filter materialized nodes." + }, + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schema names in format '{database}.{schema}' to filter materialized nodes." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for table/view names in format '{database}.{schema}.{table}' to filter materialized nodes." + } + }, + "title": "MaterializedNodePatternConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "incremental_lineage": { + "default": true, + "description": "When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run. This would also require enabling 'incremental_lineage' in the counterpart warehouse ingestion (_e.g._ BigQuery, Redshift, etc).", + "title": "Incremental Lineage", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "Environment to use in namespace when constructing URNs.", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "DBT Stateful Ingestion Config." + }, + "target_platform": { + "description": "The platform that dbt is loading onto. (e.g. bigquery / redshift / postgres etc.)", + "title": "Target Platform", + "type": "string" + }, + "target_platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The platform instance for the platform that dbt is operating on. Use this if you have multiple instances of the same platform (e.g. redshift) and need to distinguish between them.", + "title": "Target Platform Instance" + }, + "use_identifiers": { + "default": false, + "description": "Use model identifier instead of model name if defined (if not, default to model name).", + "title": "Use Identifiers", + "type": "boolean" + }, + "entities_enabled": { + "$ref": "#/$defs/DBTEntitiesEnabled", + "default": { + "models": "YES", + "sources": "YES", + "seeds": "YES", + "snapshots": "YES", + "test_definitions": "YES", + "test_results": "YES", + "model_performance": "YES", + "exposures": "YES", + "semantic_models": "YES", + "queries": "YES" + }, + "description": "Controls for enabling / disabling metadata emission for different dbt entities (models, test definitions, test results, etc.)" + }, + "prefer_sql_parser_lineage": { + "default": false, + "description": "Normally we use dbt's metadata to generate table lineage. When enabled, we prefer results from the SQL parser when generating lineage instead. This can be useful when dbt models reference tables directly, instead of using the ref() macro. This requires that `skip_sources_in_lineage` is enabled.", + "title": "Prefer Sql Parser Lineage", + "type": "boolean" + }, + "skip_sources_in_lineage": { + "default": false, + "description": "[Experimental] When enabled, dbt sources will not be included in the lineage graph. Requires that `entities_enabled.sources` is set to `NO`. This is mainly useful when you have multiple, interdependent dbt projects. ", + "title": "Skip Sources In Lineage", + "type": "boolean" + }, + "tag_prefix": { + "default": "dbt:", + "description": "Prefix added to tags during ingestion.", + "title": "Tag Prefix", + "type": "string" + }, + "node_name_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for dbt model names to filter in ingestion." + }, + "materialized_node_pattern": { + "$ref": "#/$defs/MaterializedNodePatternConfig", + "default": { + "database_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "schema_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + } + }, + "description": "Advanced filtering for materialized nodes based on their physical database location. Provides fine-grained control over database.schema.table patterns for catalog consistency." + }, + "meta_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.", + "title": "Meta Mapping", + "type": "object" + }, + "column_meta_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against dbt column meta properties. Refer to the section below on dbt meta automated mappings.", + "title": "Column Meta Mapping", + "type": "object" + }, + "enable_meta_mapping": { + "default": true, + "description": "When enabled, applies the mappings that are defined through the meta_mapping directives.", + "title": "Enable Meta Mapping", + "type": "boolean" + }, + "query_tag_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against dbt query_tag meta properties. Refer to the section below on dbt meta automated mappings.", + "title": "Query Tag Mapping", + "type": "object" + }, + "enable_query_tag_mapping": { + "default": true, + "description": "When enabled, applies the mappings that are defined through the `query_tag_mapping` directives.", + "title": "Enable Query Tag Mapping", + "type": "boolean" + }, + "write_semantics": { + "default": "PATCH", + "description": "Whether the new tags, terms and owners to be added will override the existing ones added only by this source or not. Value for this config can be \"PATCH\" or \"OVERRIDE\"", + "title": "Write Semantics", + "type": "string" + }, + "strip_user_ids_from_email": { + "default": false, + "description": "Whether or not to strip email id while adding owners using dbt meta actions.", + "title": "Strip User Ids From Email", + "type": "boolean" + }, + "enable_owner_extraction": { + "default": true, + "description": "When enabled, ownership info will be extracted from the dbt meta", + "title": "Enable Owner Extraction", + "type": "boolean" + }, + "max_queries_per_model": { + "default": 100, + "description": "Maximum number of Query entities to emit per dbt model. Prevents metadata explosion from malformed manifests. Set to 0 for unlimited.", + "minimum": 0, + "title": "Max Queries Per Model", + "type": "integer" + }, + "owner_extraction_pattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Regex string to extract owner from the dbt node using the `(?P...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r\"(?P(.*)): (\\w+) (\\w+)\"` will extract `jdoe` as the owner from `\"jdoe: John Doe\"` (2) `r\"@(?P(.*))\"` will extract `alice` as the owner from `\"@alice\"`.", + "title": "Owner Extraction Pattern" + }, + "include_env_in_assertion_guid": { + "default": false, + "description": "Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion that are only distinguished by env, then you should set this flag to True.", + "title": "Include Env In Assertion Guid", + "type": "boolean" + }, + "convert_column_urns_to_lowercase": { + "default": false, + "description": "When enabled, converts column URNs to lowercase to ensure cross-platform compatibility. If `target_platform` is Snowflake, the default is True.", + "title": "Convert Column Urns To Lowercase", + "type": "boolean" + }, + "test_warnings_are_errors": { + "default": false, + "description": "When enabled, dbt test warnings will be treated as failures.", + "title": "Test Warnings Are Errors", + "type": "boolean" + }, + "infer_dbt_schemas": { + "default": true, + "description": "When enabled, schemas will be inferred from the dbt node definition.", + "title": "Infer Dbt Schemas", + "type": "boolean" + }, + "include_column_lineage": { + "default": true, + "description": "When enabled, column-level lineage will be extracted from the dbt node definition. Requires `infer_dbt_schemas` to be enabled. If you run into issues where the column name casing does not match up with properly, providing a datahub_api or using the rest sink will improve accuracy.", + "title": "Include Column Lineage", + "type": "boolean" + }, + "include_compiled_code": { + "default": true, + "description": "When enabled, includes the compiled code in the emitted metadata.", + "title": "Include Compiled Code", + "type": "boolean" + }, + "include_database_name": { + "default": true, + "description": "Whether to add database name to the table urn. Set to False to skip it for engines like AWS Athena where it's not required.", + "title": "Include Database Name", + "type": "boolean" + }, + "dbt_is_primary_sibling": { + "default": true, + "description": "Experimental: Controls sibling relationship primary designation between dbt entities and target platform entities. When True (default), dbt entities are primary and target platform entities are secondary. When False, target platform entities are primary and dbt entities are secondary. Uses aspect patches for precise control. Requires DataHub server 1.3.0+.", + "title": "Dbt Is Primary Sibling", + "type": "boolean" + }, + "drop_duplicate_sources": { + "default": true, + "description": "When enabled, drops sources that have the same name in the target platform as a model. This ensures that lineage is generated reliably, but will lose any documentation associated only with the source.", + "title": "Drop Duplicate Sources", + "type": "boolean" + }, + "manifest_path": { + "description": "Path to dbt manifest JSON. See https://docs.getdbt.com/reference/artifacts/manifest-json. This can be a local file or a URI.", + "title": "Manifest Path", + "type": "string" + }, + "catalog_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to dbt catalog JSON. See https://docs.getdbt.com/reference/artifacts/catalog-json. This file is optional, but highly recommended. Without it, some metadata like column info will be incomplete or missing. This can be a local file or a URI.", + "title": "Catalog Path" + }, + "sources_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. If not specified, last-modified fields will not be populated. This can be a local file or a URI.", + "title": "Sources Path" + }, + "run_results_paths": { + "default": [], + "description": "Path to output of dbt test run as run_results files in JSON format. If not specified, test execution results and model performance metadata will not be populated in DataHub.If invoking dbt multiple times, you can provide paths to multiple run result files. See https://docs.getdbt.com/reference/artifacts/run-results-json.", + "items": { + "type": "string" + }, + "title": "Run Results Paths", + "type": "array" + }, + "only_include_if_in_catalog": { + "default": false, + "description": "[experimental] If true, only include nodes that are also present in the catalog file. This is useful if you only want to include models that have been built by the associated run.", + "title": "Only Include If In Catalog", + "type": "boolean" + }, + "aws_connection": { + "anyOf": [ + { + "$ref": "#/$defs/AwsConnectionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "When fetching manifest files from s3, configuration for aws connection details" + }, + "git_info": { + "anyOf": [ + { + "$ref": "#/$defs/GitReference" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Reference to your git location to enable easy navigation from DataHub to your dbt files." + } + }, + "required": [ + "target_platform", + "manifest_path" + ], + "title": "DBTCoreConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### dbt meta automated mappings + +dbt allows authors to define meta properties for datasets. Checkout this link to know more - [dbt meta](https://docs.getdbt.com/reference/resource-configs/meta). Our dbt source allows users to define +actions such as add a tag, term or owner. For example if a dbt model has a meta config `"has_pii": True`, we can define an action +that evaluates if the property is set to true and add, lets say, a `pii` tag. +To leverage this feature we require users to define mappings as part of the recipe. The following section describes how you can build these mappings. Listed below is a `meta_mapping` and `column_meta_mapping` section that among other things, looks for keys like `business_owner` and adds owners that are listed there. + +```yaml +meta_mapping: + business_owner: + match: ".*" + operation: "add_owner" + config: + owner_type: user + owner_category: BUSINESS_OWNER + has_pii: + match: True + operation: "add_tag" + config: + tag: "has_pii_test" + int_property: + match: 1 + operation: "add_tag" + config: + tag: "int_meta_property" + double_property: + match: 2.5 + operation: "add_term" + config: + term: "double_meta_property" + data_governance.team_owner: + match: "Finance" + operation: "add_term" + config: + term: "Finance_test" + terms_list: + match: ".*" + operation: "add_terms" + config: + separator: "," + documentation_link: + match: "(?:https?)?\:\/\/\w*[^#]*" + operation: "add_doc_link" + config: + link: {{ $match }} + description: "Documentation Link" +column_meta_mapping: + terms_list: + match: ".*" + operation: "add_terms" + config: + separator: "," + is_sensitive: + match: True + operation: "add_tag" + config: + tag: "sensitive" + gdpr.pii: + match: true + operation: "add_tag" + config: + tag: "pii" +``` + +We support the following operations: + +1. add_tag - Requires `tag` property in config. +2. add_term - Requires `term` property in config. +3. add_terms - Accepts an optional `separator` property in config. +4. add_owner - Requires `owner_type` property in config which can be either `user` or `group`. Optionally accepts the `owner_category` config property which can be set to either a [custom ownership type](../../../../docs/ownership/ownership-types.md) urn like `urn:li:ownershipType:architect` or one of `['TECHNICAL_OWNER', 'BUSINESS_OWNER', 'DATA_STEWARD', 'DATAOWNER'` (defaults to `DATAOWNER`). + + - The `owner_type` property will be ignored if the owner is a fully qualified urn. + - You can use commas to specify multiple owners - e.g. `business_owner: "jane,john,urn:li:corpGroup:data-team"`. + +5. add_doc_link - Requires `link` and `description` properties in config. Upon ingestion run, this will overwrite current links in the institutional knowledge section with this new link. The anchor text is defined here in the meta_mappings as `description`. + +Note: + +1. The dbt `meta_mapping` config works at the model level, while the `column_meta_mapping` config works at the column level. The `add_owner` operation is not supported at the column level. +2. For string meta properties we support regex matching. +3. **List support**: YAML lists are now supported in meta properties. Each item in the list that matches the regex pattern will be processed. + +With regex matching, you can also use the matched value to customize how you populate the tag, term or owner fields. Here are a few advanced examples: + +##### Data Tier - Bronze, Silver, Gold + +If your meta section looks like this: + +```yaml +meta: + data_tier: Bronze # chosen from [Bronze,Gold,Silver] +``` + +and you wanted to attach a glossary term like `urn:li:glossaryTerm:Bronze` for all the models that have this value in the meta section attached to them, the following meta_mapping section would achieve that outcome: + +```yaml +meta_mapping: + data_tier: + match: "Bronze|Silver|Gold" + operation: "add_term" + config: + term: "{{ $match }}" +``` + +to match any data_tier of Bronze, Silver or Gold and maps it to a glossary term with the same name. + +##### Case Numbers - create tags + +If your meta section looks like this: + +```yaml +meta: + case: PLT-4678 # internal Case Number +``` + +and you want to generate tags that look like `case_4678` from this, you can use the following meta_mapping section: + +```yaml +meta_mapping: + case: + match: "PLT-(.*)" + operation: "add_tag" + config: + tag: "case_{{ $match }}" +``` + +##### Nested meta properties + +If your meta section has nested properties and looks like this: + +```yaml +meta: + data_governance: + team_owner: "Finance" +``` + +and you want attach term Finance_test in case of data_governance.team_owner is set to Finance, you can use the following meta_mapping section: + +```yaml +meta_mapping: + data_governance.team_owner: + match: "Finance" + operation: "add_term" + config: + term: "Finance_test" +``` + +Note: nested meta properties mapping is supported also for column_meta_mapping + +##### Stripping out leading @ sign + +You can also match specific groups within the value to extract subsets of the matched value. e.g. if you have a meta section that looks like this: + +```yaml +meta: + owner: "@finance-team" + business_owner: "@janet" +``` + +and you want to mark the finance-team as a group that owns the dataset (skipping the leading @ sign), while marking janet as an individual user (again, skipping the leading @ sign) that owns the dataset, you can use the following meta-mapping section. + +```yaml +meta_mapping: + owner: + match: "^@(.*)" + operation: "add_owner" + config: + owner_type: group + business_owner: + match: "^@(?P(.*))" + operation: "add_owner" + config: + owner_type: user + owner_category: BUSINESS_OWNER +``` + +In the examples above, we show two ways of writing the matching regexes. In the first one, `^@(.*)` the first matching group (a.k.a. match.group(1)) is automatically inferred. In the second example, `^@(?P(.*))`, we use a named matching group (called owner, since we are matching an owner) to capture the string we want to provide to the ownership urn. + +##### Working with Lists + +YAML lists are fully supported in dbt meta properties. Each item in the list is evaluated against the match pattern, and only matching items are processed. + +```yaml +meta: + owners: + - alice@company.com + - bob@company.com + - contractor@external.com +``` + +```yaml +meta_mapping: + owners: + match: ".*@company.com" + operation: "add_owner" + config: + owner_type: user +``` + +This will add `alice@company.com` and `bob@company.com` as owners (matching `.*@company.com`) but skip `contractor@external.com` (doesn't match the pattern). + +#### dbt query_tag automated mappings + +This works similarly as the dbt meta mapping but for the query tags + +We support the below actions - + +1. add_tag - Requires `tag` property in config. + +The below example set as global tag the query tag `tag` key's value. + +```json +"query_tag_mapping": +{ + "tag": + "match": ".*" + "operation": "add_tag" + "config": + "tag": "{{ $match }}" +} +``` + +#### Integrating with dbt test + +To integrate with dbt tests, the `dbt` source needs access to the `run_results.json` file generated after a `dbt test` or `dbt build` execution. Typically, this is written to the `target` directory. A common pattern you can follow is: + +1. Run `dbt build` +2. Copy the `target/run_results.json` file to a separate location. This is important, because otherwise subsequent `dbt` commands will overwrite the run results. +3. Run `dbt docs generate` to generate the `manifest.json` and `catalog.json` files +4. The dbt source makes use of the manifest, catalog, and run results file, and hence will need to be moved to a location accessible to the `dbt` source (e.g. s3 or local file system). In the ingestion recipe, the `run_results_paths` config must be set to the location of the `run_results.json` file from the `dbt build` or `dbt test` run. + +The connector will produce the following things: + +- Assertion definitions that are attached to the dataset (or datasets) +- Results from running the tests attached to the timeline of the dataset + +:::note Missing test results? + +The most common reason for missing test results is that the `run_results.json` with the test result information is getting overwritten by a subsequent `dbt` command. We recommend copying the `run_results.json` file before running other `dbt` commands. + +```sh +dbt source snapshot-freshness +dbt build +cp target/run_results.json target/run_results_backup.json +dbt docs generate +cp target/run_results_backup.json target/run_results.json +``` + +::: + +##### View of dbt tests for a dataset + +![test view](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/dbt-tests-view.png) + +##### Viewing the SQL for a dbt test + +![test logic view](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/dbt-test-logic-view.png) + +##### Viewing timeline for a failed dbt test + +![test view](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/dbt-tests-failure-view.png) + +##### Separating test result emission from other metadata emission + +You can segregate emission of test results from the emission of other dbt metadata using the `entities_enabled` config flag. +The following recipe shows you how to emit only test results. + +```yaml +source: + type: dbt + config: + manifest_path: _path_to_manifest_json + catalog_path: _path_to_catalog_json + run_results_paths: + - _path_to_run_results_json + target_platform: postgres + entities_enabled: + test_results: Only +``` + +Similarly, the following recipe shows you how to emit everything (i.e. models, sources, seeds, test definitions) but not test results: + +```yaml +source: + type: dbt + config: + manifest_path: _path_to_manifest_json + catalog_path: _path_to_catalog_json + run_results_paths: + - _path_to_run_results_json + target_platform: postgres + entities_enabled: + test_results: No +``` + +:::note Tests not showing up in the Assertion UI? + +Double check you are using the same `job_id` for your `test_results: Only` and `test_results: No` recipes. Otherwise you might end up reporting `tests_results` for tests that haven't had their `test_definitions` ingested yet. This can lead to orphaned assertions that do not show up under your dataset. + +If you choose not to share a `job_id`, you should adjust your recipe to also ingest the `test_definitions` + +```yaml +entities_enabled: + models: No + sources: No + seeds: No + snapshots: No + test_definitions: Yes + test_results: Yes +``` + +::: + +#### Multiple dbt projects + +In more complex dbt setups, you may have multiple dbt projects, where models from one project are used as sources in another project. +DataHub supports this setup natively. + +Each dbt project should have its own dbt ingestion recipe, and the `platform_instance` field in the recipe should be set to the dbt project name. + +For example, if you have two dbt projects `analytics` and `data_mart`, you would have two ingestion recipes. +If you have models in the `data_mart` project that are used as sources in the `analytics` project, the lineage will be automatically captured. + +```yaml +# Analytics dbt project +source: + type: dbt + config: + platform_instance: analytics + target_platform: postgres + manifest_path: analytics/target/manifest.json + catalog_path: analytics/target/catalog.json + # ... other configs +``` + +```yaml +# Data Mart dbt project +source: + type: dbt + config: + platform_instance: data_mart + target_platform: postgres + manifest_path: data_mart/target/manifest.json + catalog_path: data_mart/target/catalog.json + # ... other configs +``` + +If you have models that have tons of sources from other projects listed in the "Composed Of" section, it may also make sense to hide sources. + +#### Reducing "composed of" sprawl by hiding sources + +When many dbt projects use a single table as a source, the "Composed Of" relationships can become very large and difficult to navigate +and extra source nodes can clutter the lineage graph. + +This is particularly useful for multi-project setups, but can be useful in single-project setups as well. + +The benefit is that your entire dbt estate becomes much easier to navigate, and the borders between projects less noticeable. +The downside is that we will not pick up any documentation or meta mappings applied to dbt sources. + +To enable this, set `entities_enabled.sources: No` and `skip_sources_in_lineage: true` in your dbt source config: + +```yaml +source: + type: dbt + config: + platform_instance: analytics + target_platform: postgres + manifest_path: analytics/target/manifest.json + catalog_path: analytics/target/catalog.json + # ... other configs + entities_enabled: + sources: No + skip_sources_in_lineage: true +``` + +[Experimental] It's also possible to use `skip_sources_in_lineage: true` without disabling sources entirely. If you do this, sources will not participate in the lineage graph - they'll have upstreams but no downstreams. However, they will still contribute to docs, tags, etc to the warehouse entity. + +#### Semantic Views + +DataHub can ingest dbt models that have been materialized as `semantic_view` objects, a pattern used to define a semantic layer directly in warehouses like Snowflake. + +##### What are Materialized Semantic Views? + +A materialized [semantic view](https://docs.snowflake.com/en/user-guide/views-semantic/overview) is a dbt model (a `.sql` file) that uses the `materialized='semantic_view'` configuration via the [dbt_semantic_view package](https://github.com/Snowflake-Labs/dbt_semantic_view). This creates a `SEMANTIC VIEW` object in Snowflake, containing a rich set of metadata including dimensions and metrics. + +When you define a dbt model as a semantic view: + +```sql +-- models/sales_analytics.sql +{{ config( + materialized='semantic_view' +) }} + +TABLES ( + OrdersTable AS {{ source('coffee_shop_source', 'ORDERS') }} +) +DIMENSIONS ( + OrdersTable.CUSTOMER_ID AS CUSTOMER_ID +) +METRICS ( + OrdersTable.GROSS_REVENUE AS SUM(ORDER_TOTAL) +) +``` + +DataHub will: + +1. Create a dataset with the subtype `Semantic View`. +2. Create sibling relationships to the underlying Snowflake `SEMANTIC VIEW` object. +3. Extract column-level lineage from the semantic view's DDL. + +##### Configuration + +Semantic views are dbt models with `materialized='semantic_view'`. They are emitted by default along with other models when `entities_enabled.models: Yes` (the default). + +##### How Semantic Views Appear in DataHub + +- **Subtype**: Datasets are tagged with the subtype `Semantic View`. +- **Lineage**: Upstream lineage to the source dbt model is created, with column-level lineage where available. + +##### Column-Level Lineage for Snowflake Semantic Views + +For dbt models materialized as semantic views in Snowflake, DataHub can extract column-level lineage from the compiled DDL. This requires: + +1. Using Snowflake as the `target_platform`. +2. Having the `compiled_code` for the model available in the dbt manifest or dbt Cloud API. + +If these conditions are not met, warnings will appear in the ingestion report: + +| Condition | Warning | +| ----------------------- | --------------------------------------- | +| Non-Snowflake adapter | `Semantic View CLL Unsupported Adapter` | +| Missing `compiled_code` | `Semantic View Missing compiled_code` | +| Empty CLL results | `Semantic View CLL Empty` | +| Parsing failure | `Semantic View CLL Parsing Failed` | + +> **Note: Limitations** +> +> Column-level lineage is currently only supported for Snowflake semantic views, as it relies on parsing the Snowflake-specific DDL. + +#### Exposures + +DataHub supports ingesting [dbt exposures](https://docs.getdbt.com/docs/build/exposures) - downstream consumers of your dbt models such as dashboards, notebooks, ML models, and applications. + +##### What are dbt Exposures? + +Exposures define how your dbt models are used downstream. They help you understand the full picture of your data ecosystem by documenting: + +- **Dashboards** - BI tools like Looker, Tableau, or Metabase +- **Notebooks** - Jupyter notebooks or other analysis tools +- **ML Models** - Machine learning pipelines consuming your data +- **Applications** - Apps or services that use your data +- **Analysis** - Ad-hoc analysis or reports + +##### How Exposures Map to DataHub + +dbt exposures are ingested as **Dashboard** entities in DataHub, with the exposure type preserved as a subtype: + +| dbt Exposure Type | DataHub SubType | +| ----------------- | --------------- | +| `dashboard` | Dashboard | +| `notebook` | Notebook | +| `ml` | ML Model | +| `application` | Application | +| `analysis` | Analysis | + +##### Configuration + +Exposures are enabled by default. You can control their emission using the `entities_enabled.exposures` config: + +```yaml +source: + type: dbt + config: + manifest_path: _path_to_manifest_json + catalog_path: _path_to_catalog_json + target_platform: postgres + entities_enabled: + exposures: Yes # Default - emit exposures + # exposures: No # Disable exposure ingestion + # exposures: Only # Only emit exposures, skip other entities +``` + +##### Lineage + +Exposures automatically create lineage relationships to their upstream dbt models. The `depends_on` field in your exposure definition determines which models appear as upstreams: + +```yaml +# models/exposures.yml +exposures: + - name: weekly_metrics_dashboard + type: dashboard + owner: + email: analytics@company.com + depends_on: + - ref('orders') + - ref('customers') + url: https://bi.company.com/dashboards/weekly-metrics +``` + +This creates lineage: `orders` → `weekly_metrics_dashboard` and `customers` → `weekly_metrics_dashboard`. + +##### Owner Resolution + +Owner resolution for exposures is **the same** as for other dbt assets (models, sources, etc.). The `enable_owner_extraction` config applies to all of them: when `true` (default), ownership is extracted; when `false`, no ownership aspect is emitted for any dbt asset, including exposures. + +For exposures, the owner is read from the exposure definition: + +- **`owner.email`** (recommended): used as the user URN (e.g., `analytics@company.com` → `urn:li:corpuser:analytics@company.com`) +- **`owner.name`** (fallback): converted to lowercase with underscores (e.g., `"John Doe"` → `urn:li:corpuser:john_doe`) + +The same configs apply as elsewhere: `strip_user_ids_from_email` strips the domain from email-based owners when set. + +:::note +When only `owner.name` is provided (without email), the generated URN may not match existing users in DataHub. We recommend providing `owner.email` for accurate user matching. +::: + +##### Example Output + +An exposure like: + +```yaml +exposures: + - name: weekly_metrics_dashboard + type: dashboard + description: Weekly business metrics for leadership + owner: + email: analytics@company.com + maturity: high + url: https://bi.company.com/dashboards/123 + depends_on: + - ref('orders') +``` + +Will create a Dashboard entity in DataHub with: + +- **Title**: `weekly_metrics_dashboard` +- **Description**: `Weekly business metrics for leadership` +- **SubType**: `Dashboard`, `Notebook`, `ML Model`, `Application`, `Analysis` +- **Owner**: `urn:li:corpuser:analytics@company.com` +- **External URL**: `https://bi.company.com/dashboards/123` +- **Upstream Lineage**: Link to the `orders` dbt model +- **Custom Properties**: `exposure_type`, `maturity`, `dbt_unique_id` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.dbt.dbt_core.DBTCoreSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py) + + + +## Module `dbt-cloud` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Column-level Lineage | ✅ | Enabled by default, configure using `include_column_lineage`. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Table-Level Lineage | ✅ | Enabled by default. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `dbt-cloud` module ingests metadata from Dbt into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +#### Setup + +Extracts dbt metadata from dbt Cloud APIs. + +Create a [service account token](https://docs.getdbt.com/docs/dbt-cloud-apis/service-tokens) with "Metadata Only" permission (read-only). + +##### Operating Modes + +The dbt Cloud source supports two modes of operation: + +##### 1. Explicit Mode (Default) + +Specify a single dbt Cloud job to ingest metadata from. The job must have "Generate docs on run" enabled and should process all/most models (otherwise multiple job ingestion may be required). + +To get the required IDs, go to the job details page (this is the one with the "Run History" table), and look at the URL. +It should look something like this: https://cloud.getdbt.com/next/deploy/107298/projects/175705/jobs/148094. +In this example, the account ID is 107298, the project ID is 175705, and the job ID is 148094. + +##### 2. Auto-Discovery Mode + +Automatically discovers and ingests metadata from all eligible jobs in a dbt Cloud project. This mode: + +- Discovers all jobs in the specified project's **production environment only** +- Filters to jobs with **"Generate docs on run" enabled** (`generate_docs=True`) +- Always uses the **latest run** for each job (ignores `run_id` configuration) +- Supports optional regex-based filtering to include/exclude specific job IDs +- Ingests metadata from multiple jobs in a single run + +**When to use auto-discovery:** + +- You have multiple dbt Cloud jobs in a project and want to ingest all of them +- You want to automatically pick up new jobs without updating configuration + +**Requirements:** + +- Jobs must be in the production environment +- Jobs must have "Generate docs on run" enabled + + +### Install the Plugin +```shell +pip install 'acryl-datahub[dbt-cloud]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "dbt-cloud" + config: + token: ${DBT_CLOUD_TOKEN} + + # In the URL https://cloud.getdbt.com/next/deploy/107298/projects/175705/jobs/148094, + # 107298 is the account_id, 175705 is the project_id, and 148094 is the job_id + + account_id: "${DBT_ACCOUNT_ID}" # set to your dbt cloud account id + project_id: "${DBT_PROJECT_ID}" # set to your dbt cloud project id + + # Mode 1: Explicit Mode (specify a single job) + job_id: "${DBT_JOB_ID}" # set to your dbt cloud job id + run_id: # optional: set to a specific dbt cloud run id. Defaults to the latest run + + # Mode 2: Auto-Discovery Mode (automatically discover all eligible jobs) + # Uncomment the section below to enable auto-discovery + # Note: When auto_discovery is enabled, job_id can be omitted (will be ignored if provided) + # and run_id is ignored (always uses the latest run) + # auto_discovery: + # enabled: true + # job_id_pattern: # optional + # allow: + # - ".*" # regex pattern to include jobs (default: include all) + # # deny: + # # - "test.*" # optional: regex pattern to exclude specific jobs + + target_platform: "${TARGET_PLATFORM_ID}" # e.g. bigquery/postgres/snowflake/etc. + # convert_urns_to_lowercase: true # optional: lowercase dbt URNs to avoid duplicates from casing differences (default: true for Snowflake) + +# sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
account_id ✅
integer
| The DBT Cloud account ID to use. | +|
project_id ✅
integer
| The dbt Cloud project ID to use. | +|
target_platform ✅
string
| The platform that dbt is loading onto. (e.g. bigquery / redshift / postgres etc.) | +|
token ✅
string(password)
| The API token to use to authenticate with DBT Cloud. | +|
access_url
string
| The base URL of the dbt Cloud instance to use. This should be the URL you use to access the dbt Cloud UI. It should include the scheme (http/https) and not include a trailing slash. See the access url for your dbt Cloud region here: https://docs.getdbt.com/docs/cloud/about-cloud/regions-ip-addresses
Default: https://cloud.getdbt.com
| +|
column_meta_mapping
object
| mapping rules that will be executed against dbt column meta properties. Refer to the section below on dbt meta automated mappings.
Default: {}
| +|
convert_column_urns_to_lowercase
boolean
| When enabled, converts column URNs to lowercase to ensure cross-platform compatibility. If `target_platform` is Snowflake, the default is True.
Default: False
| +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
dbt_is_primary_sibling
boolean
| Experimental: Controls sibling relationship primary designation between dbt entities and target platform entities. When True (default), dbt entities are primary and target platform entities are secondary. When False, target platform entities are primary and dbt entities are secondary. Uses aspect patches for precise control. Requires DataHub server 1.3.0+.
Default: True
| +|
drop_duplicate_sources
boolean
| When enabled, drops sources that have the same name in the target platform as a model. This ensures that lineage is generated reliably, but will lose any documentation associated only with the source.
Default: True
| +|
enable_meta_mapping
boolean
| When enabled, applies the mappings that are defined through the meta_mapping directives.
Default: True
| +|
enable_owner_extraction
boolean
| When enabled, ownership info will be extracted from the dbt meta
Default: True
| +|
enable_query_tag_mapping
boolean
| When enabled, applies the mappings that are defined through the `query_tag_mapping` directives.
Default: True
| +|
external_url_mode
Enum
| One of: "explore", "ide"
Default: explore
| +|
include_column_lineage
boolean
| When enabled, column-level lineage will be extracted from the dbt node definition. Requires `infer_dbt_schemas` to be enabled. If you run into issues where the column name casing does not match up with properly, providing a datahub_api or using the rest sink will improve accuracy.
Default: True
| +|
include_compiled_code
boolean
| When enabled, includes the compiled code in the emitted metadata.
Default: True
| +|
include_database_name
boolean
| Whether to add database name to the table urn. Set to False to skip it for engines like AWS Athena where it's not required.
Default: True
| +|
include_env_in_assertion_guid
boolean
| Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion that are only distinguished by env, then you should set this flag to True.
Default: False
| +|
incremental_lineage
boolean
| When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run. This would also require enabling 'incremental_lineage' in the counterpart warehouse ingestion (_e.g._ BigQuery, Redshift, etc).
Default: True
| +|
infer_dbt_schemas
boolean
| When enabled, schemas will be inferred from the dbt node definition.
Default: True
| +|
job_id
One of integer, null
| The ID of the job to ingest metadata from. Required in explicit mode (when auto_discovery is disabled).
Default: None
| +|
max_queries_per_model
integer
| Maximum number of Query entities to emit per dbt model. Prevents metadata explosion from malformed manifests. Set to 0 for unlimited.
Default: 100
| +|
meta_mapping
object
| mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.
Default: {}
| +|
metadata_endpoint
string
| The dbt Cloud metadata API endpoint. If not provided, we will try to infer it from the access_url.
Default: https://metadata.cloud.getdbt.com/graphql
| +|
owner_extraction_pattern
One of string, null
| Regex string to extract owner from the dbt node using the `(?P...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r"(?P(.*)): (\w+) (\w+)"` will extract `jdoe` as the owner from `"jdoe: John Doe"` (2) `r"@(?P(.*))"` will extract `alice` as the owner from `"@alice"`.
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
prefer_sql_parser_lineage
boolean
| Normally we use dbt's metadata to generate table lineage. When enabled, we prefer results from the SQL parser when generating lineage instead. This can be useful when dbt models reference tables directly, instead of using the ref() macro. This requires that `skip_sources_in_lineage` is enabled.
Default: False
| +|
query_tag_mapping
object
| mapping rules that will be executed against dbt query_tag meta properties. Refer to the section below on dbt meta automated mappings.
Default: {}
| +|
run_id
One of integer, null
| The ID of the run to ingest metadata from. If not specified, defaults to the latest run. In auto-discovery mode, always uses the latest run for each job.
Default: None
| +|
skip_sources_in_lineage
boolean
| [Experimental] When enabled, dbt sources will not be included in the lineage graph. Requires that `entities_enabled.sources` is set to `NO`. This is mainly useful when you have multiple, interdependent dbt projects.
Default: False
| +|
strip_user_ids_from_email
boolean
| Whether or not to strip email id while adding owners using dbt meta actions.
Default: False
| +|
tag_prefix
string
| Prefix added to tags during ingestion.
Default: dbt:
| +|
target_platform_instance
One of string, null
| The platform instance for the platform that dbt is operating on. Use this if you have multiple instances of the same platform (e.g. redshift) and need to distinguish between them.
Default: None
| +|
test_warnings_are_errors
boolean
| When enabled, dbt test warnings will be treated as failures.
Default: False
| +|
use_identifiers
boolean
| Use model identifier instead of model name if defined (if not, default to model name).
Default: False
| +|
write_semantics
string
| Whether the new tags, terms and owners to be added will override the existing ones added only by this source or not. Value for this config can be "PATCH" or "OVERRIDE"
Default: PATCH
| +|
env
string
| Environment to use in namespace when constructing URNs.
Default: PROD
| +|
auto_discovery
One of AutoDiscoveryConfig, null
| Auto-discovery configuration. When enabled, automatically discovers jobs for the specified project.
Default: None
| +|
auto_discovery.enabled
boolean
| Enable/disable auto-discovery mode. When enabled, discovers jobs for the specified project. Only production jobs with generate_docs=True are ingested.
Default: False
| +|
auto_discovery.job_id_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
auto_discovery.job_id_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
entities_enabled
DBTEntitiesEnabled
| Controls which dbt entities are going to be emitted by this source | +|
entities_enabled.exposures
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.model_performance
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.models
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.queries
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.seeds
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.semantic_models
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.snapshots
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.sources
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.test_definitions
Enum
| One of: "YES", "NO", "ONLY" | +|
entities_enabled.test_results
Enum
| One of: "YES", "NO", "ONLY" | +|
materialized_node_pattern
MaterializedNodePatternConfig
| Configuration for filtering materialized nodes based on their physical location | +|
materialized_node_pattern.database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
materialized_node_pattern.database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
materialized_node_pattern.schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
materialized_node_pattern.schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
materialized_node_pattern.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
materialized_node_pattern.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
node_name_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
node_name_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| DBT Stateful Ingestion Config.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AutoDiscoveryConfig": { + "additionalProperties": false, + "description": "Configuration for auto-discovery mode that automatically discovers jobs for a project.\nRef: DBT Jobs: http://docs.getdbt.com/docs/deploy/jobs\nTODO: The configuration is oraganised this way to allow for future expansion to project discovery at account level.", + "properties": { + "enabled": { + "default": false, + "description": "Enable/disable auto-discovery mode. When enabled, discovers jobs for the specified project. Only production jobs with generate_docs=True are ingested.", + "title": "Enabled", + "type": "boolean" + }, + "job_id_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "description": "Regex patterns to filter jobs by job_id when auto-discovering." + } + }, + "title": "AutoDiscoveryConfig", + "type": "object" + }, + "DBTEntitiesEnabled": { + "additionalProperties": false, + "description": "Controls which dbt entities are going to be emitted by this source", + "properties": { + "models": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt models when set to Yes or Only" + }, + "sources": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt sources when set to Yes or Only" + }, + "seeds": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt seeds when set to Yes or Only" + }, + "snapshots": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt snapshots when set to Yes or Only" + }, + "test_definitions": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for test definitions when enabled when set to Yes or Only" + }, + "test_results": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for test results when set to Yes or Only" + }, + "model_performance": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit model performance metadata when set to Yes or Only. Only supported with dbt core." + }, + "exposures": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt exposures when set to Yes or Only. Exposures represent downstream consumers like dashboards, notebooks, or applications." + }, + "semantic_models": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit metadata for dbt semantic models when set to Yes or Only. Semantic models define entities, dimensions, and measures for the dbt semantic layer (dbt 1.6+)." + }, + "queries": { + "$ref": "#/$defs/EmitDirective", + "default": "YES", + "description": "Emit Query entities from meta.queries field when set to Yes or Only." + } + }, + "title": "DBTEntitiesEnabled", + "type": "object" + }, + "EmitDirective": { + "description": "A holder for directives for emission for specific types of entities", + "enum": [ + "YES", + "NO", + "ONLY" + ], + "title": "EmitDirective", + "type": "string" + }, + "MaterializedNodePatternConfig": { + "additionalProperties": false, + "description": "Configuration for filtering materialized nodes based on their physical location", + "properties": { + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for database names to filter materialized nodes." + }, + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schema names in format '{database}.{schema}' to filter materialized nodes." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for table/view names in format '{database}.{schema}.{table}' to filter materialized nodes." + } + }, + "title": "MaterializedNodePatternConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "incremental_lineage": { + "default": true, + "description": "When enabled, emits incremental/patch lineage for non-dbt entities. When disabled, re-states lineage on each run. This would also require enabling 'incremental_lineage' in the counterpart warehouse ingestion (_e.g._ BigQuery, Redshift, etc).", + "title": "Incremental Lineage", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "Environment to use in namespace when constructing URNs.", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "DBT Stateful Ingestion Config." + }, + "target_platform": { + "description": "The platform that dbt is loading onto. (e.g. bigquery / redshift / postgres etc.)", + "title": "Target Platform", + "type": "string" + }, + "target_platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The platform instance for the platform that dbt is operating on. Use this if you have multiple instances of the same platform (e.g. redshift) and need to distinguish between them.", + "title": "Target Platform Instance" + }, + "use_identifiers": { + "default": false, + "description": "Use model identifier instead of model name if defined (if not, default to model name).", + "title": "Use Identifiers", + "type": "boolean" + }, + "entities_enabled": { + "$ref": "#/$defs/DBTEntitiesEnabled", + "default": { + "models": "YES", + "sources": "YES", + "seeds": "YES", + "snapshots": "YES", + "test_definitions": "YES", + "test_results": "YES", + "model_performance": "YES", + "exposures": "YES", + "semantic_models": "YES", + "queries": "YES" + }, + "description": "Controls for enabling / disabling metadata emission for different dbt entities (models, test definitions, test results, etc.)" + }, + "prefer_sql_parser_lineage": { + "default": false, + "description": "Normally we use dbt's metadata to generate table lineage. When enabled, we prefer results from the SQL parser when generating lineage instead. This can be useful when dbt models reference tables directly, instead of using the ref() macro. This requires that `skip_sources_in_lineage` is enabled.", + "title": "Prefer Sql Parser Lineage", + "type": "boolean" + }, + "skip_sources_in_lineage": { + "default": false, + "description": "[Experimental] When enabled, dbt sources will not be included in the lineage graph. Requires that `entities_enabled.sources` is set to `NO`. This is mainly useful when you have multiple, interdependent dbt projects. ", + "title": "Skip Sources In Lineage", + "type": "boolean" + }, + "tag_prefix": { + "default": "dbt:", + "description": "Prefix added to tags during ingestion.", + "title": "Tag Prefix", + "type": "string" + }, + "node_name_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for dbt model names to filter in ingestion." + }, + "materialized_node_pattern": { + "$ref": "#/$defs/MaterializedNodePatternConfig", + "default": { + "database_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "schema_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + } + }, + "description": "Advanced filtering for materialized nodes based on their physical database location. Provides fine-grained control over database.schema.table patterns for catalog consistency." + }, + "meta_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against dbt meta properties. Refer to the section below on dbt meta automated mappings.", + "title": "Meta Mapping", + "type": "object" + }, + "column_meta_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against dbt column meta properties. Refer to the section below on dbt meta automated mappings.", + "title": "Column Meta Mapping", + "type": "object" + }, + "enable_meta_mapping": { + "default": true, + "description": "When enabled, applies the mappings that are defined through the meta_mapping directives.", + "title": "Enable Meta Mapping", + "type": "boolean" + }, + "query_tag_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against dbt query_tag meta properties. Refer to the section below on dbt meta automated mappings.", + "title": "Query Tag Mapping", + "type": "object" + }, + "enable_query_tag_mapping": { + "default": true, + "description": "When enabled, applies the mappings that are defined through the `query_tag_mapping` directives.", + "title": "Enable Query Tag Mapping", + "type": "boolean" + }, + "write_semantics": { + "default": "PATCH", + "description": "Whether the new tags, terms and owners to be added will override the existing ones added only by this source or not. Value for this config can be \"PATCH\" or \"OVERRIDE\"", + "title": "Write Semantics", + "type": "string" + }, + "strip_user_ids_from_email": { + "default": false, + "description": "Whether or not to strip email id while adding owners using dbt meta actions.", + "title": "Strip User Ids From Email", + "type": "boolean" + }, + "enable_owner_extraction": { + "default": true, + "description": "When enabled, ownership info will be extracted from the dbt meta", + "title": "Enable Owner Extraction", + "type": "boolean" + }, + "max_queries_per_model": { + "default": 100, + "description": "Maximum number of Query entities to emit per dbt model. Prevents metadata explosion from malformed manifests. Set to 0 for unlimited.", + "minimum": 0, + "title": "Max Queries Per Model", + "type": "integer" + }, + "owner_extraction_pattern": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Regex string to extract owner from the dbt node using the `(?P...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r\"(?P(.*)): (\\w+) (\\w+)\"` will extract `jdoe` as the owner from `\"jdoe: John Doe\"` (2) `r\"@(?P(.*))\"` will extract `alice` as the owner from `\"@alice\"`.", + "title": "Owner Extraction Pattern" + }, + "include_env_in_assertion_guid": { + "default": false, + "description": "Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion that are only distinguished by env, then you should set this flag to True.", + "title": "Include Env In Assertion Guid", + "type": "boolean" + }, + "convert_column_urns_to_lowercase": { + "default": false, + "description": "When enabled, converts column URNs to lowercase to ensure cross-platform compatibility. If `target_platform` is Snowflake, the default is True.", + "title": "Convert Column Urns To Lowercase", + "type": "boolean" + }, + "test_warnings_are_errors": { + "default": false, + "description": "When enabled, dbt test warnings will be treated as failures.", + "title": "Test Warnings Are Errors", + "type": "boolean" + }, + "infer_dbt_schemas": { + "default": true, + "description": "When enabled, schemas will be inferred from the dbt node definition.", + "title": "Infer Dbt Schemas", + "type": "boolean" + }, + "include_column_lineage": { + "default": true, + "description": "When enabled, column-level lineage will be extracted from the dbt node definition. Requires `infer_dbt_schemas` to be enabled. If you run into issues where the column name casing does not match up with properly, providing a datahub_api or using the rest sink will improve accuracy.", + "title": "Include Column Lineage", + "type": "boolean" + }, + "include_compiled_code": { + "default": true, + "description": "When enabled, includes the compiled code in the emitted metadata.", + "title": "Include Compiled Code", + "type": "boolean" + }, + "include_database_name": { + "default": true, + "description": "Whether to add database name to the table urn. Set to False to skip it for engines like AWS Athena where it's not required.", + "title": "Include Database Name", + "type": "boolean" + }, + "dbt_is_primary_sibling": { + "default": true, + "description": "Experimental: Controls sibling relationship primary designation between dbt entities and target platform entities. When True (default), dbt entities are primary and target platform entities are secondary. When False, target platform entities are primary and dbt entities are secondary. Uses aspect patches for precise control. Requires DataHub server 1.3.0+.", + "title": "Dbt Is Primary Sibling", + "type": "boolean" + }, + "drop_duplicate_sources": { + "default": true, + "description": "When enabled, drops sources that have the same name in the target platform as a model. This ensures that lineage is generated reliably, but will lose any documentation associated only with the source.", + "title": "Drop Duplicate Sources", + "type": "boolean" + }, + "access_url": { + "default": "https://cloud.getdbt.com", + "description": "The base URL of the dbt Cloud instance to use. This should be the URL you use to access the dbt Cloud UI. It should include the scheme (http/https) and not include a trailing slash. See the access url for your dbt Cloud region here: https://docs.getdbt.com/docs/cloud/about-cloud/regions-ip-addresses", + "title": "Access Url", + "type": "string" + }, + "metadata_endpoint": { + "default": "https://metadata.cloud.getdbt.com/graphql", + "description": "The dbt Cloud metadata API endpoint. If not provided, we will try to infer it from the access_url.", + "title": "Metadata Endpoint", + "type": "string" + }, + "token": { + "description": "The API token to use to authenticate with DBT Cloud.", + "format": "password", + "title": "Token", + "type": "string", + "writeOnly": true + }, + "account_id": { + "description": "The DBT Cloud account ID to use.", + "title": "Account Id", + "type": "integer" + }, + "project_id": { + "description": "The dbt Cloud project ID to use.", + "title": "Project Id", + "type": "integer" + }, + "job_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the job to ingest metadata from. Required in explicit mode (when auto_discovery is disabled).", + "title": "Job Id" + }, + "run_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The ID of the run to ingest metadata from. If not specified, defaults to the latest run. In auto-discovery mode, always uses the latest run for each job.", + "title": "Run Id" + }, + "auto_discovery": { + "anyOf": [ + { + "$ref": "#/$defs/AutoDiscoveryConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Auto-discovery configuration. When enabled, automatically discovers jobs for the specified project." + }, + "external_url_mode": { + "default": "explore", + "description": "Where should the \"View in dbt\" link point to - either the \"Explore\" UI or the dbt Cloud IDE", + "enum": [ + "explore", + "ide" + ], + "title": "External Url Mode", + "type": "string" + } + }, + "required": [ + "target_platform", + "token", + "account_id", + "project_id" + ], + "title": "DBTCloudConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.dbt.dbt_cloud.DBTCloudSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for dbt, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/delta-lake.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/delta-lake.md new file mode 100644 index 00000000..c0aa33fa --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/delta-lake.md @@ -0,0 +1,728 @@ +--- +sidebar_position: 21 +title: Delta Lake +slug: /generated/ingestion/sources/delta-lake +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Delta Lake + +## Overview + +Delta Lake is a data platform used to store and query analytical or operational data. Learn more in the [official Delta Lake documentation](https://delta.io/). + +The DataHub integration for Delta Lake covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `delta-lake` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Folder. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Extract Tags | ✅ | Can extract S3 object/bucket tags if enabled. | + +### Overview + +The `delta-lake` module ingests metadata from Delta Lake into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[delta-lake]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: delta-lake + config: + env: "PROD" + platform_instance: "my-delta-lake" + base_path: "/path/to/data/folder" + +sink: + # sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
base_path ✅
string
| Path to table (s3 or local file system). If path is not a delta table path then all subfolders will be scanned to detect and ingest delta tables. | +|
platform
string
| The platform that this source connects to
Default: delta-lake
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to
Default: None
| +|
relative_path
One of string, null
| If set, delta-tables will be searched at location '/' and URNs will be created using relative_path only.
Default: None
| +|
require_files
One of boolean, null
| Whether DeltaTable should track files. Consider setting this to `False` for large delta tables, resulting in significant memory reduction for ingestion process.When set to `False`, number_of_files in delta table can not be reported.
Default: True
| +|
version_history_lookback
One of integer, null
| Number of previous version histories to be ingested. Defaults to 1. If set to -1 all version history will be ingested.
Default: 1
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
s3
One of S3, null
|
Default: None
| +|
s3.use_s3_bucket_tags
One of boolean, null
| Whether or not to create tags in datahub from the s3 bucket
Default: False
| +|
s3.use_s3_object_tags
One of boolean, null
| # Whether or not to create tags in datahub from the s3 object
Default: False
| +|
s3.aws_config
One of AwsConnectionConfig, null
| AWS configuration
Default: None
| +|
s3.aws_config.aws_access_key_id
One of string, null
| AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
s3.aws_config.aws_advanced_config
object
| Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html). | +|
s3.aws_config.aws_endpoint_url
One of string, null
| The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.
Default: None
| +|
s3.aws_config.aws_profile
One of string, null
| The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.
Default: None
| +|
s3.aws_config.aws_proxy
One of string, null
| A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.
Default: None
| +|
s3.aws_config.aws_region
One of string, null
| AWS region code.
Default: None
| +|
s3.aws_config.aws_retry_mode
Enum
| One of: "legacy", "standard", "adaptive"
Default: standard
| +|
s3.aws_config.aws_retry_num
integer
| Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.
Default: 5
| +|
s3.aws_config.aws_secret_access_key
One of string(password), null
| AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
s3.aws_config.aws_session_token
One of string(password), null
| AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
s3.aws_config.read_timeout
number
| The timeout for reading from the connection (in seconds).
Default: 60
| +|
s3.aws_config.aws_role
One of string, array, null
| AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).
Default: None
| +|
s3.aws_config.aws_role.union
One of string, AwsAssumeRoleConfig
| | +|
s3.aws_config.aws_role.union.RoleArn ❓
string
| ARN of the role to assume. | +|
s3.aws_config.aws_role.union.ExternalId
One of string, null
| External ID to use when assuming the role.
Default: None
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Stateful Ingestion Config with stale metadata removal
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AwsAssumeRoleConfig": { + "additionalProperties": true, + "properties": { + "RoleArn": { + "description": "ARN of the role to assume.", + "title": "Rolearn", + "type": "string" + }, + "ExternalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "External ID to use when assuming the role.", + "title": "Externalid" + } + }, + "required": [ + "RoleArn" + ], + "title": "AwsAssumeRoleConfig", + "type": "object" + }, + "AwsConnectionConfig": { + "additionalProperties": false, + "description": "Common AWS credentials config.\n\nCurrently used by:\n - Glue source\n - SageMaker source\n - dbt source", + "properties": { + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Session Token" + }, + "aws_role": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/AwsAssumeRoleConfig" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).", + "title": "Aws Role" + }, + "aws_profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.", + "title": "Aws Profile" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region code.", + "title": "Aws Region" + }, + "aws_endpoint_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.", + "title": "Aws Endpoint Url" + }, + "aws_proxy": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", + "title": "Aws Proxy" + }, + "aws_retry_num": { + "default": 5, + "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "title": "Aws Retry Num", + "type": "integer" + }, + "aws_retry_mode": { + "default": "standard", + "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "enum": [ + "legacy", + "standard", + "adaptive" + ], + "title": "Aws Retry Mode", + "type": "string" + }, + "read_timeout": { + "default": 60, + "description": "The timeout for reading from the connection (in seconds).", + "title": "Read Timeout", + "type": "number" + }, + "aws_advanced_config": { + "additionalProperties": true, + "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).", + "title": "Aws Advanced Config", + "type": "object" + } + }, + "title": "AwsConnectionConfig", + "type": "object" + }, + "S3": { + "additionalProperties": false, + "properties": { + "aws_config": { + "anyOf": [ + { + "$ref": "#/$defs/AwsConnectionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS configuration" + }, + "use_s3_bucket_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether or not to create tags in datahub from the s3 bucket", + "title": "Use S3 Bucket Tags" + }, + "use_s3_object_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "# Whether or not to create tags in datahub from the s3 object", + "title": "Use S3 Object Tags" + } + }, + "title": "S3", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Stateful Ingestion Config with stale metadata removal" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to", + "title": "Platform Instance" + }, + "base_path": { + "description": "Path to table (s3 or local file system). If path is not a delta table path then all subfolders will be scanned to detect and ingest delta tables.", + "title": "Base Path", + "type": "string" + }, + "relative_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If set, delta-tables will be searched at location '/' and URNs will be created using relative_path only.", + "title": "Relative Path" + }, + "platform": { + "const": "delta-lake", + "default": "delta-lake", + "description": "The platform that this source connects to", + "title": "Platform", + "type": "string" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for tables to filter in ingestion." + }, + "version_history_lookback": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 1, + "description": "Number of previous version histories to be ingested. Defaults to 1. If set to -1 all version history will be ingested.", + "title": "Version History Lookback" + }, + "require_files": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether DeltaTable should track files. Consider setting this to `False` for large delta tables, resulting in significant memory reduction for ingestion process.When set to `False`, number_of_files in delta table can not be reported.", + "title": "Require Files" + }, + "s3": { + "anyOf": [ + { + "$ref": "#/$defs/S3" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "base_path" + ], + "title": "DeltaLakeSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Delta Table on Local File System + +##### Step 1 + +Create a delta table using the sample PySpark code below if you don't have a delta table you can point to. + +```python +import uuid +import random +from pyspark.sql import SparkSession +from delta.tables import DeltaTable + +def generate_data(): + return [(y, m, d, str(uuid.uuid4()), str(random.randrange(10000) % 26 + 65) * 3, random.random()*10000) + for d in range(1, 29) + for m in range(1, 13) + for y in range(2000, 2021)] + +jar_packages = ["org.apache.hadoop:hadoop-aws:3.2.3", "io.delta:delta-core_2.12:1.2.1"] +spark = SparkSession.builder \ + .appName("quickstart") \ + .master("local[*]") \ + .config("spark.jars.packages", ",".join(jar_packages)) \ + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ + .getOrCreate() + +table_path = "quickstart/my-table" +columns = ["year", "month", "day", "sale_id", "customer", "total_cost"] +spark.sparkContext.parallelize(generate_data()).toDF(columns).repartition(1).write.format("delta").save(table_path) + +df = spark.read.format("delta").load(table_path) +df.show() + +``` + +##### Step 2 + +Create a datahub ingestion yaml file (delta.dhub.yaml) to ingest metadata from the delta table you just created. + +```yaml +source: + type: "delta-lake" + config: + base_path: "quickstart/my-table" + +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +Note: Make sure you run the Spark code as well as recipe from same folder otherwise use absolute paths. + +##### Step 3 + +Execute the ingestion recipe: + +```shell +datahub ingest -c delta.dhub.yaml +``` + +#### Delta Table on S3 + +##### Step 1 + +Set up your AWS credentials by creating an AWS credentials config file; typically in '$HOME/.aws/credentials'. + +``` +[my-creds] +aws_access_key_id: ###### +aws_secret_access_key: ###### +``` + +Step 2: Create a Delta Table using the PySpark sample code below unless you already have Delta Tables on your S3. + +```python +from pyspark.sql import SparkSession +from delta.tables import DeltaTable +from configparser import ConfigParser +import uuid +import random +def generate_data(): + return [(y, m, d, str(uuid.uuid4()), str(random.randrange(10000) % 26 + 65) * 3, random.random()*10000) + for d in range(1, 29) + for m in range(1, 13) + for y in range(2000, 2021)] + +jar_packages = ["org.apache.hadoop:hadoop-aws:3.2.3", "io.delta:delta-core_2.12:1.2.1"] +spark = SparkSession.builder \ + .appName("quickstart") \ + .master("local[*]") \ + .config("spark.jars.packages", ",".join(jar_packages)) \ + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension") \ + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") \ + .getOrCreate() + + +config_object = ConfigParser() +config_object.read("$HOME/.aws/credentials") +profile_info = config_object["my-creds"] +access_id = profile_info["aws_access_key_id"] +access_key = profile_info["aws_secret_access_key"] + +hadoop_conf = spark._jsc.hadoopConfiguration() +hadoop_conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") +hadoop_conf.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider") +hadoop_conf.set("fs.s3a.access.key", access_id) +hadoop_conf.set("fs.s3a.secret.key", access_key) + +table_path = "s3a://my-bucket/my-folder/sales-table" +columns = ["year", "month", "day", "sale_id", "customer", "total_cost"] +spark.sparkContext.parallelize(generate_data()).toDF(columns).repartition(1).write.format("delta").save(table_path) +df = spark.read.format("delta").load(table_path) +df.show() + +``` + +##### Step 3 + +Create a datahub ingestion yaml file (delta.s3.dhub.yaml) to ingest metadata from the delta table you just created. + +```yml +source: + type: "delta-lake" + config: + base_path: "s3://my-bucket/my-folder/sales-table" + s3: + aws_config: + aws_access_key_id: <> + aws_secret_access_key: <> + +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +##### Step 4 + +Execute the ingestion recipe: + +```shell +datahub ingest -c delta.s3.dhub.yaml +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.delta_lake.source.DeltaLakeSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/delta_lake/source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Delta Lake, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/demo-data.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/demo-data.md new file mode 100644 index 00000000..935fd221 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/demo-data.md @@ -0,0 +1,126 @@ +--- +sidebar_position: 22 +title: Demo Data +slug: /generated/ingestion/sources/demo-data +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Demo Data + +## Overview + +Demo Data is a DataHub utility or metadata-focused integration. Learn more in the [official Demo Data documentation](https://datahub.com/docs/). + +The DataHub integration for Demo Data covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `demo-data` + +### Important Capabilities +Capability metadata is not explicitly declared for this module. Refer to module documentation and configuration sections below. + +### Overview + +The `demo-data` module ingests metadata from Demo Data into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[demo-data]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: demo-data + config: {} + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "additionalProperties": false, + "properties": {}, + "title": "DemoDataConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.demo_data.DemoDataSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/demo_data.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Demo Data, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/doris.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/doris.md new file mode 100644 index 00000000..be8b82c1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/doris.md @@ -0,0 +1,1288 @@ +--- +sidebar_position: 2 +title: Apache Doris +slug: /generated/ingestion/sources/doris +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Apache Doris + +## Overview + +Doris is a data platform used to store and query analytical or operational data. Learn more in the [official Doris documentation](https://doris.apache.org/). + +The DataHub integration for Doris covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `doris` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_column_lineage`. Supported for types - View. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_lineage`. Supported for types - View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `doris` module ingests metadata from Doris into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +#### Profiling + +Doris-specific types (HLL, BITMAP, QUANTILE_STATE, ARRAY, JSONB) are automatically excluded from field-level profiling as they don't support standard aggregation operations. Table-level statistics are still collected for all tables. + +#### Stored Procedures + +Stored procedure ingestion is disabled by default because Doris's `information_schema.ROUTINES` table is always empty. + +### Prerequisites + +#### Doris Version + +Doris 3.0.x is required. Doris 2.0+ may work but is untested. + +#### Required Permissions + +Your Doris user requires specific privileges to extract metadata. + +```sql +-- Create user +CREATE USER 'datahub'@'%' IDENTIFIED BY 'your_password'; + +-- Grant required privileges +GRANT SELECT_PRIV ON *.* TO 'datahub'@'%'; +GRANT SHOW_VIEW_PRIV ON *.* TO 'datahub'@'%'; +``` + +- `SELECT_PRIV`: Required for table and column metadata +- `SHOW_VIEW_PRIV`: Required for view definitions and lineage + + +### Install the Plugin +```shell +pip install 'acryl-datahub[doris]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: doris + config: + # Connection (Doris FE query port is 9030) + host_port: localhost:9030 + database: your_database + username: root + password: "" + + # Optional: Schema/Table Filtering + # schema_pattern: + # allow: ["^production_.*"] + # deny: [".*_temp$"] + # table_pattern: + # allow: ["^fact_.*", "^dim_.*"] + # deny: [".*_backup$"] + + # Optional: Data Profiling + # Note: Doris types (HLL, BITMAP, QUANTILE_STATE, ARRAY, JSONB) are + # automatically excluded from field-level profiling + # profiling: + # enabled: true + # profile_table_level_only: false + + # Optional: Stateful Ingestion (tracks previously ingested entities) + # stateful_ingestion: + # enabled: true + # remove_stale_metadata: true + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
auth_mode
Enum
| One of: "PASSWORD", "AWS_IAM" | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
host_port
string
| Doris FE (Frontend) host and port. Default port is 9030.
Default: localhost:9030
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
aws_config
AwsConnectionConfig
| Common AWS credentials config.

Currently used by:
- Glue source
- SageMaker source
- dbt source | +|
aws_config.aws_access_key_id
One of string, null
| AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.aws_advanced_config
object
| Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html). | +|
aws_config.aws_endpoint_url
One of string, null
| The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.
Default: None
| +|
aws_config.aws_profile
One of string, null
| The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.
Default: None
| +|
aws_config.aws_proxy
One of string, null
| A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.
Default: None
| +|
aws_config.aws_region
One of string, null
| AWS region code.
Default: None
| +|
aws_config.aws_retry_mode
Enum
| One of: "legacy", "standard", "adaptive"
Default: standard
| +|
aws_config.aws_retry_num
integer
| Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.
Default: 5
| +|
aws_config.aws_secret_access_key
One of string(password), null
| AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.aws_session_token
One of string(password), null
| AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.read_timeout
number
| The timeout for reading from the connection (in seconds).
Default: 60
| +|
aws_config.aws_role
One of string, array, null
| AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).
Default: None
| +|
aws_config.aws_role.union
One of string, AwsAssumeRoleConfig
| | +|
aws_config.aws_role.union.RoleArn ❓
string
| ARN of the role to assume. | +|
aws_config.aws_role.union.ExternalId
One of string, null
| External ID to use when assuming the role.
Default: None
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AwsAssumeRoleConfig": { + "additionalProperties": true, + "properties": { + "RoleArn": { + "description": "ARN of the role to assume.", + "title": "Rolearn", + "type": "string" + }, + "ExternalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "External ID to use when assuming the role.", + "title": "Externalid" + } + }, + "required": [ + "RoleArn" + ], + "title": "AwsAssumeRoleConfig", + "type": "object" + }, + "AwsConnectionConfig": { + "additionalProperties": false, + "description": "Common AWS credentials config.\n\nCurrently used by:\n - Glue source\n - SageMaker source\n - dbt source", + "properties": { + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Session Token" + }, + "aws_role": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/AwsAssumeRoleConfig" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).", + "title": "Aws Role" + }, + "aws_profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.", + "title": "Aws Profile" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region code.", + "title": "Aws Region" + }, + "aws_endpoint_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.", + "title": "Aws Endpoint Url" + }, + "aws_proxy": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", + "title": "Aws Proxy" + }, + "aws_retry_num": { + "default": 5, + "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "title": "Aws Retry Num", + "type": "integer" + }, + "aws_retry_mode": { + "default": "standard", + "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "enum": [ + "legacy", + "standard", + "adaptive" + ], + "title": "Aws Retry Mode", + "type": "string" + }, + "read_timeout": { + "default": 60, + "description": "The timeout for reading from the connection (in seconds).", + "title": "Read Timeout", + "type": "number" + }, + "aws_advanced_config": { + "additionalProperties": true, + "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).", + "title": "Aws Advanced Config", + "type": "object" + } + }, + "title": "AwsConnectionConfig", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "MySQLAuthMode": { + "description": "Authentication mode for MySQL connection.", + "enum": [ + "PASSWORD", + "AWS_IAM" + ], + "title": "MySQLAuthMode", + "type": "string" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "description": "Configuration for profiling Doris tables. Note: Doris types (HLL, BITMAP, QUANTILE_STATE, ARRAY, JSONB) are automatically excluded from field-level profiling as they don't support COUNT DISTINCT." + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "default": "localhost:9030", + "description": "Doris FE (Frontend) host and port. Default port is 9030.", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for databases to filter in ingestion." + }, + "auth_mode": { + "$ref": "#/$defs/MySQLAuthMode", + "default": "PASSWORD", + "description": "Authentication mode to use for the MySQL connection. Options are 'PASSWORD' (default) for standard username/password authentication, or 'AWS_IAM' for AWS RDS IAM authentication." + }, + "aws_config": { + "$ref": "#/$defs/AwsConnectionConfig", + "description": "AWS configuration for RDS IAM authentication (only used when auth_mode is AWS_IAM). Provides full control over AWS credentials, region, profiles, role assumption, retry logic, and proxy settings. If not explicitly configured, boto3 will automatically use the default credential chain and region from environment variables (AWS_DEFAULT_REGION, AWS_REGION), AWS config files (~/.aws/config), or IAM role metadata." + } + }, + "title": "DorisConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Migration from MySQL Connector + +If you were previously ingesting Doris using the MySQL connector, switch to the dedicated Doris connector for better support: + +**Configuration changes:** + +- Change `type: mysql` → `type: doris` +- Change port: `3306` → `9030` + +**Important:** Dataset URNs will change from `platform:mysql` to `platform:doris`. This creates new entities in DataHub. Enable stateful ingestion with `remove_stale_metadata: true` to automatically clean up old MySQL-based entities. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.doris.doris_source.DorisSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/doris/doris_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Apache Doris, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dremio.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dremio.md new file mode 100644 index 00000000..b7e9de35 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dremio.md @@ -0,0 +1,925 @@ +--- +sidebar_position: 23 +title: Dremio +slug: /generated/ingestion/sources/dremio +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Dremio + +## Overview + +Dremio is a DataHub utility or metadata-focused integration. + +The DataHub integration for Dremio covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| -------------------------- | --------------- | ---------------------------------------------------------- | +| **Physical Dataset/Table** | `Dataset` | Subtype: `Table` | +| **Virtual Dataset/Views** | `Dataset` | Subtype: `View` | +| **Spaces** | `Container` | Mapped to DataHub’s `Container` aspect. Subtype: `Space` | +| **Folders** | `Container` | Mapped as a `Container` in DataHub. Subtype: `Folder` | +| **Sources** | `Container` | Represented as a `Container` in DataHub. Subtype: `Source` | + + +## Module `dremio` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Dremio Space, Dremio Source. | +| Column-level Lineage | ✅ | Extract column-level lineage. Supported for types - Table. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Dataset Usage | ✅ | Enabled by default to get usage stats. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| Extract Ownership | ✅ | Enabled by default. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default. Supported for types - Table. | + +### Overview + +The `dremio` module ingests metadata from Dremio into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin integrates with Dremio to extract and ingest metadata into DataHub. The following types of metadata are extracted: + +- Metadata for Spaces, Folders, Sources, and Datasets: + - Includes physical and virtual datasets, with detailed information about each dataset. + - Extracts metadata about Dremio's organizational hierarchy: Spaces (top-level), Folders (sub-level), and Sources (external data connections). + \*Schema and Column Information: + - Column types and schema metadata associated with each physical and virtual dataset. + - Extracts column-level metadata, such as names, data types, and descriptions, if available. +- Lineage Information: + - Dataset-level and column-level lineage tracking: + - Dataset-level lineage shows dependencies and relationships between physical and virtual datasets. + - Column-level lineage tracks transformations applied to individual columns across datasets. + - Lineage information helps trace the flow of data and transformations within Dremio. +- Ownership and Glossary Terms: + - Metadata related to ownership of datasets, extracted from Dremio’s ownership model. + - Glossary terms and business metadata associated with datasets, providing additional context to the data. + - Note: Ownership information will only be available for the Cloud and Enterprise editions, it will not be available for the Community edition. +- Optional SQL Profiling (if enabled): + - Table, row, and column statistics can be profiled and ingested via optional SQL queries. + - Extracts statistics about tables and columns, such as row counts and data distribution, for better insight into the dataset structure. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +1. **Generate an API Token**: + + - Log in to your Dremio instance. + - Navigate to your user profile in the top-right corner. + - Select **Generate API Token** to create an API token for programmatic access. + +2. **Permissions**: + + - The token should have **read-only** or **admin** permissions that allow it to: + - View all datasets (physical and virtual). + - Access all spaces, folders, and sources. + - Retrieve dataset and column-level lineage information. + +3. **Verify External Data Source Permissions**: + - If Dremio is connected to external data sources (e.g., AWS S3, relational databases), ensure that Dremio has access to the credentials required for querying those sources. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[dremio]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: dremio + config: + # Coordinates + hostname: localhost + port: 9047 + tls: true + + # Credentials with personal access token(recommended) + authentication_method: PAT + password: pass + # OR Credentials with basic auth + # authentication_method: password + # username: user + # password: pass + + #For cloud instance + #is_dremio_cloud: True + #dremio_cloud_project_id: + + include_query_lineage: True + + ingest_owner: true + + #Optional + source_mappings: + - platform: s3 + source_name: samples + + #Optional + schema_pattern: + allow: + - "." + +sink: + # sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
authentication_method
One of string, null
| Authentication method: 'password' or 'PAT' (Personal Access Token)
Default: PAT
| +|
bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
disable_certificate_verification
One of boolean, null
| Disable TLS certificate verification
Default: False
| +|
domain
One of string, null
| Domain for all source objects.
Default: None
| +|
dremio_cloud_project_id
One of string, null
| ID of Dremio Cloud Project. Found in Project Settings in the Dremio Cloud UI
Default: None
| +|
dremio_cloud_region
Enum
| One of: "US", "EU"
Default: US
| +|
end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
hostname
One of string, null
| Hostname or IP Address of the Dremio server
Default: None
| +|
include_query_lineage
boolean
| Whether to include query-based lineage information.
Default: False
| +|
ingest_owner
boolean
| Ingest Owner from source. This will override Owner info entered from UI
Default: True
| +|
is_dremio_cloud
boolean
| Whether this is a Dremio Cloud instance
Default: False
| +|
max_workers
integer
| Number of worker threads to use for parallel processing
Default: 50
| +|
password
One of string(password), null
| Dremio password or Personal Access Token
Default: None
| +|
path_to_certificates
string
| Path to SSL certificates
Default: /Users/dleifker/Code/datahub/metadata-ingestion/ve...
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
port
integer
| Port of the Dremio REST API
Default: 9047
| +|
start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
tls
boolean
| Whether the Dremio REST API port is encrypted
Default: True
| +|
username
One of string, null
| Dremio username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
dataset_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
dataset_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
source_mappings
One of array, null
| Mappings from Dremio sources to DataHub platforms and datasets.
Default: None
| +|
source_mappings.DremioSourceMapping
DremioSourceMapping
| | +|
source_mappings.DremioSourceMapping.platform ❓
string
| Source connection made by Dremio (e.g. S3, Snowflake) | +|
source_mappings.DremioSourceMapping.source_name ❓
string
| Alias of platform in Dremio connection | +|
source_mappings.DremioSourceMapping.platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
source_mappings.DremioSourceMapping.env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
usage
BaseUsageConfig
| | +|
usage.bucket_duration
Enum
| One of: "DAY", "HOUR" | +|
usage.end_time
string(date-time)
| Latest date of lineage/usage to consider. Default: Current time in UTC | +|
usage.format_sql_queries
boolean
| Whether to format sql queries
Default: False
| +|
usage.include_operational_stats
boolean
| Whether to display operational stats.
Default: True
| +|
usage.include_read_operational_stats
boolean
| Whether to report read operational stats. Experimental.
Default: False
| +|
usage.include_top_n_queries
boolean
| Whether to ingest the top_n_queries.
Default: True
| +|
usage.start_time
string(date-time)
| Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.
Default: None
| +|
usage.top_n_queries
integer
| Number of top queries to save to each table.
Default: 10
| +|
usage.user_email_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
usage.user_email_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
ProfileConfig
| | +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.query_timeout
integer
| Time before cancelling Dremio profiling query
Default: 300
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "BaseUsageConfig": { + "additionalProperties": false, + "properties": { + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "top_n_queries": { + "default": 10, + "description": "Number of top queries to save to each table.", + "exclusiveMinimum": 0, + "title": "Top N Queries", + "type": "integer" + }, + "user_email_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for user emails to filter in usage." + }, + "include_operational_stats": { + "default": true, + "description": "Whether to display operational stats.", + "title": "Include Operational Stats", + "type": "boolean" + }, + "include_read_operational_stats": { + "default": false, + "description": "Whether to report read operational stats. Experimental.", + "title": "Include Read Operational Stats", + "type": "boolean" + }, + "format_sql_queries": { + "default": false, + "description": "Whether to format sql queries", + "title": "Format Sql Queries", + "type": "boolean" + }, + "include_top_n_queries": { + "default": true, + "description": "Whether to ingest the top_n_queries.", + "title": "Include Top N Queries", + "type": "boolean" + } + }, + "title": "BaseUsageConfig", + "type": "object" + }, + "BucketDuration": { + "enum": [ + "DAY", + "HOUR" + ], + "title": "BucketDuration", + "type": "string" + }, + "DremioSourceMapping": { + "additionalProperties": false, + "properties": { + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform": { + "description": "Source connection made by Dremio (e.g. S3, Snowflake)", + "title": "Platform", + "type": "string" + }, + "source_name": { + "description": "Alias of platform in Dremio connection", + "title": "Source Name", + "type": "string" + } + }, + "required": [ + "platform", + "source_name" + ], + "title": "DremioSourceMapping", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "ProfileConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "query_timeout": { + "default": 300, + "description": "Time before cancelling Dremio profiling query", + "title": "Query Timeout", + "type": "integer" + } + }, + "title": "ProfileConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "bucket_duration": { + "$ref": "#/$defs/BucketDuration", + "default": "DAY", + "description": "Size of the time window to aggregate usage stats." + }, + "end_time": { + "description": "Latest date of lineage/usage to consider. Default: Current time in UTC", + "format": "date-time", + "title": "End Time", + "type": "string" + }, + "start_time": { + "default": null, + "description": "Earliest date of lineage/usage to consider. Default: Last full day in UTC (or hour, depending on `bucket_duration`). You can also specify relative time with respect to end_time such as '-7 days' Or '-7d'.", + "format": "date-time", + "title": "Start Time", + "type": "string" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "hostname": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Hostname or IP Address of the Dremio server", + "title": "Hostname" + }, + "port": { + "default": 9047, + "description": "Port of the Dremio REST API", + "title": "Port", + "type": "integer" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dremio username", + "title": "Username" + }, + "authentication_method": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "PAT", + "description": "Authentication method: 'password' or 'PAT' (Personal Access Token)", + "title": "Authentication Method" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dremio password or Personal Access Token", + "title": "Password" + }, + "tls": { + "default": true, + "description": "Whether the Dremio REST API port is encrypted", + "title": "Tls", + "type": "boolean" + }, + "disable_certificate_verification": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Disable TLS certificate verification", + "title": "Disable Certificate Verification" + }, + "path_to_certificates": { + "default": "/Users/dleifker/Code/datahub/metadata-ingestion/venv/lib/python3.11/site-packages/certifi/cacert.pem", + "description": "Path to SSL certificates", + "title": "Path To Certificates", + "type": "string" + }, + "is_dremio_cloud": { + "default": false, + "description": "Whether this is a Dremio Cloud instance", + "title": "Is Dremio Cloud", + "type": "boolean" + }, + "dremio_cloud_region": { + "default": "US", + "description": "Dremio Cloud region ('US' or 'EU')", + "enum": [ + "US", + "EU" + ], + "title": "Dremio Cloud Region", + "type": "string" + }, + "dremio_cloud_project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "ID of Dremio Cloud Project. Found in Project Settings in the Dremio Cloud UI", + "title": "Dremio Cloud Project Id" + }, + "domain": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Domain for all source objects.", + "title": "Domain" + }, + "source_mappings": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/DremioSourceMapping" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mappings from Dremio sources to DataHub platforms and datasets.", + "title": "Source Mappings" + }, + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schemas to filter" + }, + "dataset_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables and views to filter in ingestion. Specify regex to match the entire table name in dremio.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'dremio.public.customer.*'" + }, + "usage": { + "$ref": "#/$defs/BaseUsageConfig", + "default": { + "bucket_duration": "DAY", + "end_time": "2026-03-30T15:26:11.257478Z", + "start_time": "2026-03-29T00:00:00Z", + "queries_character_limit": 24000, + "top_n_queries": 10, + "user_email_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "include_operational_stats": true, + "include_read_operational_stats": false, + "format_sql_queries": false, + "include_top_n_queries": true + }, + "description": "The usage config to use when generating usage statistics" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to profile" + }, + "profiling": { + "$ref": "#/$defs/ProfileConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": false, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "query_timeout": 300 + }, + "description": "Configuration for profiling" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for parallel processing", + "title": "Max Workers", + "type": "integer" + }, + "include_query_lineage": { + "default": false, + "description": "Whether to include query-based lineage information.", + "title": "Include Query Lineage", + "type": "boolean" + }, + "ingest_owner": { + "default": true, + "description": "Ingest Owner from source. This will override Owner info entered from UI", + "title": "Ingest Owner", + "type": "boolean" + } + }, + "title": "DremioSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.dremio.dremio_source.DremioSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Dremio, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/druid.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/druid.md new file mode 100644 index 00000000..4e2b86de --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/druid.md @@ -0,0 +1,1064 @@ +--- +sidebar_position: 24 +title: Druid +slug: /generated/ingestion/sources/druid +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Druid + +## Overview + +Apache Druid is a data platform used to store and query analytical or operational data. Learn more in the [official Apache Druid documentation](https://druid.apache.org/). + +The DataHub integration for Apache Druid covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `druid` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_column_lineage`. Supported for types - View. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Enabled by default. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_lineage`. Supported for types - View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `druid` module ingests metadata from Druid into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin extracts the following: + +- Metadata for databases, schemas, and tables +- Column types associated with each table +- Table, row, and column statistics via optional SQL profiling. + +:::tip Schema pattern + +It is important to explicitly define the deny schema pattern for internal Druid databases (lookup & sys) if adding a schema pattern. Otherwise, the crawler may crash before processing relevant databases. This deny pattern is defined by default but is overriden by user-submitted configurations. +::: + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[druid]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: druid + config: + # Coordinates + host_port: "localhost:8082" + + # Credentials + username: admin + password: password + +sink: + # sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
host_port ✅
string
| host URL | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [ + "^(lookup|sysgit|view).*" + ], + "ignoreCase": true + }, + "description": "regex patterns for schemas to filter in ingestion." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "description": "host URL", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + } + }, + "required": [ + "host_port" + ], + "title": "DruidConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.druid.DruidSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/druid.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Druid, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dynamodb.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dynamodb.md new file mode 100644 index 00000000..67311945 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/dynamodb.md @@ -0,0 +1,729 @@ +--- +sidebar_position: 25 +title: DynamoDB +slug: /generated/ingestion/sources/dynamodb +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DynamoDB + +## Overview + +Dynamodb is a data platform used to store and query analytical or operational data. Learn more in the [official Dynamodb documentation](https://aws.amazon.com/dynamodb/). + +The DataHub integration for Dynamodb covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| -------------- | --------------------------------------------------------- | ----- | +| `"dynamodb"` | [Data Platform](../../metamodel/entities/dataPlatform.md) | | +| DynamoDB Table | [Dataset](../../metamodel/entities/dataset.md) | | + + +## Module `dynamodb` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Extract Tags | ✅ | Optionally enabled via `extract_table_tags` to extract dynamoDB table tags as DataHub tags. | +| [Platform Instance](../../../platform-instances.md) | ✅ | By default, platform_instance will use the AWS account id. | + +### Overview + +The `dynamodb` module ingests metadata from Dynamodb into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +:::note Breaking Change +Starting v0.13.3, `aws_region` is required. The connector no longer loops through all AWS regions and only uses the region specified in your recipe. +::: + +Attach the `AmazonDynamoDBReadOnlyAccess` policy to a user in your AWS account, then create an API access key and secret. The following privileges are required: + +``` +dynamodb:ListTables +dynamodb:DescribeTable +dynamodb:Scan +dynamodb:ListTagsOfResource +``` + +`dynamodb:Scan` is required because DynamoDB does not return schema information in `dynamodb:DescribeTable`. The connector samples a few values to infer the schema. + +`dynamodb:ListTagsOfResource` is required only when `extract_table_tags` is enabled to extract DynamoDB table tags. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[dynamodb]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: dynamodb + config: + aws_access_key_id: "${AWS_ACCESS_KEY_ID}" + aws_secret_access_key: "${AWS_SECRET_ACCESS_KEY}" + aws_region: "${AWS_REGION}" + + # Number of items to sample from each table for schema inference (default: 100) + # schema_sampling_size: 100 + + # If there are items that have most representative fields of the table, users could use the + # `include_table_item` option to provide a list of primary keys of the table in dynamodb format. + # For each `region.table`, the list of primary keys can be at most 100. + # We include these items in addition to the first 100 items in the table when we scan it. + # + # include_table_item: + # region.table_name: + # [ + # { + # "partition_key_name": { "attribute_type": "attribute_value" }, + # "sort_key_name": { "attribute_type": "attribute_value" }, + # }, + # ] + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
aws_access_key_id
One of string, null
| AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_advanced_config
object
| Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html). | +|
aws_endpoint_url
One of string, null
| The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.
Default: None
| +|
aws_profile
One of string, null
| The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.
Default: None
| +|
aws_proxy
One of string, null
| A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.
Default: None
| +|
aws_region
One of string, null
| AWS region code.
Default: None
| +|
aws_retry_mode
Enum
| One of: "legacy", "standard", "adaptive"
Default: standard
| +|
aws_retry_num
integer
| Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.
Default: 5
| +|
aws_secret_access_key
One of string(password), null
| AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_session_token
One of string(password), null
| AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
extract_table_tags
boolean
| When enabled, tags associated with DynamoDB tables in AWS will be emitted as DataHub tags. Tags are applied using OVERWRITE mode, meaning any existing tags including tags manually added or modified from the UI will be replaced. Use with caution.
Default: False
| +|
max_schema_size
integer
| Maximum number of fields to include in the schema.
Default: 300
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
read_timeout
number
| The timeout for reading from the connection (in seconds).
Default: 60
| +|
schema_sampling_size
integer
| Number of items to sample from each table for schema inference. This controls how many items are scanned to determine the table's schema structure.
Default: 100
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
aws_role
One of string, array, null
| AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).
Default: None
| +|
aws_role.union
One of string, AwsAssumeRoleConfig
| | +|
aws_role.union.RoleArn ❓
string
| ARN of the role to assume. | +|
aws_role.union.ExternalId
One of string, null
| External ID to use when assuming the role.
Default: None
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
include_table_item
One of array, null
| [Advanced] The primary keys of items of a table in dynamodb format the user would like to include in schema. Refer "Advanced Configurations" section for more details
Default: None
| +|
include_table_item.`key`.object
object
| | +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AwsAssumeRoleConfig": { + "additionalProperties": true, + "properties": { + "RoleArn": { + "description": "ARN of the role to assume.", + "title": "Rolearn", + "type": "string" + }, + "ExternalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "External ID to use when assuming the role.", + "title": "Externalid" + } + }, + "required": [ + "RoleArn" + ], + "title": "AwsAssumeRoleConfig", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Session Token" + }, + "aws_role": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/AwsAssumeRoleConfig" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).", + "title": "Aws Role" + }, + "aws_profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.", + "title": "Aws Profile" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region code.", + "title": "Aws Region" + }, + "aws_endpoint_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.", + "title": "Aws Endpoint Url" + }, + "aws_proxy": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", + "title": "Aws Proxy" + }, + "aws_retry_num": { + "default": 5, + "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "title": "Aws Retry Num", + "type": "integer" + }, + "aws_retry_mode": { + "default": "standard", + "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "enum": [ + "legacy", + "standard", + "adaptive" + ], + "title": "Aws Retry Mode", + "type": "string" + }, + "read_timeout": { + "default": 60, + "description": "The timeout for reading from the connection (in seconds).", + "title": "Read Timeout", + "type": "number" + }, + "aws_advanced_config": { + "additionalProperties": true, + "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).", + "title": "Aws Advanced Config", + "type": "object" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for databases to filter in ingestion." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. The table name format is 'region.table'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "regex patterns for tables to filter to assign domain_key. ", + "title": "Domain", + "type": "object" + }, + "include_table_item": { + "anyOf": [ + { + "additionalProperties": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[Advanced] The primary keys of items of a table in dynamodb format the user would like to include in schema. Refer \"Advanced Configurations\" section for more details", + "title": "Include Table Item" + }, + "max_schema_size": { + "default": 300, + "description": "Maximum number of fields to include in the schema.", + "exclusiveMinimum": 0, + "title": "Max Schema Size", + "type": "integer" + }, + "schema_sampling_size": { + "default": 100, + "description": "Number of items to sample from each table for schema inference. This controls how many items are scanned to determine the table's schema structure.", + "exclusiveMinimum": 0, + "title": "Schema Sampling Size", + "type": "integer" + }, + "extract_table_tags": { + "default": false, + "description": "When enabled, tags associated with DynamoDB tables in AWS will be emitted as DataHub tags. Tags are applied using OVERWRITE mode, meaning any existing tags including tags manually added or modified from the UI will be replaced. Use with caution.", + "title": "Extract Table Tags", + "type": "boolean" + } + }, + "title": "DynamoDBConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Using `schema_sampling_size` config + +By default, the connector samples 100 items from each table to infer the schema. You can adjust this using the `schema_sampling_size` configuration option if you need more comprehensive schema coverage: + +```yml +# Sample 500 items instead of default 100 +schema_sampling_size: 500 +``` + +#### Using `include_table_item` config + +If there are items that have most representative fields of the table, users could use the `include_table_item` option to provide a list of primary keys of the table in dynamodb format. We include these items in addition to the items sampled based on `schema_sampling_size` (default 100) when we scan the table. + +Take [AWS DynamoDB Developer Guide Example tables and data](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/AppendixSampleTables.html) as an example, if a account has a table `Reply` in the `us-west-2` region with composite primary key `Id` and `ReplyDateTime`, users can use `include_table_item` to include 2 items as following: + +Example: + +```yml +# The table name should be in the format of region.table_name +# The primary keys should be in the DynamoDB format +include_table_item: + us-west-2.Reply: + [ + { + "ReplyDateTime": { "S": "2015-09-22T19:58:22.947Z" }, + "Id": { "S": "Amazon DynamoDB#DynamoDB Thread 1" }, + }, + { + "ReplyDateTime": { "S": "2015-10-05T19:58:22.947Z" }, + "Id": { "S": "Amazon DynamoDB#DynamoDB Thread 2" }, + }, + ] +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.dynamodb.dynamodb.DynamoDBSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/dynamodb/dynamodb.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for DynamoDB, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/elasticsearch.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/elasticsearch.md new file mode 100644 index 00000000..a88f8087 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/elasticsearch.md @@ -0,0 +1,519 @@ +--- +sidebar_position: 26 +title: Elasticsearch +slug: /generated/ingestion/sources/elasticsearch +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Elasticsearch + +## Overview + +Elasticsearch is a DataHub utility or metadata-focused integration. + +The DataHub integration for Elasticsearch covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `elasticsearch` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | + +### Overview + +The `elasticsearch` module ingests metadata from Elasticsearch into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin extracts the following: + +- Metadata for indexes +- Column types associated with each index field + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[elasticsearch]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "elasticsearch" + config: + # Coordinates + host: 'localhost:9200' + + # Credentials + username: user # optional + password: pass # optional + + # SSL support + use_ssl: False + verify_certs: False + ca_certs: "./path/ca.cert" + client_cert: "./path/client.cert" + client_key: "./path/client.key" + ssl_assert_hostname: False + ssl_assert_fingerprint: "./path/cert.fingerprint" + + # Options + url_prefix: "" # optional url_prefix + env: "PROD" + index_pattern: + allow: [".*some_index_name_pattern*"] + deny: [".*skip_index_name_pattern*"] + ingest_index_templates: False + index_template_pattern: + allow: [".*some_index_template_name_pattern*"] + +sink: +# sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
api_key
One of object, string, null
| API Key authentication. Accepts either a list with id and api_key (UTF-8 representation), or a base64 encoded string of id and api_key combined by ':'.
Default: None
| +|
ca_certs
One of string, null
| Path to a certificate authority (CA) certificate.
Default: None
| +|
client_cert
One of string, null
| Path to the file containing the private key and the certificate, or cert only if using client_key.
Default: None
| +|
client_key
One of string, null
| Path to the file containing the private key if using separate cert and key files.
Default: None
| +|
host
string
| The elastic search host URI.
Default: localhost:9200
| +|
ingest_index_templates
boolean
| Ingests ES index templates if enabled.
Default: False
| +|
password
One of string(password), null
| The password credential.
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
ssl_assert_fingerprint
One of string, null
| Verify the supplied certificate fingerprint if not None.
Default: None
| +|
ssl_assert_hostname
boolean
| Use hostname verification if not False.
Default: False
| +|
url_prefix
string
| There are cases where an enterprise would have multiple elastic search clusters. One way for them to manage is to have a single endpoint for all the elastic search clusters and use url_prefix for routing requests to different clusters.
Default:
| +|
use_ssl
boolean
| Whether to use SSL for the connection or not.
Default: False
| +|
username
One of string, null
| The username credential.
Default: None
| +|
verify_certs
boolean
| Whether to verify SSL certificates.
Default: False
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
collapse_urns
CollapseUrns
| | +|
collapse_urns.urns_suffix_regex
array
| List of regex patterns to remove from the name of the URN. All of the indices before removal of URNs are considered as the same dataset. These are applied in order for each URN.
The main case where you would want to have multiple of these if the name where you are trying to remove suffix from have different formats.
e.g. ending with -YYYY-MM-DD as well as ending -epochtime would require you to have 2 regex patterns to remove the suffixes across all URNs. | +|
collapse_urns.urns_suffix_regex.string
string
| | +|
index_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
index_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
index_template_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
index_template_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
ElasticProfiling
| | +|
profiling.enabled
boolean
| Whether to enable profiling for the elastic search source.
Default: False
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
stateful_ingestion
One of StatefulIngestionConfig, null
| Stateful Ingestion Config
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "CollapseUrns": { + "additionalProperties": false, + "properties": { + "urns_suffix_regex": { + "description": "List of regex patterns to remove from the name of the URN. All of the indices before removal of URNs are considered as the same dataset. These are applied in order for each URN.\n The main case where you would want to have multiple of these if the name where you are trying to remove suffix from have different formats.\n e.g. ending with -YYYY-MM-DD as well as ending -epochtime would require you to have 2 regex patterns to remove the suffixes across all URNs.", + "items": { + "type": "string" + }, + "title": "Urns Suffix Regex", + "type": "array" + } + }, + "title": "CollapseUrns", + "type": "object" + }, + "ElasticProfiling": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether to enable profiling for the elastic search source.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + } + }, + "title": "ElasticProfiling", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulIngestionConfig": { + "additionalProperties": false, + "description": "Basic Stateful Ingestion Specific Configuration for any source.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + } + }, + "title": "StatefulIngestionConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulIngestionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Stateful Ingestion Config" + }, + "host": { + "default": "localhost:9200", + "description": "The elastic search host URI.", + "title": "Host", + "type": "string" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The username credential.", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "The password credential.", + "title": "Password" + }, + "api_key": { + "anyOf": [ + {}, + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "API Key authentication. Accepts either a list with id and api_key (UTF-8 representation), or a base64 encoded string of id and api_key combined by ':'.", + "title": "Api Key" + }, + "use_ssl": { + "default": false, + "description": "Whether to use SSL for the connection or not.", + "title": "Use Ssl", + "type": "boolean" + }, + "verify_certs": { + "default": false, + "description": "Whether to verify SSL certificates.", + "title": "Verify Certs", + "type": "boolean" + }, + "ca_certs": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to a certificate authority (CA) certificate.", + "title": "Ca Certs" + }, + "client_cert": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to the file containing the private key and the certificate, or cert only if using client_key.", + "title": "Client Cert" + }, + "client_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to the file containing the private key if using separate cert and key files.", + "title": "Client Key" + }, + "ssl_assert_hostname": { + "default": false, + "description": "Use hostname verification if not False.", + "title": "Ssl Assert Hostname", + "type": "boolean" + }, + "ssl_assert_fingerprint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Verify the supplied certificate fingerprint if not None.", + "title": "Ssl Assert Fingerprint" + }, + "url_prefix": { + "default": "", + "description": "There are cases where an enterprise would have multiple elastic search clusters. One way for them to manage is to have a single endpoint for all the elastic search clusters and use url_prefix for routing requests to different clusters.", + "title": "Url Prefix", + "type": "string" + }, + "index_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [ + "^_.*", + "^ilm-history.*" + ], + "ignoreCase": true + }, + "description": "regex patterns for indexes to filter in ingestion." + }, + "ingest_index_templates": { + "default": false, + "description": "Ingests ES index templates if enabled.", + "title": "Ingest Index Templates", + "type": "boolean" + }, + "index_template_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [ + "^_.*" + ], + "ignoreCase": true + }, + "description": "The regex patterns for filtering index templates to ingest." + }, + "profiling": { + "$ref": "#/$defs/ElasticProfiling", + "description": "Configs to ingest data profiles from ElasticSearch." + }, + "collapse_urns": { + "$ref": "#/$defs/CollapseUrns", + "description": "List of regex patterns to remove from the name of the URN. All of the indices before removal of URNs are considered as the same dataset. These are applied in order for each URN.\n The main case where you would want to have multiple of these if the name where you are trying to remove suffix from have different formats.\n e.g. ending with -YYYY-MM-DD as well as ending -epochtime would require you to have 2 regex patterns to remove the suffixes across all URNs." + } + }, + "title": "ElasticsearchSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.elastic_search.ElasticsearchSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/elastic_search.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Elasticsearch, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/excel.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/excel.md new file mode 100644 index 00000000..d922c7fa --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/excel.md @@ -0,0 +1,1286 @@ +--- +sidebar_position: 27 +title: Excel +slug: /generated/ingestion/sources/excel +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Excel + +## Overview + +Excel is a storage and lakehouse platform. Learn more in the [official Excel documentation](https://www.microsoft.com/microsoft-365/excel). + +The DataHub integration for Excel covers file/lakehouse metadata entities such as datasets, paths, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Excel Entity | DataHub Entity | Description | +| ---------------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| **Excel Worksheet** | **Dataset** | Each worksheet becomes a dataset with URN pattern: `urn:li:dataset:(urn:li:dataPlatform:excel,{path}/[{filename}]{sheet_name},PROD)` | +| **File/Directory Structure** | **Container** | Directory hierarchy creates containers with obfuscated URNs for organizing datasets | + +:::info Excel workbook + +The Excel workbook file itself does not become a separate DataHub entity - only the individual worksheets within it are ingested as datasets. +::: + + +## Module `excel` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Optionally enabled via `stateful_ingestion.remove_stale_metadata`. | +| Schema Metadata | ✅ | Enabled by default. | + +### Overview + +The `excel` module ingests metadata from Excel into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +#### Supported file types + +Supported file types are as follows: + +- Excel workbook (`*.xlsx`) +- Excel macro-enabled workbook (`*.xlsm`) + +The connector will attempt to identify which cells contain table data. A table is defined as a header row, which is used to derive the column names, followed by data rows. The schema is inferred from the data types that are present in a column. + +Rows that are directly above or directly below the table where only the first two columns have values are assumed to contain metadata. If such rows are located, they are converted to custom properties where the first column is the key, and the second column is the value. Additionally, the workbook standard and custom properties are also imported as dataset custom properties. + +### Prerequisites + +#### AWS S3 + +When configuring an S3 ingestion source to access files in an S3 bucket, the AWS account referenced in your ingestion recipe must have appropriate S3 permissions. Create a policy with the minimum required permissions by following these steps: + +1. **Create an IAM Policy**: Create a policy that grants read access to specific S3 buckets. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation", "s3:GetObject"], + "Resource": [ + "arn:aws:s3:::your-bucket-name", + "arn:aws:s3:::your-bucket-name/*" + ] + } + ] +} +``` + +**Permissions Explanation**: + +- `s3:ListBucket`: Allows listing the objects in the bucket. This permission is necessary for the S3 ingestion source to know which objects are available to read. +- `s3:GetBucketLocation`: Allows retrieving the location of the bucket. +- `s3:GetObject`: Allows reading the actual content of the objects in the bucket. This is needed to infer schema from sample files. + +2. **Link Policy to Identity**: Associate your newly created policy with the appropriate IAM user or role that will be used by the S3 ingestion process. + +3. **Set Up S3 Data Source**: When configuring your S3 ingestion source, specify the IAM user to whom you assigned the permissions in the previous step. + +#### Azure Blob Storage + +To access files on Azure Blob Storage, you will need the following: + +1. **Azure Storage Account**: A storage account that provides a unique namespace for your data in Azure. + +2. **Authentication Credentials**: One of these supported authentication methods: + + - **Account Key**: Use your storage account's access key + - **Client Secret**: Use a service principal with client ID and client secret for Microsoft Entra ID authentication + - **SAS Token**: Use a Shared Access Signature token that provides limited, time-bound access + +3. **Container**: A blob container that organizes your blobs (similar to a directory in a file system). + +4. **Access Permissions**: Appropriate authorization for the authentication method: + - For account key: Full access to the storage account + - For client secret: Appropriate Azure role assignments (like Storage Blob Data Contributor) + - For SAS token: Permissions are defined within the token itself + + +### Install the Plugin +```shell +pip install 'acryl-datahub[excel]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: excel + config: + path_list: + - "/data/path/reporting/excel/*.xlsx" + profiling: + enabled: false + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
path_list ✅
array
| List of paths to Excel files or folders to ingest. | +|
path_list.string
string
| | +|
active_sheet_only
boolean
| Enable to only ingest the active sheet of the workbook. If not set, all sheets will be ingested.
Default: False
| +|
convert_urns_to_lowercase
boolean
| Enable to convert the Excel asset urns to lowercase
Default: False
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
use_abs_blob_tags
One of boolean, null
| Whether to create tags in datahub from the abs blob tags
Default: False
| +|
use_s3_bucket_tags
One of boolean, null
| Whether or not to create tags in datahub from the s3 bucket
Default: False
| +|
use_s3_object_tags
One of boolean, null
| Whether or not to create tags in datahub from the s3 object
Default: False
| +|
verify_ssl
One of boolean, string
| Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use.
Default: True
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
aws_config
One of AwsConnectionConfig, null
| AWS configuration
Default: None
| +|
aws_config.aws_access_key_id
One of string, null
| AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.aws_advanced_config
object
| Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html). | +|
aws_config.aws_endpoint_url
One of string, null
| The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.
Default: None
| +|
aws_config.aws_profile
One of string, null
| The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.
Default: None
| +|
aws_config.aws_proxy
One of string, null
| A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.
Default: None
| +|
aws_config.aws_region
One of string, null
| AWS region code.
Default: None
| +|
aws_config.aws_retry_mode
Enum
| One of: "legacy", "standard", "adaptive"
Default: standard
| +|
aws_config.aws_retry_num
integer
| Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.
Default: 5
| +|
aws_config.aws_secret_access_key
One of string(password), null
| AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.aws_session_token
One of string(password), null
| AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_config.read_timeout
number
| The timeout for reading from the connection (in seconds).
Default: 60
| +|
aws_config.aws_role
One of string, array, null
| AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).
Default: None
| +|
aws_config.aws_role.union
One of string, AwsAssumeRoleConfig
| | +|
aws_config.aws_role.union.RoleArn ❓
string
| ARN of the role to assume. | +|
aws_config.aws_role.union.ExternalId
One of string, null
| External ID to use when assuming the role.
Default: None
| +|
azure_config
One of AzureConnectionConfig, null
| Azure configuration
Default: None
| +|
azure_config.account_name ❓
string
| Name of the Azure storage account. See [Microsoft official documentation on how to create a storage account.](https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account) | +|
azure_config.container_name ❓
string
| Azure storage account container name. | +|
azure_config.account_key
One of string(password), null
| Azure storage account access key that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**
Default: None
| +|
azure_config.base_path
string
| Base folder in hierarchical namespaces to start from.
Default: /
| +|
azure_config.client_id
One of string, null
| Azure client (Application) ID required when a `client_secret` is used as a credential.
Default: None
| +|
azure_config.client_secret
One of string(password), null
| Azure client secret that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**
Default: None
| +|
azure_config.sas_token
One of string(password), null
| Azure storage account Shared Access Signature (SAS) token that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**
Default: None
| +|
azure_config.tenant_id
One of string, null
| Azure tenant (Directory) ID required when a `client_secret` is used as a credential.
Default: None
| +|
path_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
path_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
worksheet_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
worksheet_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Configuration for stateful ingestion and stale metadata removal.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AwsAssumeRoleConfig": { + "additionalProperties": true, + "properties": { + "RoleArn": { + "description": "ARN of the role to assume.", + "title": "Rolearn", + "type": "string" + }, + "ExternalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "External ID to use when assuming the role.", + "title": "Externalid" + } + }, + "required": [ + "RoleArn" + ], + "title": "AwsAssumeRoleConfig", + "type": "object" + }, + "AwsConnectionConfig": { + "additionalProperties": false, + "description": "Common AWS credentials config.\n\nCurrently used by:\n - Glue source\n - SageMaker source\n - dbt source", + "properties": { + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Session Token" + }, + "aws_role": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/AwsAssumeRoleConfig" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).", + "title": "Aws Role" + }, + "aws_profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.", + "title": "Aws Profile" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region code.", + "title": "Aws Region" + }, + "aws_endpoint_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.", + "title": "Aws Endpoint Url" + }, + "aws_proxy": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", + "title": "Aws Proxy" + }, + "aws_retry_num": { + "default": 5, + "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "title": "Aws Retry Num", + "type": "integer" + }, + "aws_retry_mode": { + "default": "standard", + "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "enum": [ + "legacy", + "standard", + "adaptive" + ], + "title": "Aws Retry Mode", + "type": "string" + }, + "read_timeout": { + "default": 60, + "description": "The timeout for reading from the connection (in seconds).", + "title": "Read Timeout", + "type": "number" + }, + "aws_advanced_config": { + "additionalProperties": true, + "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).", + "title": "Aws Advanced Config", + "type": "object" + } + }, + "title": "AwsConnectionConfig", + "type": "object" + }, + "AzureConnectionConfig": { + "additionalProperties": false, + "description": "Common Azure credentials config.\n\nhttps://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-directory-file-acl-python", + "properties": { + "base_path": { + "default": "/", + "description": "Base folder in hierarchical namespaces to start from.", + "title": "Base Path", + "type": "string" + }, + "container_name": { + "description": "Azure storage account container name.", + "title": "Container Name", + "type": "string" + }, + "account_name": { + "description": "Name of the Azure storage account. See [Microsoft official documentation on how to create a storage account.](https://docs.microsoft.com/en-us/azure/storage/blobs/create-data-lake-storage-account)", + "title": "Account Name", + "type": "string" + }, + "account_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure storage account access key that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", + "title": "Account Key" + }, + "sas_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure storage account Shared Access Signature (SAS) token that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", + "title": "Sas Token" + }, + "client_secret": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure client secret that can be used as a credential. **An account key, a SAS token or a client secret is required for authentication.**", + "title": "Client Secret" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure client (Application) ID required when a `client_secret` is used as a credential.", + "title": "Client Id" + }, + "tenant_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure tenant (Directory) ID required when a `client_secret` is used as a credential.", + "title": "Tenant Id" + } + }, + "required": [ + "container_name", + "account_name" + ], + "title": "AzureConnectionConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for stateful ingestion and stale metadata removal." + }, + "path_list": { + "description": "List of paths to Excel files or folders to ingest.", + "items": { + "type": "string" + }, + "title": "Path List", + "type": "array" + }, + "path_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for file paths to filter in ingestion." + }, + "aws_config": { + "anyOf": [ + { + "$ref": "#/$defs/AwsConnectionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS configuration" + }, + "use_s3_bucket_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether or not to create tags in datahub from the s3 bucket", + "title": "Use S3 Bucket Tags" + }, + "use_s3_object_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether or not to create tags in datahub from the s3 object", + "title": "Use S3 Object Tags" + }, + "verify_ssl": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "string" + } + ], + "default": true, + "description": "Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use.", + "title": "Verify Ssl" + }, + "azure_config": { + "anyOf": [ + { + "$ref": "#/$defs/AzureConnectionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure configuration" + }, + "use_abs_blob_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "Whether to create tags in datahub from the abs blob tags", + "title": "Use Abs Blob Tags" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Enable to convert the Excel asset urns to lowercase", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "active_sheet_only": { + "default": false, + "description": "Enable to only ingest the active sheet of the workbook. If not set, all sheets will be ingested.", + "title": "Active Sheet Only", + "type": "boolean" + }, + "worksheet_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for worksheets to ingest. Worksheets are specified as 'filename_without_extension.worksheet_name'. For example to allow the worksheet Sheet1 from file report.xlsx, use the pattern: 'report.Sheet1'." + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for worksheets to profile. Worksheets are specified as 'filename_without_extension.worksheet_name'. For example to allow the worksheet Sheet1 from file report.xlsx, use the pattern: 'report.Sheet1'." + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + }, + "description": "Configuration for profiling" + } + }, + "required": [ + "path_list" + ], + "title": "ExcelSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +This connector ingests Excel worksheet datasets into DataHub. Workbooks (Excel files) can be ingested from the local filesystem, from S3 buckets, or from Azure Blob Storage. An asterisk (`*`) can be used in place of a directory or as part of a file name to match multiple directories or files with a single path specification. + +:::tip +By default, this connector will ingest all worksheets in a workbook (an Excel file). To filter worksheets use the `worksheet_pattern` config option, or to only ingest the active worksheet use the `active_sheet_only` config option. +::: + +#### Starter Recipes + +Check out the following recipes to get started with ingestion. + +For general pointers on writing and running a recipe, see our [main recipe guide](/docs/metadata-ingestion#recipes). + +#### S3 + +```yaml +source: + type: excel + config: + path_list: + - "s3://bucket/data/excel/*/*.xlsx" + aws_config: + aws_access_key_id: AKIAIOSFODNN7EXAMPLE + aws_secret_access_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + aws_region: us-east-1 + profiling: + enabled: false +``` + +#### Azure Blob Storage + +```yaml +source: + type: excel + config: + path_list: + - "https://storageaccountname.blob.core.windows.net/abs-data/excel/*/*.xlsx" + azure_config: + account_name: storageaccountname + sas_token: sv=2022-11-02&ss=b&srt=sco&sp=rwdlacx&se=2025-06-07T21:00:00Z&st=2025-05-07T13:00:00Z&spr=https&sig=a1B2c3D4%3D + container_name: abs-data + profiling: + enabled: false +``` + +#### Local Files + +```yaml +source: + type: excel + config: + path_list: + - "/data/path/reporting/excel/*.xlsx" + profiling: + enabled: false +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.excel.source.ExcelSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/excel/source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Excel, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fabric-onelake.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fabric-onelake.md new file mode 100644 index 00000000..7701cfb9 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fabric-onelake.md @@ -0,0 +1,1071 @@ +--- +sidebar_position: 28 +title: Fabric OneLake +slug: /generated/ingestion/sources/fabric-onelake +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Fabric OneLake + +## Overview + +Microsoft Fabric OneLake is a storage and lakehouse platform. Learn more in the [official Microsoft Fabric OneLake documentation](https://learn.microsoft.com/fabric/onelake/onelake-overview). + +The DataHub integration for Microsoft Fabric OneLake covers file/lakehouse metadata entities such as datasets, paths, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Microsoft Fabric | DataHub Entity | Notes | +| ---------------- | ----------------------------------------- | ------------------------------------------- | +| **Workspace** | `Container` (subtype: `Fabric Workspace`) | Top-level organizational unit | +| **Lakehouse** | `Container` (subtype: `Fabric Lakehouse`) | Contains schemas and tables | +| **Warehouse** | `Container` (subtype: `Fabric Warehouse`) | Contains schemas and tables | +| **Schema** | `Container` (subtype: `Fabric Schema`) | Logical grouping within lakehouse/warehouse | +| **Table** | `Dataset` | Tables within schemas | + +### Hierarchy Structure + +``` +Platform (fabric-onelake) +└── Workspace (Container) + ├── Lakehouse (Container) + │ └── Schema (Container) + │ └── Table (Dataset) + └── Warehouse (Container) + └── Schema (Container) + └── Table/View (Dataset) +``` + +### Platform Instance as Tenant + +The Fabric REST API does not expose tenant-level endpoints. To represent tenant-level organization in DataHub, set the `platform_instance` configuration field to your tenant identifier (e.g., "contoso-tenant"). This will be included in all container and dataset URNs, effectively grouping all workspaces under the specified platform instance/tenant. + + +## Module `fabric-onelake` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | + +### Overview + +The `fabric-onelake` module ingests metadata from Fabric Onelake into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +:::tip Quick Start + +1. **Set up authentication** - Configure Azure credentials (see [Prerequisites](#prerequisites)) +2. **Grant permissions** - Ensure your identity has `Workspace.Read.All` and workspace access +3. **Configure recipe** - Use `fabric-onelake_recipe.yml` as a template +4. **Run ingestion** - Execute `datahub ingest -c fabric-onelake_recipe.yml` + ::: + +#### Key Features + +- Workspace, Lakehouse, Warehouse, and Schema containers +- Table datasets with proper subtypes +- Automatic detection and handling of schemas-enabled and schemas-disabled lakehouses +- Pattern-based filtering for workspaces, lakehouses, warehouses, and tables +- Stateful ingestion for stale entity removal +- Multiple authentication methods (Service Principal, Managed Identity, Azure CLI, DefaultAzureCredential) + +#### References + +Azure Authentication + +- [Register an application with Microsoft Entra ID](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app) +- [Azure Identity Library](https://learn.microsoft.com/en-us/python/api/overview/azure/identity-readme) +- [Service Principal Authentication](https://learn.microsoft.com/en-us/entra/identity-platform/app-objects-and-service-principals) +- [Managed Identities](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) + +Fabric Concepts + +- [Microsoft Fabric Overview](https://learn.microsoft.com/en-us/fabric/get-started/microsoft-fabric-overview) +- [OneLake Overview](https://learn.microsoft.com/en-us/fabric/onelake/onelake-overview) +- [Workspaces in Fabric](https://learn.microsoft.com/en-us/fabric/get-started/workspaces) +- [Lakehouses in Fabric](https://learn.microsoft.com/en-us/fabric/data-engineering/lakehouse-overview) +- [Warehouses in Fabric](https://learn.microsoft.com/en-us/fabric/data-warehouse/data-warehousing) + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +#### Authentication + +The connector supports multiple Azure authentication methods: + +| Method | Best For | Configuration | +| -------------------------- | ------------------------------------------------ | --------------------------------------------------- | +| **Service Principal** | Production environments | `authentication_method: service_principal` | +| **Managed Identity** | Azure-hosted deployments (VMs, AKS, App Service) | `authentication_method: managed_identity` | +| **Azure CLI** | Local development | `authentication_method: cli` (run `az login` first) | +| **DefaultAzureCredential** | Flexible environments | `authentication_method: default` | + +For service principal setup, see [Register an application with Microsoft Entra ID](https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app). + +#### Required Permissions + +The connector requires **read-only access** to Fabric workspaces and their contents. The authenticated identity (service principal, managed identity, or user) must have: + +**Workspace-Level Permissions:** + +- **Workspace.Read.All** or **Workspace.ReadWrite.All** (Microsoft Entra delegated scope) +- **Viewer** role or higher in the Fabric workspaces you want to ingest + +**API Permissions:** +The service principal or user must have the following Microsoft Entra API permissions: + +- `Workspace.Read.All` (delegated) - Required to list and read workspace metadata +- Or `Workspace.ReadWrite.All` (delegated) - Provides read and write access + +**Token Audiences:** +The connector uses two different token audiences depending on the operation: + +- **Fabric REST API** (`https://api.fabric.microsoft.com`): Uses Power BI API scope (`https://analysis.windows.net/powerbi/api/.default`) for listing workspaces, lakehouses, warehouses, and basic table metadata +- **OneLake Delta Table APIs** (`https://onelake.table.fabric.microsoft.com`): Uses Storage audience (`https://storage.azure.com/.default`) for accessing schemas and tables in **schemas-enabled lakehouses** + +The connector automatically handles both token audiences. For schemas-enabled lakehouses, it will use OneLake Delta Table APIs with Storage audience tokens. For schemas-disabled lakehouses, it uses the standard Fabric REST API. + +**OneLake Data Access Permissions:** +For schemas-enabled lakehouses, you may also need OneLake data access permissions: + +- If OneLake security is enabled on your lakehouse, ensure your identity has **Read** or **ReadWrite** permissions on the lakehouse item +- These permissions are separate from workspace roles and are managed in the Fabric portal under the lakehouse's security settings + +**Note:** The connector automatically detects whether a lakehouse has schemas enabled and uses the appropriate API endpoint and token audience. No additional configuration is required. + +For detailed information on permissions, see: + +- [Fabric REST API Permissions](https://learn.microsoft.com/en-us/rest/api/fabric/articles/scopes) +- [Workspace Roles and Permissions](https://learn.microsoft.com/en-us/fabric/admin/roles) +- [OneLake Data Access Control](https://learn.microsoft.com/en-us/fabric/onelake/security/data-access-control-model) + +#### Granting Permissions + +**For Service Principal:** + +1. Register an application in Microsoft Entra ID (Azure AD) +2. Grant API permissions: + - Navigate to **Azure Portal** > **App registrations** > Your app > **API permissions** + - Add permission: **Power BI Service** > **Delegated permissions** > `Workspace.Read.All` + - Click **Grant admin consent** (if you have admin rights) +3. Assign workspace roles: + - In Fabric portal, navigate to each workspace + - Go to **Workspace settings** > **Access** + - Add your service principal and assign **Viewer** role or higher + +**For Managed Identity:** + +1. Enable system-assigned managed identity on your Azure resource (VM, AKS, App Service, etc.) +2. Assign the managed identity to Fabric workspaces with **Viewer** role or higher +3. The connector will automatically use the managed identity for authentication + + +### Install the Plugin +```shell +pip install 'acryl-datahub[fabric-onelake]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +# Example recipe for Microsoft Fabric OneLake source + +source: + type: fabric-onelake + config: + # Authentication (using service principal) + credential: + authentication_method: service_principal + client_id: ${AZURE_CLIENT_ID} + client_secret: ${AZURE_CLIENT_SECRET} + tenant_id: ${AZURE_TENANT_ID} + + # Optional: Platform instance (use as tenant identifier) + # This groups all workspaces under a tenant-level container + # platform_instance: "contoso-tenant" + + # Optional: Environment + # env: PROD + + # Optional: Filter workspaces by name pattern + # workspace_pattern: + # allow: + # - ".*" # Allow all workspaces by default + # deny: [] + + # Optional: Filter lakehouses by name pattern + # lakehouse_pattern: + # allow: + # - ".*" # Allow all lakehouses by default + # deny: [] + + # Optional: Filter warehouses by name pattern + # warehouse_pattern: + # allow: + # - ".*" # Allow all warehouses by default + # deny: [] + + # Optional: Filter tables by name pattern + # Format: 'schema.table' or just 'table' for default schema + # table_pattern: + # allow: + # - ".*" # Allow all tables by default + # deny: [] + + # Feature flags + extract_lakehouses: true + extract_warehouses: true + extract_schemas: true # Set to false to skip schema containers + + # Optional: API timeout (seconds) + # api_timeout: 30 + + # Optional: Stateful ingestion for stale entity removal + # stateful_ingestion: + # enabled: true + # remove_stale_metadata: true + + # Optional: Schema extraction configuration + # extract_schema: + # enabled: true # Enable schema extraction (default: true) + # method: sql_analytics_endpoint + + # Optional: SQL Analytics Endpoint configuration + # sql_endpoint: + # enabled: true # Enable SQL endpoint connection (default: true) + # odbc_driver: "ODBC Driver 18 for SQL Server" # ODBC driver name (default: "ODBC Driver 18 for SQL Server") + # encrypt: "yes" # Enable encryption (default: "yes") + # trust_server_certificate: "no" # Trust server certificate (default: "no") + # query_timeout: 30 # Timeout for SQL queries in seconds (default: 30) + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
api_timeout
integer
| Timeout for REST API calls in seconds.
Default: 30
| +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
extract_lakehouses
boolean
| Whether to extract lakehouses and their tables.
Default: True
| +|
extract_schemas
boolean
| Whether to extract schema containers. If False, tables will be directly under lakehouse/warehouse containers.
Default: True
| +|
extract_warehouses
boolean
| Whether to extract warehouses and their tables.
Default: True
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
credential
AzureCredentialConfig
| Unified Azure authentication configuration.

This class provides a reusable authentication configuration that can be
composed into any Azure connector's configuration. It supports multiple
authentication methods and returns a TokenCredential that works with
any Azure SDK client.

Example usage in a connector config:
class MyAzureConnectorConfig(ConfigModel):
credential: AzureCredentialConfig = Field(
default_factory=AzureCredentialConfig,
description="Azure authentication configuration"
)
subscription_id: str = Field(...) | +|
credential.authentication_method
Enum
| One of: "default", "service_principal", "managed_identity", "cli" | +|
credential.client_id
One of string, null
| Azure Application (client) ID. Required for service_principal authentication. Find this in Azure Portal > App registrations > Your app > Overview.
Default: None
| +|
credential.client_secret
One of string(password), null
| Azure client secret. Required for service_principal authentication. Create in Azure Portal > App registrations > Your app > Certificates & secrets.
Default: None
| +|
credential.exclude_cli_credential
boolean
| When using 'default' authentication, exclude Azure CLI credential. Useful in production to avoid accidentally using developer credentials.
Default: False
| +|
credential.exclude_environment_credential
boolean
| When using 'default' authentication, exclude environment variables. Environment variables checked: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID.
Default: False
| +|
credential.exclude_managed_identity_credential
boolean
| When using 'default' authentication, exclude managed identity. Useful during local development when managed identity is not available.
Default: False
| +|
credential.managed_identity_client_id
One of string, null
| Client ID for user-assigned managed identity. Leave empty to use system-assigned managed identity. Only used when authentication_method is 'managed_identity'.
Default: None
| +|
credential.tenant_id
One of string, null
| Azure tenant (directory) ID. Required for service_principal authentication. Find this in Azure Portal > Microsoft Entra ID > Overview.
Default: None
| +|
extract_schema
ExtractSchemaConfig
| Configuration for schema extraction. | +|
extract_schema.enabled
boolean
| Enable schema extraction
Default: True
| +|
extract_schema.method
string
| Schema extraction method. Currently only 'sql_analytics_endpoint' is supported.
Default: sql_analytics_endpoint
| +|
lakehouse_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
lakehouse_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
sql_endpoint
One of SqlEndpointConfig, null
| SQL Analytics Endpoint configuration for schema extraction. Required when extract_schema.enabled=True and extract_schema.method='sql_analytics_endpoint'. | +|
sql_endpoint.enabled
boolean
| Enable SQL Analytics Endpoint connection
Default: True
| +|
sql_endpoint.encrypt
Enum
| One of: "yes", "no", "mandatory", "optional", "strict"
Default: yes
| +|
sql_endpoint.odbc_driver
string
| ODBC driver name for SQL Server connections.
Default: ODBC Driver 18 for SQL Server
| +|
sql_endpoint.query_timeout
integer
| Timeout for SQL queries in seconds
Default: 30
| +|
sql_endpoint.trust_server_certificate
Enum
| One of: "yes", "no"
Default: no
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
warehouse_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
warehouse_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
workspace_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
workspace_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Configuration for stateful ingestion and stale entity removal. When enabled, tracks ingested entities and removes those that no longer exist in Fabric.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AzureAuthenticationMethod": { + "description": "Supported Azure authentication methods.\n\n- DEFAULT: Uses DefaultAzureCredential which auto-detects credentials from\n environment variables, managed identity, Azure CLI, etc.\n- SERVICE_PRINCIPAL: Uses client ID, client secret, and tenant ID\n- MANAGED_IDENTITY: Uses Azure Managed Identity (system or user-assigned)\n- CLI: Uses Azure CLI credential (requires `az login`)", + "enum": [ + "default", + "service_principal", + "managed_identity", + "cli" + ], + "title": "AzureAuthenticationMethod", + "type": "string" + }, + "AzureCredentialConfig": { + "additionalProperties": false, + "description": "Unified Azure authentication configuration.\n\nThis class provides a reusable authentication configuration that can be\ncomposed into any Azure connector's configuration. It supports multiple\nauthentication methods and returns a TokenCredential that works with\nany Azure SDK client.\n\nExample usage in a connector config:\n class MyAzureConnectorConfig(ConfigModel):\n credential: AzureCredentialConfig = Field(\n default_factory=AzureCredentialConfig,\n description=\"Azure authentication configuration\"\n )\n subscription_id: str = Field(...)", + "properties": { + "authentication_method": { + "$ref": "#/$defs/AzureAuthenticationMethod", + "default": "default", + "description": "Authentication method to use. Options: 'default' (auto-detects from environment), 'service_principal' (client ID + secret + tenant), 'managed_identity' (Azure Managed Identity), 'cli' (Azure CLI credential). Recommended: Use 'default' which tries multiple methods automatically." + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure Application (client) ID. Required for service_principal authentication. Find this in Azure Portal > App registrations > Your app > Overview.", + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure client secret. Required for service_principal authentication. Create in Azure Portal > App registrations > Your app > Certificates & secrets.", + "title": "Client Secret" + }, + "tenant_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure tenant (directory) ID. Required for service_principal authentication. Find this in Azure Portal > Microsoft Entra ID > Overview.", + "title": "Tenant Id" + }, + "managed_identity_client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Client ID for user-assigned managed identity. Leave empty to use system-assigned managed identity. Only used when authentication_method is 'managed_identity'.", + "title": "Managed Identity Client Id" + }, + "exclude_cli_credential": { + "default": false, + "description": "When using 'default' authentication, exclude Azure CLI credential. Useful in production to avoid accidentally using developer credentials.", + "title": "Exclude Cli Credential", + "type": "boolean" + }, + "exclude_environment_credential": { + "default": false, + "description": "When using 'default' authentication, exclude environment variables. Environment variables checked: AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID.", + "title": "Exclude Environment Credential", + "type": "boolean" + }, + "exclude_managed_identity_credential": { + "default": false, + "description": "When using 'default' authentication, exclude managed identity. Useful during local development when managed identity is not available.", + "title": "Exclude Managed Identity Credential", + "type": "boolean" + } + }, + "title": "AzureCredentialConfig", + "type": "object" + }, + "ExtractSchemaConfig": { + "additionalProperties": false, + "description": "Configuration for schema extraction.", + "properties": { + "enabled": { + "default": true, + "description": "Enable schema extraction", + "title": "Enabled", + "type": "boolean" + }, + "method": { + "const": "sql_analytics_endpoint", + "default": "sql_analytics_endpoint", + "description": "Schema extraction method. Currently only 'sql_analytics_endpoint' is supported.", + "title": "Method", + "type": "string" + } + }, + "title": "ExtractSchemaConfig", + "type": "object" + }, + "SqlEndpointConfig": { + "additionalProperties": false, + "description": "Configuration for SQL Analytics Endpoint schema extraction.\n\nReferences:\n- https://learn.microsoft.com/en-us/fabric/data-warehouse/warehouse-connectivity\n- https://learn.microsoft.com/en-us/fabric/data-warehouse/connect-to-fabric-data-warehouse\n- https://learn.microsoft.com/en-us/fabric/data-warehouse/what-is-the-sql-analytics-endpoint-for-a-lakehouse\n- https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?view=sql-server-ver17#encrypt", + "properties": { + "enabled": { + "default": true, + "description": "Enable SQL Analytics Endpoint connection", + "title": "Enabled", + "type": "boolean" + }, + "odbc_driver": { + "default": "ODBC Driver 18 for SQL Server", + "description": "ODBC driver name for SQL Server connections.", + "title": "Odbc Driver", + "type": "string" + }, + "encrypt": { + "default": "yes", + "description": "Enable encryption for SQL Server connections. Valid values: 'yes'/'mandatory' (enable encryption, default in ODBC Driver 18.0+), 'no'/'optional' (disable encryption), or 'strict' (ODBC Driver 18.0+, TDS 8.0 protocol only, always verifies server certificate). See: https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?view=sql-server-ver17#encrypt", + "enum": [ + "yes", + "no", + "mandatory", + "optional", + "strict" + ], + "title": "Encrypt", + "type": "string" + }, + "trust_server_certificate": { + "default": "no", + "description": "Trust server certificate without validation. Set to 'yes' only if certificate validation fails. When 'encrypt=strict', this setting is ignored and certificate validation is always performed. See: https://learn.microsoft.com/en-us/sql/connect/odbc/dsn-connection-string-attribute?view=sql-server-ver17", + "enum": [ + "yes", + "no" + ], + "title": "Trust Server Certificate", + "type": "string" + }, + "query_timeout": { + "default": 30, + "description": "Timeout for SQL queries in seconds", + "maximum": 300, + "minimum": 1, + "title": "Query Timeout", + "type": "integer" + } + }, + "title": "SqlEndpointConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Configuration for Fabric OneLake source.\n\nThis connector extracts metadata from Microsoft Fabric OneLake including:\n- Workspaces as Containers\n- Lakehouses as Containers\n- Warehouses as Containers\n- Schemas as Containers\n- Tables as Datasets with schema metadata\n\nNote on Tenant/Platform Instance:\nThe Fabric REST API does not expose tenant-level endpoints or operations.\nAll API operations are performed at the workspace level. To represent tenant-level\norganization in DataHub, users should set the `platform_instance` configuration\nfield to their tenant identifier (e.g., \"contoso-tenant\"). This will be included\nin all container and dataset URNs, effectively grouping all workspaces under the\nspecified platform instance/tenant.", + "properties": { + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for stateful ingestion and stale entity removal. When enabled, tracks ingested entities and removes those that no longer exist in Fabric." + }, + "credential": { + "$ref": "#/$defs/AzureCredentialConfig", + "description": "Azure authentication configuration. Supports service principal, managed identity, Azure CLI, or auto-detection (DefaultAzureCredential). See AzureCredentialConfig for detailed options." + }, + "workspace_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter workspaces by name. Example: allow=['prod-.*'], deny=['.*-test']" + }, + "lakehouse_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter lakehouses by name. Applied to all workspaces matching workspace_pattern." + }, + "warehouse_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter warehouses by name. Applied to all workspaces matching workspace_pattern." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables by name. Format: 'schema.table' or just 'table' for default schema." + }, + "extract_lakehouses": { + "default": true, + "description": "Whether to extract lakehouses and their tables.", + "title": "Extract Lakehouses", + "type": "boolean" + }, + "extract_warehouses": { + "default": true, + "description": "Whether to extract warehouses and their tables.", + "title": "Extract Warehouses", + "type": "boolean" + }, + "extract_schemas": { + "default": true, + "description": "Whether to extract schema containers. If False, tables will be directly under lakehouse/warehouse containers.", + "title": "Extract Schemas", + "type": "boolean" + }, + "api_timeout": { + "default": 30, + "description": "Timeout for REST API calls in seconds.", + "maximum": 300, + "minimum": 1, + "title": "Api Timeout", + "type": "integer" + }, + "extract_schema": { + "$ref": "#/$defs/ExtractSchemaConfig", + "description": "Configuration for schema extraction from tables." + }, + "sql_endpoint": { + "anyOf": [ + { + "$ref": "#/$defs/SqlEndpointConfig" + }, + { + "type": "null" + } + ], + "description": "SQL Analytics Endpoint configuration for schema extraction. Required when extract_schema.enabled=True and extract_schema.method='sql_analytics_endpoint'." + } + }, + "title": "FabricOneLakeSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Basic Recipe + +```yaml +source: + type: fabric-onelake + config: + # Authentication (using service principal) + credential: + authentication_method: service_principal + client_id: ${AZURE_CLIENT_ID} + client_secret: ${AZURE_CLIENT_SECRET} + tenant_id: ${AZURE_TENANT_ID} + + # Optional: Platform instance (use as tenant identifier) + # platform_instance: "contoso-tenant" + + # Optional: Environment + # env: PROD + + # Optional: Filter workspaces by name pattern + # workspace_pattern: + # allow: + # - "prod-.*" + # deny: + # - ".*-test" + + # Optional: Filter lakehouses by name pattern + # lakehouse_pattern: + # allow: + # - ".*" + # deny: [] + + # Optional: Filter warehouses by name pattern + # warehouse_pattern: + # allow: + # - ".*" + # deny: [] + + # Optional: Filter tables by name pattern + # table_pattern: + # allow: + # - ".*" + # deny: [] + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" +``` + +#### Advanced Configuration + +```yaml +source: + type: fabric-onelake + config: + credential: + authentication_method: service_principal + client_id: ${AZURE_CLIENT_ID} + client_secret: ${AZURE_CLIENT_SECRET} + tenant_id: ${AZURE_TENANT_ID} + + # Platform instance (represents tenant) + platform_instance: "contoso-tenant" + + # Environment + env: PROD + + # Filtering + workspace_pattern: + allow: + - "prod-.*" + - "shared-.*" + deny: + - ".*-test" + - ".*-dev" + + lakehouse_pattern: + allow: + - ".*" + deny: + - ".*-backup" + + warehouse_pattern: + allow: + - ".*" + deny: [] + + table_pattern: + allow: + - ".*" + deny: + - ".*_temp" + - ".*_backup" + + # Feature flags + extract_lakehouses: true + extract_warehouses: true + extract_schemas: true # Set to false to skip schema containers + + # API timeout (seconds) + api_timeout: 30 + + # Stateful ingestion (optional) + stateful_ingestion: + enabled: true + remove_stale_metadata: true + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" +``` + +#### Using Managed Identity + +```yaml +source: + type: fabric-onelake + config: + credential: + authentication_method: managed_identity + # For user-assigned managed identity, specify client_id + # client_id: ${MANAGED_IDENTITY_CLIENT_ID} + + platform_instance: "contoso-tenant" + env: PROD + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" +``` + +#### Using Azure CLI (Local Development) + +```yaml +source: + type: fabric-onelake + config: + credential: + authentication_method: cli + # Run 'az login' first + + platform_instance: "contoso-tenant" + env: DEV + +sink: + type: datahub-rest + config: + server: "http://localhost:8080" +``` + +#### Schema Extraction + +Schema extraction (column metadata) is supported via the SQL Analytics Endpoint. This feature extracts column names, data types, nullability, and ordinal positions from tables in both Lakehouses and Warehouses. + +#### Prerequisites for Schema Extraction + +Schema extraction via SQL Analytics Endpoint requires ODBC drivers to be installed on the system. + +##### 1. ODBC Driver Manager + +First, install the ODBC driver manager (UnixODBC) on your system: + +**Ubuntu/Debian:** + +```bash +sudo apt-get update +sudo apt-get install -y unixodbc unixodbc-dev +``` + +**RHEL/CentOS/Fedora:** + +```bash +# RHEL/CentOS 7/8 +sudo yum install -y unixODBC unixODBC-devel + +# Fedora / RHEL 9+ +sudo dnf install -y unixODBC unixODBC-devel +``` + +**macOS:** + +```bash +brew install unixodbc +``` + +##### 2. Microsoft ODBC Driver for SQL Server + +Install the Microsoft ODBC Driver 18 for SQL Server (required for connecting to Fabric SQL Analytics Endpoint): + +**Ubuntu 20.04/22.04:** + +```bash +curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - +curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list +sudo apt-get update +sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 +``` + +**RHEL/CentOS 7/8:** + +```bash +sudo curl -o /etc/yum.repos.d/mssql-release.repo https://packages.microsoft.com/config/rhel/$(rpm -E %{rhel})/mssql-release.repo +sudo ACCEPT_EULA=Y yum install -y msodbcsql18 +``` + +**RHEL 9 / Fedora:** + +```bash +sudo curl -o /etc/yum.repos.d/mssql-release.repo https://packages.microsoft.com/config/rhel/9/mssql-release.repo +sudo ACCEPT_EULA=Y dnf install -y msodbcsql18 +``` + +**macOS:** + +```bash +brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release +brew update +HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 mssql-tools18 +``` + +**Windows:** +Download and install from [Microsoft ODBC Driver for SQL Server](https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server). + +##### 3. Verify Installation + +After installation, verify that the ODBC driver is available: + +```bash +odbcinst -q -d +``` + +You should see `ODBC Driver 18 for SQL Server` in the list. + +##### 4. Permissions + +Your Azure identity must have access to query the SQL Analytics Endpoint (same permissions as accessing the endpoint via SQL tools). + +##### 5. Python Dependencies + +The `fabric-onelake` extra includes `sqlalchemy` and `pyodbc` dependencies. Install them with: + +```bash +pip install 'acryl-datahub[fabric-onelake]' +``` + +**Note:** If you encounter `libodbc.so.2: cannot open shared object file` errors, ensure the ODBC driver manager is installed (step 1 above). + +#### Schema Extraction Configuration + +Schema extraction is enabled by default. You can configure it as follows: + +```yaml +source: + type: fabric-onelake + config: + credential: + authentication_method: service_principal + client_id: ${AZURE_CLIENT_ID} + client_secret: ${AZURE_CLIENT_SECRET} + tenant_id: ${AZURE_TENANT_ID} + + # Schema extraction configuration + extract_schema: + enabled: true # Enable schema extraction (default: true) + method: sql_analytics_endpoint # Currently only this method is supported + + # SQL Analytics Endpoint configuration + sql_endpoint: + enabled: true # Enable SQL endpoint connection (default: true) + # Optional: ODBC connection options + # odbc_driver: "ODBC Driver 18 for SQL Server" # Default: "ODBC Driver 18 for SQL Server" + # encrypt: "yes" # Enable encryption (default: "yes") + # trust_server_certificate: "no" # Trust server certificate (default: "no") + query_timeout: 30 # Timeout for SQL queries in seconds (default: 30) +``` + +#### How It Works + +1. **Endpoint Discovery**: The SQL Analytics Endpoint URL is automatically fetched from the Fabric API for each Lakehouse/Warehouse. The endpoint format is `.datawarehouse.fabric.microsoft.com` and cannot be constructed from workspace_id alone. +2. **Authentication**: Uses the same Azure credentials configured for REST API access with Azure AD token injection +3. **Connection**: Connects to the SQL Analytics Endpoint using ODBC with the discovered endpoint URL +4. **Query**: Queries `INFORMATION_SCHEMA.COLUMNS` to extract column metadata (required for schema extraction) +5. **Type Mapping**: SQL Server data types are automatically mapped to DataHub types using the standard type mapping system + +**References:** + +- [What is the SQL analytics endpoint for a lakehouse?](https://learn.microsoft.com/en-us/fabric/data-engineering/lakehouse-sql-analytics-endpoint) +- [Warehouse connectivity in Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/data-warehouse/connectivity) +- [Connect to Fabric Data Warehouse](https://learn.microsoft.com/en-us/fabric/data-warehouse/how-to-connect) + +#### Important Notes + +- **Endpoint URL Discovery**: The SQL Analytics Endpoint URL is automatically fetched from the Fabric API for each Lakehouse/Warehouse. The endpoint format is `.datawarehouse.fabric.microsoft.com` and cannot be constructed from workspace_id alone. If the endpoint URL cannot be retrieved from the API, schema extraction will fail for that item. +- **No Fallback**: Unlike legacy Power BI Premium endpoints, Fabric SQL Analytics Endpoints do not support fallback connection strings. The endpoint must be obtained from the API. + +#### Disabling Schema Extraction + +To disable schema extraction and ingest tables without column metadata: + +```yaml +source: + type: fabric-onelake + config: + extract_schema: + enabled: false +``` + +#### Schemas-Enabled vs Schemas-Disabled Lakehouses + +The connector automatically handles both schemas-enabled and schemas-disabled lakehouses: + +- **Schemas-Enabled Lakehouses**: The connector uses OneLake Delta Table APIs to list schemas first, then tables within each schema. This requires Storage audience tokens (`https://storage.azure.com/.default`). +- **Schemas-Disabled Lakehouses**: The connector uses the standard Fabric REST API `/tables` endpoint, which lists all tables. Tables without an explicit schema are automatically assigned to the `dbo` schema in DataHub. This uses Power BI API scope tokens. + +**Important**: All tables in DataHub will have a schema in their URN, even for schemas-disabled lakehouses. Tables without an explicit schema are normalized to use the `dbo` schema by default. This ensures consistent URN structure across all Fabric entities. + +The connector automatically detects the lakehouse type and uses the appropriate API endpoint. No configuration changes are needed. + +#### Stateful Ingestion + +The connector supports stateful ingestion to track ingested entities and remove stale metadata. Enable it with: + +```yaml +stateful_ingestion: + enabled: true + remove_stale_metadata: true +``` + +When enabled, the connector will: + +- Track all ingested workspaces, lakehouses, warehouses, schemas, and tables +- Remove entities from DataHub that no longer exist in Fabric +- Maintain state across ingestion runs + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +- **Metadata Sync Delays**: The SQL Analytics Endpoint may have delays in reflecting schema changes. New columns or schema modifications may take minutes to hours to appear. +- **Missing Tables**: Some tables may not be visible in the SQL endpoint due to: + - Unsupported data types + - Permission issues + - Table count limits in very large databases +- **Graceful Degradation**: If schema extraction fails for a table, the table will still be ingested without column metadata (no ingestion failure) + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.fabric.onelake.source.FabricOneLakeSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/fabric/onelake/source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Fabric OneLake, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/feast.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/feast.md new file mode 100644 index 00000000..93099dd1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/feast.md @@ -0,0 +1,234 @@ +--- +sidebar_position: 29 +title: Feast +slug: /generated/ingestion/sources/feast +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Feast + +## Overview + +Feast is a machine learning platform. Learn more in the [official Feast documentation](https://feast.dev/). + +The DataHub integration for Feast covers ML entities such as models, features, and related lineage metadata. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +- Entities as `MLPrimaryKey` +- Fields as `MLFeature` +- Feature views and on-demand feature views as `MLFeatureTable` +- Batch and stream source details as `Dataset` +- Column types associated with each entity and feature + +Use this mapping to align feature-store metadata with existing ML entity governance patterns in DataHub. + + +## Module `feast` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default. | + +### Overview + +The `feast` module ingests metadata from Feast into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[feast]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: feast + config: + # Coordinates + path: "/path/to/repository/" + # Options + environment: "PROD" + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
path ✅
string
| Path to Feast repository | +|
enable_owner_extraction
boolean
| If this is disabled, then we NEVER try to map owners. If this is enabled, then owner_mappings is REQUIRED to extract ownership.
Default: False
| +|
enable_tag_extraction
boolean
| If this is disabled, then we NEVER try to extract tags.
Default: False
| +|
environment
string
| Environment to use when constructing URNs
Default: PROD
| +|
fs_yaml_file
One of string(path), null
| Path to the `feature_store.yaml` file used to configure the feature store
Default: None
| +|
owner_mappings
One of array, null
| Mapping of owner names to owner types
Default: None
| +|
owner_mappings.map
map(str,string)
| | +|
stateful_ingestion
One of StatefulIngestionConfig, null
| Stateful Ingestion Config
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "StatefulIngestionConfig": { + "additionalProperties": false, + "description": "Basic Stateful Ingestion Specific Configuration for any source.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + } + }, + "title": "StatefulIngestionConfig", + "type": "object" + } + }, + "properties": { + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulIngestionConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Stateful Ingestion Config" + }, + "path": { + "description": "Path to Feast repository", + "title": "Path", + "type": "string" + }, + "fs_yaml_file": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to the `feature_store.yaml` file used to configure the feature store", + "title": "Fs Yaml File" + }, + "environment": { + "default": "PROD", + "description": "Environment to use when constructing URNs", + "title": "Environment", + "type": "string" + }, + "owner_mappings": { + "anyOf": [ + { + "items": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mapping of owner names to owner types", + "title": "Owner Mappings" + }, + "enable_owner_extraction": { + "default": false, + "description": "If this is disabled, then we NEVER try to map owners. If this is enabled, then owner_mappings is REQUIRED to extract ownership.", + "title": "Enable Owner Extraction", + "type": "boolean" + }, + "enable_tag_extraction": { + "default": false, + "description": "If this is disabled, then we NEVER try to extract tags.", + "title": "Enable Tag Extraction", + "type": "boolean" + } + }, + "required": [ + "path" + ], + "title": "FeastRepositorySourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.feast.FeastRepositorySource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/feast.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Feast, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/file-based-lineage.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/file-based-lineage.md new file mode 100644 index 00000000..d7fa1288 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/file-based-lineage.md @@ -0,0 +1,203 @@ +--- +sidebar_position: 30 +title: File Based Lineage +slug: /generated/ingestion/sources/file-based-lineage +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# File Based Lineage + +## Overview + +File Based Lineage is a storage and lakehouse platform. Learn more in the [official File Based Lineage documentation](https://datahub.com/docs/). + +The DataHub integration for File Based Lineage covers file/lakehouse metadata entities such as datasets, paths, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `datahub-lineage-file` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Column-level Lineage | ✅ | Specified in the lineage file. | +| Table-Level Lineage | ✅ | Specified in the lineage file. | + +### Overview + +The `datahub-lineage-file` module ingests metadata from File Based Lineage into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin pulls lineage metadata from a yaml-formatted file. An example of one such file is located in the examples directory [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/bootstrap_data/file_lineage.yml). + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[datahub-lineage-file]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: datahub-lineage-file + config: + # Coordinates + file: /path/to/file_lineage.yml + # Whether we want to query datahub-gms for upstream data + preserve_upstream: False + +sink: +# sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
file ✅
string
| File path or URL to lineage file to ingest. | +|
preserve_upstream
boolean
| Whether we want to query datahub-gms for upstream data. False means it will hard replace upstream data for a given entity. True means it will query the backend for existing upstreams and include it in the ingestion run
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "additionalProperties": false, + "properties": { + "file": { + "description": "File path or URL to lineage file to ingest.", + "title": "File", + "type": "string" + }, + "preserve_upstream": { + "default": true, + "description": "Whether we want to query datahub-gms for upstream data. False means it will hard replace upstream data for a given entity. True means it will query the backend for existing upstreams and include it in the ingestion run", + "title": "Preserve Upstream", + "type": "boolean" + } + }, + "required": [ + "file" + ], + "title": "LineageFileSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Lineage File Format + +The lineage source file should be a `.yml` file with the following top-level keys: + +**version**: the version of lineage file config the config conforms to. Currently, the only version released +is `1`. + +**lineage**: the top level key of the lineage file containing a list of **EntityNodeConfig** objects + +**EntityNodeConfig**: + +- **entity**: **EntityConfig** object +- **upstream**: (optional) list of child **EntityNodeConfig** objects +- **fineGrainedLineages**: (optional) list of **FineGrainedLineageConfig** objects + +**EntityConfig**: + +- **name**: identifier of the entity. Typically name or guid, as used in constructing entity urn. +- **type**: type of the entity (only `dataset` is supported as of now) +- **env**: the environment of this entity. Should match the values in the + table [here](/docs/graphql/enums/#fabrictype) +- **platform**: a valid platform like kafka, snowflake, etc.. +- **platform_instance**: optional string specifying the platform instance of this entity + +For example if dataset URN is `urn:li:dataset:(urn:li:dataPlatform:redshift,userdb.public.customer_table,DEV)` then **EntityConfig** will look like: + +```yml +name: userdb.public.customer_table +type: dataset +env: DEV +platform: redshift +``` + +**FineGrainedLineageConfig**: + +- **upstreamType**: type of upstream entity in a fine-grained lineage; default = "FIELD_SET" +- **upstreams**: (optional) list of upstream schema field urns +- **downstreamType**: type of downstream entity in a fine-grained lineage; default = "FIELD_SET" +- **downstreams**: (optional) list of downstream schema field urns +- **transformOperation**: (optional) transform operation applied to the upstream entities to produce the downstream field(s) +- **confidenceScore**: (optional) the confidence in this lineage between 0 (low confidence) and 1 (high confidence); default = 1.0 + +**FineGrainedLineageConfig** can be used to display fine grained lineage, also referred to as column-level lineage, +for custom sources. + +You can also view an example lineage file checked in [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/bootstrap_data/file_lineage.yml) + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.metadata.lineage.LineageFileSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/metadata/lineage.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for File Based Lineage, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fivetran.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fivetran.md new file mode 100644 index 00000000..156c9173 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/fivetran.md @@ -0,0 +1,1425 @@ +--- +sidebar_position: 31 +title: Fivetran +slug: /generated/ingestion/sources/fivetran +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Fivetran + +## Overview + +Fivetran is a streaming or integration platform. Learn more in the [official Fivetran documentation](https://www.fivetran.com/). + +The DataHub integration for Fivetran covers streaming/integration entities such as topics, connectors, pipelines, or jobs. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Fivetran | Datahub | +| --------------- | ----------------------------------------------------------------------------------------------------- | +| `Connector` | [DataJob](/docs/generated/metamodel/entities/datajob/) | +| `Source` | [Dataset](/docs/generated/metamodel/entities/dataset/) | +| `Destination` | [Dataset](/docs/generated/metamodel/entities/dataset/) | +| `Connector Run` | [DataProcessInstance](/docs/generated/metamodel/entities/dataprocessinstance) | + +Source and destination are mapped to Dataset as an Input and Output of Connector. + + +## Module `fivetran` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Column-level Lineage | ✅ | Enabled by default, can be disabled via configuration `include_column_lineage`. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | + +### Overview + +The `fivetran` module ingests metadata from Fivetran into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +#### Integration Details + +This source extracts the following: + +- Connectors in fivetran as Data Pipelines and Data Jobs to represent data lineage information between source and destination. +- Connector sources - DataJob input Datasets. +- Connector destination - DataJob output Datasets. +- Connector runs - DataProcessInstances as DataJob runs. + +#### Configuration Notes + +**Prerequisites:** + +1. Set up and complete initial sync of the [Fivetran Platform Connector](https://fivetran.com/docs/logs/fivetran-platform/setup-guide) +2. Enable automatic schema updates (default) to avoid sync inconsistencies +3. Configure the destination platform (Snowflake, BigQuery, or Databricks) in your recipe + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +To use the Fivetran REST API integration, you need: + +**Required API Permissions**: + +- **Read access** to connection details (`GET /v1/connections/{connection_id}`) +- The API key must be associated with a user or service account that has access to the connectors you want to ingest +- The API key inherits permissions from the user or service account it's associated with + +#### Fivetran REST API Configuration + +The Fivetran REST API configuration is **required** for Google Sheets connectors and optional for other use cases. It provides access to connection details that aren't available in the Platform Connector logs. + +##### Setup + +To obtain API credentials: + +1. Log in to your Fivetran account +2. Go to **Settings** → **API Config** +3. Create or use an existing API key and secret + +```yaml +api_config: + api_key: "your_api_key" + api_secret: "your_api_secret" + base_url: "https://api.fivetran.com" # Optional, defaults to this + request_timeout_sec: 30 # Optional, defaults to 30 seconds +``` + +#### Google Sheets Connector Support + +Google Sheets connectors require special handling because Google Sheets is not yet natively supported as a DataHub source. As a workaround, the Fivetran source creates Dataset entities for Google Sheets and includes them in the lineage. + +##### Requirements + +- **Fivetran REST API configuration** (`api_config`) is required for Google Sheets connectors +- The API is used to fetch connection details that aren't available in Platform Connector logs + +##### What Gets Created + +For each Google Sheets connector, two Dataset entities are created: + +1. **Google Sheet Dataset**: Represents the entire Google Sheet + + - Platform: `google_sheets` + - Subtype: `GOOGLE_SHEETS` + - Contains the sheet ID extracted from the Google Sheets URL + +2. **Named Range Dataset**: Represents the specific named range being synced + - Platform: `google_sheets` + - Subtype: `GOOGLE_SHEETS_NAMED_RANGE` + - Contains the named range identifier + - Has upstream lineage to the Google Sheet Dataset + +##### Limitations + +- **Column lineage is disabled** for Google Sheets connectors due to stale metadata issues in the Fivetran Platform Connector (as of October 2025) +- This is a workaround that will be removed once DataHub natively supports Google Sheets as a source +- If the Fivetran API is unavailable or the connector details can't be fetched, the connector will be skipped with a warning + +##### Example Configuration + +```yaml +source: + type: fivetran + config: + # Required for Google Sheets connectors + api_config: + api_key: "your_api_key" + api_secret: "your_api_secret" + + # ... other configuration ... +``` + + +### Install the Plugin +```shell +pip install 'acryl-datahub[fivetran]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: fivetran + config: + # Fivetran log connector destination server configurations + fivetran_log_config: + destination_platform: snowflake + # Optional - If destination platform is 'snowflake', provide snowflake configuration. + snowflake_destination_config: + # Coordinates + account_id: "abc48144" + warehouse: "COMPUTE_WH" + database: "MY_SNOWFLAKE_DB" + log_schema: "FIVETRAN_LOG" + + # Credentials + username: "${SNOWFLAKE_USER}" + password: "${SNOWFLAKE_PASS}" + role: "snowflake_role" + # Optional - If destination platform is 'bigquery', provide bigquery configuration. + bigquery_destination_config: + # Credentials + credential: + private_key_id: "project_key_id" + project_id: "project_id" + client_email: "client_email" + client_id: "client_id" + private_key: "private_key" + dataset: "fivetran_log_dataset" + # Optional - If destination platform is 'databricks', provide databricks configuration. + databricks_destination_config: + # Credentials + credential: + token: "token" + workspace_url: "workspace_url" + warehouse_id: "warehouse_id" + + # Coordinates + catalog: "fivetran_catalog" + log_schema: "fivetran_log" + + # Optional - filter for certain connector names instead of ingesting everything. + # connector_patterns: + # allow: + # - connector_name + + # Optional -- A mapping of the connector's all sources to its database. + # sources_to_database: + # connector_id: source_db + + # Optional - Fivetran REST API configuration (required for Google Sheets connectors) + # api_config: + # api_key: "your_api_key" + # api_secret: "your_api_secret" + # base_url: "https://api.fivetran.com" # Optional + # request_timeout_sec: 30 # Optional + # Optional -- This mapping is optional and only required to configure platform-instance for source + # A mapping of Fivetran connector id to data platform instance + # sources_to_platform_instance: + # connector_id: + # platform_instance: cloud_instance + # env: DEV + + # Optional -- This mapping is optional and only required to configure platform-instance for destination. + # A mapping of Fivetran destination id to data platform instance + # destination_to_platform_instance: + # destination_id: + # platform_instance: cloud_instance + # env: DEV + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
fivetran_log_config ✅
FivetranLogConfig
| | +|
fivetran_log_config.destination_platform
Enum
| One of: "snowflake", "bigquery", "databricks"
Default: snowflake
| +|
fivetran_log_config.max_column_lineage_per_connector
integer
| Maximum number of column lineage entries to retrieve per connector. This acts as a safety net to prevent excessive data ingestion. When this limit is exceeded, only the most recent entries are ingested.
Default: 1000
| +|
fivetran_log_config.max_jobs_per_connector
integer
| Maximum number of sync jobs to retrieve per connector. This acts as a safety net to prevent excessive data ingestion. Increase cautiously if you need to see more historical sync runs.
Default: 500
| +|
fivetran_log_config.max_table_lineage_per_connector
integer
| Maximum number of table lineage entries to retrieve per connector. This acts as a safety net to prevent excessive data ingestion. When this limit is exceeded, only the most recent entries are ingested.
Default: 120
| +|
fivetran_log_config.bigquery_destination_config
One of BigQueryDestinationConfig, null
| If destination platform is 'bigquery', provide bigquery configuration.
Default: None
| +|
fivetran_log_config.bigquery_destination_config.dataset ❓
string
| The fivetran connector log dataset. | +|
fivetran_log_config.bigquery_destination_config.extra_client_options
object
| Additional options to pass to google.cloud.logging_v2.client.Client.
Default: {}
| +|
fivetran_log_config.bigquery_destination_config.project_on_behalf
One of string, null
| [Advanced] The BigQuery project in which queries are executed. Will be passed when creating a job. If not passed, falls back to the project associated with the service account.
Default: None
| +|
fivetran_log_config.bigquery_destination_config.credential
One of GCPCredential, null
| BigQuery credential informations
Default: None
| +|
fivetran_log_config.bigquery_destination_config.credential.client_email ❓
string
| Client email | +|
fivetran_log_config.bigquery_destination_config.credential.client_id ❓
string
| Client Id | +|
fivetran_log_config.bigquery_destination_config.credential.private_key ❓
string(password)
| Private key in a form of '-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n' | +|
fivetran_log_config.bigquery_destination_config.credential.private_key_id ❓
string
| Private key id | +|
fivetran_log_config.bigquery_destination_config.credential.auth_provider_x509_cert_url
string
| Auth provider x509 certificate url
Default: https://www.googleapis.com/oauth2/v1/certs
| +|
fivetran_log_config.bigquery_destination_config.credential.auth_uri
string
| Authentication uri
Default: https://accounts.google.com/o/oauth2/auth
| +|
fivetran_log_config.bigquery_destination_config.credential.client_x509_cert_url
One of string, null
| If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email
Default: None
| +|
fivetran_log_config.bigquery_destination_config.credential.project_id
One of string, null
| Project id to set the credentials
Default: None
| +|
fivetran_log_config.bigquery_destination_config.credential.token_uri
string
| Token uri
Default: https://oauth2.googleapis.com/token
| +|
fivetran_log_config.bigquery_destination_config.credential.type
string
| Authentication type
Default: service_account
| +|
fivetran_log_config.databricks_destination_config
One of DatabricksDestinationConfig, null
| If destination platform is 'databricks', provide databricks configuration.
Default: None
| +|
fivetran_log_config.databricks_destination_config.catalog ❓
string
| The fivetran connector log catalog. | +|
fivetran_log_config.databricks_destination_config.log_schema ❓
string
| The fivetran connector log schema. | +|
fivetran_log_config.databricks_destination_config.workspace_url ❓
string
| Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com | +|
fivetran_log_config.databricks_destination_config.client_id
One of string, null
| Databricks service principal client ID
Default: None
| +|
fivetran_log_config.databricks_destination_config.client_secret
One of string(password), null
| Databricks service principal client secret
Default: None
| +|
fivetran_log_config.databricks_destination_config.extra_client_options
object
| Additional options to pass to Databricks SQLAlchemy client.
Default: {}
| +|
fivetran_log_config.databricks_destination_config.scheme
string
|
Default: databricks
| +|
fivetran_log_config.databricks_destination_config.token
One of string(password), null
| Databricks personal access token
Default: None
| +|
fivetran_log_config.databricks_destination_config.warehouse_id
One of string, null
| SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. Required for the following features that need SQL access: 1) Tag extraction (include_tags=True) - queries system.information_schema.tags 2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog 3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage 4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue.
Default: None
| +|
fivetran_log_config.databricks_destination_config.azure_auth
One of AzureAuthConfig, null
| Azure configuration
Default: None
| +|
fivetran_log_config.databricks_destination_config.azure_auth.client_id ❓
string
| Azure application (client) ID. This is the unique identifier for the registered Azure AD application. | +|
fivetran_log_config.databricks_destination_config.azure_auth.client_secret ❓
string(password)
| Azure application client secret used for authentication. This is a confidential credential that should be kept secure. | +|
fivetran_log_config.databricks_destination_config.azure_auth.tenant_id ❓
string
| Azure tenant (directory) ID. This identifies the Azure AD tenant where the application is registered. | +|
fivetran_log_config.snowflake_destination_config
One of SnowflakeDestinationConfig, null
| If destination platform is 'snowflake', provide snowflake configuration.
Default: None
| +|
fivetran_log_config.snowflake_destination_config.account_id ❓
string
| Snowflake account identifier. e.g. xy12345, xy12345.us-east-2.aws, xy12345.us-central1.gcp, xy12345.central-us.azure, xy12345.us-west-2.privatelink. Refer [Account Identifiers](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#format-2-legacy-account-locator-in-a-region) for more details. | +|
fivetran_log_config.snowflake_destination_config.database ❓
string
| The fivetran connector log database. | +|
fivetran_log_config.snowflake_destination_config.log_schema ❓
string
| The fivetran connector log schema. | +|
fivetran_log_config.snowflake_destination_config.authentication_type
string
| The type of authenticator to use when connecting to Snowflake. Supports "DEFAULT_AUTHENTICATOR", "OAUTH_AUTHENTICATOR", "EXTERNAL_BROWSER_AUTHENTICATOR" and "KEY_PAIR_AUTHENTICATOR".
Default: DEFAULT_AUTHENTICATOR
| +|
fivetran_log_config.snowflake_destination_config.connect_args
One of object, null
| Connect args to pass to Snowflake SqlAlchemy driver
Default: None
| +|
fivetran_log_config.snowflake_destination_config.options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. | +|
fivetran_log_config.snowflake_destination_config.password
One of string(password), null
| Snowflake password.
Default: None
| +|
fivetran_log_config.snowflake_destination_config.private_key
One of string(password), null
| Private key in a form of '-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n' if using key pair authentication. Encrypted version of private key will be in a form of '-----BEGIN ENCRYPTED PRIVATE KEY-----\nencrypted-private-key\n-----END ENCRYPTED PRIVATE KEY-----\n' See: https://docs.snowflake.com/en/user-guide/key-pair-auth.html
Default: None
| +|
fivetran_log_config.snowflake_destination_config.private_key_password
One of string(password), null
| Password for your private key. Required if using key pair authentication with encrypted private key.
Default: None
| +|
fivetran_log_config.snowflake_destination_config.private_key_path
One of string, null
| The path to the private key if using key pair authentication. Ignored if `private_key` is set. See: https://docs.snowflake.com/en/user-guide/key-pair-auth.html
Default: None
| +|
fivetran_log_config.snowflake_destination_config.role
One of string, null
| Snowflake role.
Default: None
| +|
fivetran_log_config.snowflake_destination_config.snowflake_domain
string
| Snowflake domain. Use 'snowflakecomputing.com' for most regions or 'snowflakecomputing.cn' for China (cn-northwest-1) region.
Default: snowflakecomputing.com
| +|
fivetran_log_config.snowflake_destination_config.token
One of string(password), null
| OAuth token from external identity provider. Not recommended for most use cases because it will not be able to refresh once expired.
Default: None
| +|
fivetran_log_config.snowflake_destination_config.username
One of string, null
| Snowflake username.
Default: None
| +|
fivetran_log_config.snowflake_destination_config.warehouse
One of string, null
| Snowflake warehouse.
Default: None
| +|
fivetran_log_config.snowflake_destination_config.oauth_config
One of OAuthConfiguration, null
| oauth configuration - https://docs.snowflake.com/en/user-guide/python-connector-example.html#connecting-with-oauth
Default: None
| +|
fivetran_log_config.snowflake_destination_config.oauth_config.authority_url ❓
string
| Authority url of your identity provider | +|
fivetran_log_config.snowflake_destination_config.oauth_config.client_id ❓
string
| client id of your registered application | +|
fivetran_log_config.snowflake_destination_config.oauth_config.provider ❓
Enum
| One of: "microsoft", "okta" | +|
fivetran_log_config.snowflake_destination_config.oauth_config.scopes ❓
array
| scopes required to connect to snowflake | +|
fivetran_log_config.snowflake_destination_config.oauth_config.scopes.string
string
| | +|
fivetran_log_config.snowflake_destination_config.oauth_config.client_secret
One of string(password), null
| client secret of the application if use_certificate = false
Default: None
| +|
fivetran_log_config.snowflake_destination_config.oauth_config.encoded_oauth_private_key
One of string(password), null
| base64 encoded private key content if use_certificate = true
Default: None
| +|
fivetran_log_config.snowflake_destination_config.oauth_config.encoded_oauth_public_key
One of string, null
| base64 encoded certificate content if use_certificate = true
Default: None
| +|
fivetran_log_config.snowflake_destination_config.oauth_config.use_certificate
boolean
| Do you want to use certificate and private key to authenticate using oauth
Default: False
| +|
history_sync_lookback_period
integer
| The number of days to look back when extracting connectors' sync history.
Default: 7
| +|
include_column_lineage
boolean
| Populates table->table column lineage.
Default: True
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
api_config
One of FivetranAPIConfig, null
| Fivetran REST API configuration, used to provide wider support for connections.
Default: None
| +|
api_config.api_key ❓
string(password)
| Fivetran API key | +|
api_config.api_secret ❓
string(password)
| Fivetran API secret | +|
api_config.base_url
string
| Fivetran API base URL
Default: https://api.fivetran.com
| +|
api_config.request_timeout_sec
integer
| Request timeout in seconds
Default: 30
| +|
connector_patterns
AllowDenyPattern
| A class to store allow deny regexes | +|
connector_patterns.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
connector_patterns.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
connector_patterns.allow.string
string
| | +|
connector_patterns.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
connector_patterns.deny.string
string
| | +|
destination_patterns
AllowDenyPattern
| A class to store allow deny regexes | +|
destination_patterns.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
destination_patterns.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
destination_patterns.allow.string
string
| | +|
destination_patterns.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
destination_patterns.deny.string
string
| | +|
destination_to_platform_instance
map(str,PlatformDetail)
| | +|
destination_to_platform_instance.`key`.platform
One of string, null
| Override the platform type detection.
Default: None
| +|
destination_to_platform_instance.`key`.database
One of string, null
| The database that all assets produced by this connector belong to. For destinations, this defaults to the fivetran log config's database.
Default: None
| +|
destination_to_platform_instance.`key`.include_schema_in_urn
boolean
| Include schema in the dataset URN. In some cases, the schema is not relevant to the dataset URN and Fivetran sets it to the source and destination table names in the connector.
Default: True
| +|
destination_to_platform_instance.`key`.platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to
Default: None
| +|
destination_to_platform_instance.`key`.env
string
| The environment that all assets produced by DataHub platform ingestion source belong to
Default: PROD
| +|
sources_to_platform_instance
map(str,PlatformDetail)
| | +|
sources_to_platform_instance.`key`.platform
One of string, null
| Override the platform type detection.
Default: None
| +|
sources_to_platform_instance.`key`.database
One of string, null
| The database that all assets produced by this connector belong to. For destinations, this defaults to the fivetran log config's database.
Default: None
| +|
sources_to_platform_instance.`key`.include_schema_in_urn
boolean
| Include schema in the dataset URN. In some cases, the schema is not relevant to the dataset URN and Fivetran sets it to the source and destination table names in the connector.
Default: True
| +|
sources_to_platform_instance.`key`.platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to
Default: None
| +|
sources_to_platform_instance.`key`.env
string
| The environment that all assets produced by DataHub platform ingestion source belong to
Default: PROD
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Fivetran Stateful Ingestion Config.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AzureAuthConfig": { + "additionalProperties": false, + "properties": { + "client_secret": { + "description": "Azure application client secret used for authentication. This is a confidential credential that should be kept secure.", + "format": "password", + "title": "Client Secret", + "type": "string", + "writeOnly": true + }, + "client_id": { + "description": "Azure application (client) ID. This is the unique identifier for the registered Azure AD application.", + "title": "Client Id", + "type": "string" + }, + "tenant_id": { + "description": "Azure tenant (directory) ID. This identifies the Azure AD tenant where the application is registered.", + "title": "Tenant Id", + "type": "string" + } + }, + "required": [ + "client_secret", + "client_id", + "tenant_id" + ], + "title": "AzureAuthConfig", + "type": "object" + }, + "BigQueryDestinationConfig": { + "additionalProperties": false, + "properties": { + "credential": { + "anyOf": [ + { + "$ref": "#/$defs/GCPCredential" + }, + { + "type": "null" + } + ], + "default": null, + "description": "BigQuery credential informations" + }, + "extra_client_options": { + "additionalProperties": true, + "default": {}, + "description": "Additional options to pass to google.cloud.logging_v2.client.Client.", + "title": "Extra Client Options", + "type": "object" + }, + "project_on_behalf": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "[Advanced] The BigQuery project in which queries are executed. Will be passed when creating a job. If not passed, falls back to the project associated with the service account.", + "title": "Project On Behalf" + }, + "dataset": { + "description": "The fivetran connector log dataset.", + "title": "Dataset", + "type": "string" + } + }, + "required": [ + "dataset" + ], + "title": "BigQueryDestinationConfig", + "type": "object" + }, + "DatabricksDestinationConfig": { + "additionalProperties": false, + "properties": { + "scheme": { + "default": "databricks", + "title": "Scheme", + "type": "string" + }, + "token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Databricks personal access token", + "title": "Token" + }, + "azure_auth": { + "anyOf": [ + { + "$ref": "#/$defs/AzureAuthConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Azure configuration" + }, + "client_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Databricks service principal client ID", + "title": "Client Id" + }, + "client_secret": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Databricks service principal client secret", + "title": "Client Secret" + }, + "workspace_url": { + "description": "Databricks workspace url. e.g. https://my-workspace.cloud.databricks.com", + "title": "Workspace Url", + "type": "string" + }, + "warehouse_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "SQL Warehouse id, for running queries. Must be explicitly provided to enable SQL-based features. Required for the following features that need SQL access: 1) Tag extraction (include_tags=True) - queries system.information_schema.tags 2) Hive Metastore catalog (include_hive_metastore=True) - queries legacy hive_metastore catalog 3) System table lineage (lineage_data_source=SYSTEM_TABLES) - queries system.access.table_lineage/column_lineage 4) Data profiling (profiling.enabled=True) - runs SELECT/ANALYZE queries on tables. When warehouse_id is missing, these features will be automatically disabled (with warnings) to allow ingestion to continue.", + "title": "Warehouse Id" + }, + "extra_client_options": { + "additionalProperties": true, + "default": {}, + "description": "Additional options to pass to Databricks SQLAlchemy client.", + "title": "Extra Client Options", + "type": "object" + }, + "catalog": { + "description": "The fivetran connector log catalog.", + "title": "Catalog", + "type": "string" + }, + "log_schema": { + "description": "The fivetran connector log schema.", + "title": "Log Schema", + "type": "string" + } + }, + "required": [ + "workspace_url", + "catalog", + "log_schema" + ], + "title": "DatabricksDestinationConfig", + "type": "object" + }, + "FivetranAPIConfig": { + "additionalProperties": false, + "properties": { + "api_key": { + "description": "Fivetran API key", + "format": "password", + "title": "Api Key", + "type": "string", + "writeOnly": true + }, + "api_secret": { + "description": "Fivetran API secret", + "format": "password", + "title": "Api Secret", + "type": "string", + "writeOnly": true + }, + "base_url": { + "default": "https://api.fivetran.com", + "description": "Fivetran API base URL", + "title": "Base Url", + "type": "string" + }, + "request_timeout_sec": { + "default": 30, + "description": "Request timeout in seconds", + "title": "Request Timeout Sec", + "type": "integer" + } + }, + "required": [ + "api_key", + "api_secret" + ], + "title": "FivetranAPIConfig", + "type": "object" + }, + "FivetranLogConfig": { + "additionalProperties": false, + "properties": { + "destination_platform": { + "default": "snowflake", + "description": "The destination platform where fivetran connector log tables are dumped.", + "enum": [ + "snowflake", + "bigquery", + "databricks" + ], + "title": "Destination Platform", + "type": "string" + }, + "snowflake_destination_config": { + "anyOf": [ + { + "$ref": "#/$defs/SnowflakeDestinationConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If destination platform is 'snowflake', provide snowflake configuration." + }, + "bigquery_destination_config": { + "anyOf": [ + { + "$ref": "#/$defs/BigQueryDestinationConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If destination platform is 'bigquery', provide bigquery configuration." + }, + "databricks_destination_config": { + "anyOf": [ + { + "$ref": "#/$defs/DatabricksDestinationConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If destination platform is 'databricks', provide databricks configuration." + }, + "max_jobs_per_connector": { + "default": 500, + "description": "Maximum number of sync jobs to retrieve per connector. This acts as a safety net to prevent excessive data ingestion. Increase cautiously if you need to see more historical sync runs.", + "exclusiveMinimum": 0, + "title": "Max Jobs Per Connector", + "type": "integer" + }, + "max_table_lineage_per_connector": { + "default": 120, + "description": "Maximum number of table lineage entries to retrieve per connector. This acts as a safety net to prevent excessive data ingestion. When this limit is exceeded, only the most recent entries are ingested.", + "exclusiveMinimum": 0, + "title": "Max Table Lineage Per Connector", + "type": "integer" + }, + "max_column_lineage_per_connector": { + "default": 1000, + "description": "Maximum number of column lineage entries to retrieve per connector. This acts as a safety net to prevent excessive data ingestion. When this limit is exceeded, only the most recent entries are ingested.", + "exclusiveMinimum": 0, + "title": "Max Column Lineage Per Connector", + "type": "integer" + } + }, + "title": "FivetranLogConfig", + "type": "object" + }, + "GCPCredential": { + "additionalProperties": false, + "properties": { + "project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Project id to set the credentials", + "title": "Project Id" + }, + "private_key_id": { + "description": "Private key id", + "title": "Private Key Id", + "type": "string" + }, + "private_key": { + "description": "Private key in a form of '-----BEGIN PRIVATE KEY-----\\nprivate-key\\n-----END PRIVATE KEY-----\\n'", + "format": "password", + "title": "Private Key", + "type": "string", + "writeOnly": true + }, + "client_email": { + "description": "Client email", + "title": "Client Email", + "type": "string" + }, + "client_id": { + "description": "Client Id", + "title": "Client Id", + "type": "string" + }, + "auth_uri": { + "default": "https://accounts.google.com/o/oauth2/auth", + "description": "Authentication uri", + "title": "Auth Uri", + "type": "string" + }, + "token_uri": { + "default": "https://oauth2.googleapis.com/token", + "description": "Token uri", + "title": "Token Uri", + "type": "string" + }, + "auth_provider_x509_cert_url": { + "default": "https://www.googleapis.com/oauth2/v1/certs", + "description": "Auth provider x509 certificate url", + "title": "Auth Provider X509 Cert Url", + "type": "string" + }, + "type": { + "default": "service_account", + "description": "Authentication type", + "title": "Type", + "type": "string" + }, + "client_x509_cert_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If not set it will be default to https://www.googleapis.com/robot/v1/metadata/x509/client_email", + "title": "Client X509 Cert Url" + } + }, + "required": [ + "private_key_id", + "private_key", + "client_email", + "client_id" + ], + "title": "GCPCredential", + "type": "object" + }, + "OAuthConfiguration": { + "additionalProperties": false, + "properties": { + "provider": { + "$ref": "#/$defs/OAuthIdentityProvider", + "description": "Identity provider for oauth.Supported providers are microsoft and okta." + }, + "authority_url": { + "description": "Authority url of your identity provider", + "title": "Authority Url", + "type": "string" + }, + "client_id": { + "description": "client id of your registered application", + "title": "Client Id", + "type": "string" + }, + "scopes": { + "description": "scopes required to connect to snowflake", + "items": { + "type": "string" + }, + "title": "Scopes", + "type": "array" + }, + "use_certificate": { + "default": false, + "description": "Do you want to use certificate and private key to authenticate using oauth", + "title": "Use Certificate", + "type": "boolean" + }, + "client_secret": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "client secret of the application if use_certificate = false", + "title": "Client Secret" + }, + "encoded_oauth_public_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "base64 encoded certificate content if use_certificate = true", + "title": "Encoded Oauth Public Key" + }, + "encoded_oauth_private_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "base64 encoded private key content if use_certificate = true", + "title": "Encoded Oauth Private Key" + } + }, + "required": [ + "provider", + "authority_url", + "client_id", + "scopes" + ], + "title": "OAuthConfiguration", + "type": "object" + }, + "OAuthIdentityProvider": { + "enum": [ + "microsoft", + "okta" + ], + "title": "OAuthIdentityProvider", + "type": "string" + }, + "PlatformDetail": { + "additionalProperties": false, + "properties": { + "platform": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Override the platform type detection.", + "title": "Platform" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to", + "title": "Platform Instance" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by DataHub platform ingestion source belong to", + "title": "Env", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The database that all assets produced by this connector belong to. For destinations, this defaults to the fivetran log config's database.", + "title": "Database" + }, + "include_schema_in_urn": { + "default": true, + "description": "Include schema in the dataset URN. In some cases, the schema is not relevant to the dataset URN and Fivetran sets it to the source and destination table names in the connector.", + "title": "Include Schema In Urn", + "type": "boolean" + } + }, + "title": "PlatformDetail", + "type": "object" + }, + "SnowflakeDestinationConfig": { + "additionalProperties": false, + "properties": { + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs.", + "title": "Options", + "type": "object" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Snowflake username.", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Snowflake password.", + "title": "Password" + }, + "private_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Private key in a form of '-----BEGIN PRIVATE KEY-----\\nprivate-key\\n-----END PRIVATE KEY-----\\n' if using key pair authentication. Encrypted version of private key will be in a form of '-----BEGIN ENCRYPTED PRIVATE KEY-----\\nencrypted-private-key\\n-----END ENCRYPTED PRIVATE KEY-----\\n' See: https://docs.snowflake.com/en/user-guide/key-pair-auth.html", + "title": "Private Key" + }, + "private_key_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The path to the private key if using key pair authentication. Ignored if `private_key` is set. See: https://docs.snowflake.com/en/user-guide/key-pair-auth.html", + "title": "Private Key Path" + }, + "private_key_password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Password for your private key. Required if using key pair authentication with encrypted private key.", + "title": "Private Key Password" + }, + "oauth_config": { + "anyOf": [ + { + "$ref": "#/$defs/OAuthConfiguration" + }, + { + "type": "null" + } + ], + "default": null, + "description": "oauth configuration - https://docs.snowflake.com/en/user-guide/python-connector-example.html#connecting-with-oauth" + }, + "authentication_type": { + "default": "DEFAULT_AUTHENTICATOR", + "description": "The type of authenticator to use when connecting to Snowflake. Supports \"DEFAULT_AUTHENTICATOR\", \"OAUTH_AUTHENTICATOR\", \"EXTERNAL_BROWSER_AUTHENTICATOR\" and \"KEY_PAIR_AUTHENTICATOR\".", + "title": "Authentication Type", + "type": "string" + }, + "account_id": { + "description": "Snowflake account identifier. e.g. xy12345, xy12345.us-east-2.aws, xy12345.us-central1.gcp, xy12345.central-us.azure, xy12345.us-west-2.privatelink. Refer [Account Identifiers](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#format-2-legacy-account-locator-in-a-region) for more details.", + "title": "Account Id", + "type": "string" + }, + "warehouse": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Snowflake warehouse.", + "title": "Warehouse" + }, + "role": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Snowflake role.", + "title": "Role" + }, + "connect_args": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Connect args to pass to Snowflake SqlAlchemy driver", + "title": "Connect Args" + }, + "token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "OAuth token from external identity provider. Not recommended for most use cases because it will not be able to refresh once expired.", + "title": "Token" + }, + "snowflake_domain": { + "default": "snowflakecomputing.com", + "description": "Snowflake domain. Use 'snowflakecomputing.com' for most regions or 'snowflakecomputing.cn' for China (cn-northwest-1) region.", + "title": "Snowflake Domain", + "type": "string" + }, + "database": { + "description": "The fivetran connector log database.", + "title": "Database", + "type": "string" + }, + "log_schema": { + "description": "The fivetran connector log schema.", + "title": "Log Schema", + "type": "string" + } + }, + "required": [ + "account_id", + "database", + "log_schema" + ], + "title": "SnowflakeDestinationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fivetran Stateful Ingestion Config." + }, + "fivetran_log_config": { + "$ref": "#/$defs/FivetranLogConfig", + "description": "Fivetran log connector destination server configurations." + }, + "connector_patterns": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Filtering regex patterns for connector names." + }, + "destination_patterns": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for destination ids to filter in ingestion. Fivetran destination IDs are usually two word identifiers e.g. canyon_tolerable, and are not the same as the destination database name. They're visible in the Fivetran UI under Destinations -> Overview -> Destination Group ID." + }, + "include_column_lineage": { + "default": true, + "description": "Populates table->table column lineage.", + "title": "Include Column Lineage", + "type": "boolean" + }, + "sources_to_platform_instance": { + "additionalProperties": { + "$ref": "#/$defs/PlatformDetail" + }, + "default": {}, + "description": "A mapping from connector id to its platform/instance/env/database details.", + "title": "Sources To Platform Instance", + "type": "object" + }, + "destination_to_platform_instance": { + "additionalProperties": { + "$ref": "#/$defs/PlatformDetail" + }, + "default": {}, + "description": "A mapping of destination id to its platform/instance/env details.", + "title": "Destination To Platform Instance", + "type": "object" + }, + "api_config": { + "anyOf": [ + { + "$ref": "#/$defs/FivetranAPIConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fivetran REST API configuration, used to provide wider support for connections." + }, + "history_sync_lookback_period": { + "default": 7, + "description": "The number of days to look back when extracting connectors' sync history.", + "title": "History Sync Lookback Period", + "type": "integer" + } + }, + "required": [ + "fivetran_log_config" + ], + "title": "FivetranSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Database and Schema Name Handling + +The Fivetran source uses **quoted identifiers** for database and schema names to properly handle special characters and case-sensitive names. This follows Snowflake's quoted identifier convention, which is then transpiled to the target database dialect (Snowflake, BigQuery, or Databricks). + +**Important Notes:** + +- **Database names** are automatically wrapped in double quotes (e.g., `use database "my-database"`) +- **Schema names** are automatically wrapped in double quotes (e.g., `"my-schema".table_name`) +- This ensures proper handling of database and schema names containing: + - Hyphens (e.g., `my-database`) + - Spaces (e.g., `my database`) + - Special characters (e.g., `my.database`) + - Case-sensitive names (e.g., `MyDatabase`) + +**Migration Impact:** + +- If you have database or schema names with special characters, they will now be properly quoted in SQL queries +- This change ensures consistent behavior across all supported destination platforms +- No configuration changes are required - the quoting is handled automatically + +**Case Sensitivity Considerations:** + +- **Important**: In Snowflake, unquoted identifiers are automatically converted to uppercase when stored and resolved (e.g., `mydatabase` becomes `MYDATABASE`), while double-quoted identifiers preserve the exact case as entered (e.g., `"mydatabase"` stays as `mydatabase`). See [Snowflake's identifier documentation](https://docs.snowflake.com/en/sql-reference/identifiers-syntax#double-quoted-identifiers) for details. +- **Backward Compatibility**: The system automatically handles backward compatibility for valid unquoted identifiers (identifiers containing only letters, numbers, and underscores). These identifiers are automatically uppercased before quoting to match Snowflake's behavior for unquoted identifiers. This means: + - If your database/schema name is a valid unquoted identifier (e.g., `fivetran_logs`, `MY_SCHEMA`), it will be automatically uppercased to match existing Snowflake objects created without quotes + - No configuration changes are required for standard identifiers (letters, numbers, underscores only) +- **Recommended**: For best practices and to ensure consistency, maintain the exact case of your database and schema names in your configuration to match what's stored in Snowflake + +#### Snowflake destination Configuration Guide + +1. If your fivetran platform connector destination is snowflake, you need to provide user details and its role with correct privileges in order to fetch metadata. +2. Snowflake system admin can follow this guide to create a fivetran_datahub role, assign it the required privileges, and assign it to a user by executing the following Snowflake commands from a user with the ACCOUNTADMIN role or MANAGE GRANTS privilege. + +```sql +create or replace role fivetran_datahub; + +// Grant access to a warehouse to run queries to view metadata +grant operate, usage on warehouse "" to role fivetran_datahub; + +// Grant access to view database and schema in which your log and metadata tables exist +// Note: Database and schema names are automatically quoted, so use quoted identifiers if your names contain special characters +grant usage on DATABASE "" to role fivetran_datahub; +grant usage on SCHEMA ""."" to role fivetran_datahub; + +// Grant access to execute select query on schema in which your log and metadata tables exist +grant select on all tables in SCHEMA ""."" to role fivetran_datahub; + +// Grant the fivetran_datahub to the snowflake user. +grant role fivetran_datahub to user snowflake_user; +``` + +#### Bigquery destination Configuration Guide + +1. If your fivetran platform connector destination is bigquery, you need to setup a ServiceAccount as per [BigQuery docs](https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-console) and select BigQuery Data Viewer and BigQuery Job User IAM roles. +2. Create and Download a service account JSON keyfile and provide bigquery connection credential in bigquery destination config. + +#### Databricks destination Configuration Guide + +1. Get your Databricks instance's [workspace url](https://docs.databricks.com/workspace/workspace-details.html#workspace-instance-names-urls-and-ids) +2. Create a [Databricks Service Principal](https://docs.databricks.com/administration-guide/users-groups/service-principals.html#what-is-a-service-principal) + 1. You can skip this step and use your own account to get things running quickly, but we strongly recommend creating a dedicated service principal for production use. +3. Generate a Databricks Personal Access token following the following guides: + 1. [Service Principals](https://docs.databricks.com/administration-guide/users-groups/service-principals.html#personal-access-tokens) + 2. [Personal Access Tokens](https://docs.databricks.com/dev-tools/auth.html#databricks-personal-access-tokens) +4. Provision your service account, to ingest your workspace's metadata and lineage, your service principal must have all of the following: + 1. One of: metastore admin role, ownership of, or `USE CATALOG` privilege on any catalogs you want to ingest + 2. One of: metastore admin role, ownership of, or `USE SCHEMA` privilege on any schemas you want to ingest + 3. Ownership of or `SELECT` privilege on any tables and views you want to ingest + 4. [Ownership documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/ownership.html) + 5. [Privileges documentation](https://docs.databricks.com/data-governance/unity-catalog/manage-privileges/privileges.html) +5. Check the starter recipe below and replace `workspace_url` and `token` with your information from the previous steps. + +#### Working with Platform Instances + +If you have multiple instances of source/destination systems that are referred in your `fivetran` setup, you'd need to configure platform instance for these systems in `fivetran` recipe to generate correct lineage edges. Refer the document [Working with Platform Instances](/docs/platform-instances) to understand more about this. + +While configuring the platform instance for source system you need to provide connector id as key and for destination system provide destination id as key. +When creating the connection details in the fivetran UI make a note of the destination Group ID of the service account, as that will need to be used in the `destination_to_platform_instance` configuration. +I.e: + +

+ +

+ +In this case the configuration would be something like: + +```yaml +destination_to_platform_instance: + greyish_positive: <--- this comes from bigquery destination - see screenshot + database: + env: PROD +``` + +##### Example - Multiple Postgres Source Connectors each reading from different postgres instance + +```yml +# Map of connector source to platform instance +sources_to_platform_instance: + postgres_connector_id1: + platform_instance: cloud_postgres_instance + env: PROD + + postgres_connector_id2: + platform_instance: local_postgres_instance + env: DEV +``` + +##### Example - Multiple Snowflake Destinations each writing to different snowflake instance + +```yml +# Map of destination to platform instance +destination_to_platform_instance: + snowflake_destination_id1: + platform_instance: prod_snowflake_instance + env: PROD + + snowflake_destination_id2: + platform_instance: dev_snowflake_instance + env: PROD +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +#### Supported Destinations + +Works only for: + +- Snowflake destination +- Bigquery destination +- Databricks destination + +#### Ingestion Limits + +To prevent excessive data ingestion, the following configurable limits apply per connector: + +- **Sync History**: Maximum of 500 sync runs per connector (default: 500, configurable via `fivetran_log_config.max_jobs_per_connector`) +- **Table Lineage**: Maximum of 120 table lineage entries per connector (default: 120, configurable via `fivetran_log_config.max_table_lineage_per_connector`) +- **Column Lineage**: Maximum of 1000 column lineage entries per connector (default: 1000, configurable via `fivetran_log_config.max_column_lineage_per_connector`) + +When these limits are exceeded, only the most recent entries are ingested. Warnings will be logged during ingestion to notify you when truncation occurs. + +These limits act as safety nets to prevent excessive data ingestion. You can increase them cautiously if you need to ingest more historical data or have connectors with many tables/columns. Example configuration: + +```yaml +source: + type: fivetran + config: + fivetran_log_config: + # ... other config ... + max_jobs_per_connector: 1000 # Increase sync history limit + max_table_lineage_per_connector: 500 # Increase table lineage limit + max_column_lineage_per_connector: 5000 # Increase column lineage limit +``` + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.fivetran.fivetran.FivetranSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/fivetran/fivetran.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Fivetran, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/gcs.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/gcs.md new file mode 100644 index 00000000..8a7e7a9d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/gcs.md @@ -0,0 +1,633 @@ +--- +sidebar_position: 33 +title: Google Cloud Storage +slug: /generated/ingestion/sources/gcs +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Google Cloud Storage + +## Overview + +Google Cloud Storage is a storage and lakehouse platform. Learn more in the [official Google Cloud Storage documentation](https://cloud.google.com/storage). + +The DataHub integration for Google Cloud Storage covers file/lakehouse metadata entities such as datasets, paths, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| ------------------------------------------ | ----------------------------------------------------------------------------------------- | -------------------- | +| `"Google Cloud Storage"` | [Data Platform](/docs/generated/metamodel/entities/dataplatform/) | | +| GCS object / Folder containing GCS objects | [Dataset](/docs/generated/metamodel/entities/dataset/) | | +| GCS bucket | [Container](/docs/generated/metamodel/entities/container/) | Subtype `GCS bucket` | +| GCS folder | [Container](/docs/generated/metamodel/entities/container/) | Subtype `Folder` | + + +## Module `gcs` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - GCS bucket, Folder. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ❌ | Not supported. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Schema Metadata | ✅ | Enabled by default. | + +### Overview + +The `gcs` module ingests metadata from Gcs into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This connector ingests Google Cloud Storage datasets into DataHub. It allows mapping an individual file or a folder of files to a dataset in DataHub. +To specify the group of files that form a dataset, use `path_specs` configuration in ingestion recipe. This source leverages [Interoperability of GCS with S3](https://cloud.google.com/storage/docs/interoperability) +and uses DataHub S3 Data Lake integration source under the hood. Refer section [Path Specs](/docs/generated/ingestion/sources/s3/#path-specs) from S3 connector for more details. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +1. Create a service account with "Storage Object Viewer" Role - https://cloud.google.com/iam/docs/service-accounts-create +2. Make sure you meet following requirements to generate HMAC key - https://cloud.google.com/storage/docs/authentication/managing-hmackeys#before-you-begin +3. Create an HMAC key for service account created above - https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create . + + +### Install the Plugin +```shell +pip install 'acryl-datahub[gcs]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: gcs + config: + path_specs: + - include: gs://gcs-ingestion-bucket/parquet_example/{table}/year={partition[0]}/*.parquet + credential: + hmac_access_id: + hmac_access_secret: + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
credential ✅
HMACKey
| | +|
credential.hmac_access_id ❓
string
| Access ID | +|
credential.hmac_access_secret ❓
string(password)
| Secret | +|
path_specs ✅
array
| List of PathSpec. See [below](#path-spec) the details about PathSpec | +|
path_specs.PathSpec
PathSpec
| | +|
path_specs.PathSpec.include ❓
string
| Path to table. Name variable `{table}` is used to mark the folder with dataset. In absence of `{table}`, file level dataset will be created. Check below examples for more details. | +|
path_specs.PathSpec.allow_double_stars
boolean
| Allow double stars in the include path. This can affect performance significantly if enabled
Default: False
| +|
path_specs.PathSpec.autodetect_partitions
boolean
| Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`
Default: True
| +|
path_specs.PathSpec.default_extension
One of string, null
| For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.
Default: None
| +|
path_specs.PathSpec.enable_compression
boolean
| Enable or disable processing compressed files. Currently .gz and .bz files are supported.
Default: True
| +|
path_specs.PathSpec.include_hidden_folders
boolean
| Include hidden folders in the traversal (folders starting with . or _
Default: False
| +|
path_specs.PathSpec.sample_files
boolean
| Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled
Default: True
| +|
path_specs.PathSpec.table_name
One of string, null
| Display name of the dataset.Combination of named variables from include path and strings
Default: None
| +|
path_specs.PathSpec.traversal_method
Enum
| One of: "ALL", "MIN_MAX", "MAX" | +|
path_specs.PathSpec.exclude
One of array, null
| list of paths in glob pattern which will be excluded while scanning for the datasets
Default: []
| +|
path_specs.PathSpec.exclude.string
string
| | +|
path_specs.PathSpec.file_types
array
| Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.
Default: ['csv', 'tsv', 'json', 'parquet', 'avro']
| +|
path_specs.PathSpec.file_types.string
string
| | +|
path_specs.PathSpec.tables_filter_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
path_specs.PathSpec.tables_filter_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
max_rows
integer
| Maximum number of rows to use when inferring schemas for TSV and CSV files.
Default: 100
| +|
number_of_files_to_sample
integer
| Number of files to list to sample for schema inference. This will be ignored if sample_files is set to False in the pathspec.
Default: 100
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "FolderTraversalMethod": { + "enum": [ + "ALL", + "MIN_MAX", + "MAX" + ], + "title": "FolderTraversalMethod", + "type": "string" + }, + "HMACKey": { + "additionalProperties": false, + "properties": { + "hmac_access_id": { + "description": "Access ID", + "title": "Hmac Access Id", + "type": "string" + }, + "hmac_access_secret": { + "description": "Secret", + "format": "password", + "title": "Hmac Access Secret", + "type": "string", + "writeOnly": true + } + }, + "required": [ + "hmac_access_id", + "hmac_access_secret" + ], + "title": "HMACKey", + "type": "object" + }, + "PathSpec": { + "additionalProperties": false, + "properties": { + "include": { + "description": "Path to table. Name variable `{table}` is used to mark the folder with dataset. In absence of `{table}`, file level dataset will be created. Check below examples for more details.", + "title": "Include", + "type": "string" + }, + "exclude": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": [], + "description": "list of paths in glob pattern which will be excluded while scanning for the datasets", + "title": "Exclude" + }, + "file_types": { + "default": [ + "csv", + "tsv", + "json", + "parquet", + "avro" + ], + "description": "Files with extenstions specified here (subset of default value) only will be scanned to create dataset. Other files will be omitted.", + "items": { + "type": "string" + }, + "title": "File Types", + "type": "array" + }, + "default_extension": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "For files without extension it will assume the specified file type. If it is not set the files without extensions will be skipped.", + "title": "Default Extension" + }, + "table_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Display name of the dataset.Combination of named variables from include path and strings", + "title": "Table Name" + }, + "enable_compression": { + "default": true, + "description": "Enable or disable processing compressed files. Currently .gz and .bz files are supported.", + "title": "Enable Compression", + "type": "boolean" + }, + "sample_files": { + "default": true, + "description": "Not listing all the files but only taking a handful amount of sample file to infer the schema. File count and file size calculation will be disabled. This can affect performance significantly if enabled", + "title": "Sample Files", + "type": "boolean" + }, + "allow_double_stars": { + "default": false, + "description": "Allow double stars in the include path. This can affect performance significantly if enabled", + "title": "Allow Double Stars", + "type": "boolean" + }, + "autodetect_partitions": { + "default": true, + "description": "Autodetect partition(s) from the path. If set to true, it will autodetect partition key/value if the folder format is {partition_key}={partition_value} for example `year=2024`", + "title": "Autodetect Partitions", + "type": "boolean" + }, + "traversal_method": { + "$ref": "#/$defs/FolderTraversalMethod", + "default": "MAX", + "description": "Method to traverse the folder. ALL: Traverse all the folders, MIN_MAX: Traverse the folders by finding min and max value, MAX: Traverse the folder with max value" + }, + "include_hidden_folders": { + "default": false, + "description": "Include hidden folders in the traversal (folders starting with . or _", + "title": "Include Hidden Folders", + "type": "boolean" + }, + "tables_filter_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "The tables_filter_pattern configuration field uses regular expressions to filter the tables part of the Pathspec for ingestion, allowing fine-grained control over which tables are included or excluded based on specified patterns. The default setting allows all tables." + } + }, + "required": [ + "include" + ], + "title": "PathSpec", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "path_specs": { + "description": "List of PathSpec. See [below](#path-spec) the details about PathSpec", + "items": { + "$ref": "#/$defs/PathSpec" + }, + "title": "Path Specs", + "type": "array" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "credential": { + "$ref": "#/$defs/HMACKey", + "description": "Google cloud storage [HMAC keys](https://cloud.google.com/storage/docs/authentication/hmackeys)" + }, + "max_rows": { + "default": 100, + "description": "Maximum number of rows to use when inferring schemas for TSV and CSV files.", + "title": "Max Rows", + "type": "integer" + }, + "number_of_files_to_sample": { + "default": 100, + "description": "Number of files to list to sample for schema inference. This will be ignored if sample_files is set to False in the pathspec.", + "title": "Number Of Files To Sample", + "type": "integer" + } + }, + "required": [ + "path_specs", + "credential" + ], + "title": "GCSSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Path Specs + +**Example - Dataset per file** + +Bucket structure: + +``` +test-gs-bucket +├── employees.csv +└── food_items.csv +``` + +Path specs config + +``` +path_specs: + - include: gs://test-gs-bucket/*.csv + +``` + +**Example - Datasets with partitions** + +Bucket structure: + +``` +test-gs-bucket +├── orders +│   └── year=2022 +│   └── month=2 +│   ├── 1.parquet +│   └── 2.parquet +└── returns + └── year=2021 + └── month=2 + └── 1.parquet + +``` + +Path specs config: + +``` +path_specs: + - include: gs://test-gs-bucket/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet +``` + +**Example - Datasets with partition and exclude** + +Bucket structure: + +``` +test-gs-bucket +├── orders +│   └── year=2022 +│   └── month=2 +│   ├── 1.parquet +│   └── 2.parquet +└── tmp_orders + └── year=2021 + └── month=2 + └── 1.parquet + + +``` + +Path specs config: + +``` +path_specs: + - include: gs://test-gs-bucket/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet + exclude: + - **/tmp_orders/** +``` + +**Example - Datasets of mixed nature** + +Bucket structure: + +``` +test-gs-bucket +├── customers +│   ├── part1.json +│   ├── part2.json +│   ├── part3.json +│   └── part4.json +├── employees.csv +├── food_items.csv +├── tmp_10101000.csv +└── orders +    └── year=2022 +    └── month=2 +   ├── 1.parquet +   ├── 2.parquet +   └── 3.parquet + +``` + +Path specs config: + +``` +path_specs: + - include: gs://test-gs-bucket/*.csv + exclude: + - **/tmp_10101000.csv + - include: gs://test-gs-bucket/{table}/*.json + - include: gs://test-gs-bucket/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.parquet +``` + +**Valid path_specs.include** + +```python +gs://my-bucket/foo/tests/bar.avro # single file table +gs://my-bucket/foo/tests/*.* # mulitple file level tables +gs://my-bucket/foo/tests/{table}/*.avro #table without partition +gs://my-bucket/foo/tests/{table}/*/*.avro #table where partitions are not specified +gs://my-bucket/foo/tests/{table}/*.* # table where no partitions as well as data type specified +gs://my-bucket/{dept}/tests/{table}/*.avro # specifying keywords to be used in display name +gs://my-bucket/{dept}/tests/{table}/{partition_key[0]}={partition[0]}/{partition_key[1]}={partition[1]}/*.avro # specify partition key and value format +gs://my-bucket/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.avro # specify partition value only format +gs://my-bucket/{dept}/tests/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # for all extensions +gs://my-bucket/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # table is present at 2 levels down in bucket +gs://my-bucket/*/*/{table}/{partition[0]}/{partition[1]}/{partition[2]}/*.* # table is present at 3 levels down in bucket +``` + +**Valid path_specs.exclude** + +- \*\*/tests/\*\* +- gs://my-bucket/hr/\*\* +- \*_/tests/_.csv +- gs://my-bucket/foo/\*/my_table/\*\* + +**Notes** + +- `{table}` represents folder for which dataset will be created. +- include path must end with (`*.*` or `*.[ext]`) to represent leaf level. +- if `*.[ext]` is provided then only files with specified type will be scanned. +- `/*/ `represents single folder. +- `{partition[i]}` represents value of partition. +- `{partition_key[i]}` represents name of the partition. +- While extracting, "i" will be used to match partition_key to partition. +- all folder levels need to be specified in include. Only exclude path can have `**` like matching. +- exclude path cannot have named variables ( {} ). +- Folder names should not contain {, }, \*, / in their names. +- {folder} is reserved for internal working. please do not use in named variables. + +If you would like to write a more complicated function for resolving file names, then a {transformer} would be a good fit. + +:::caution + +Specify as long fixed prefix ( with out /\*/ ) as possible in `path_specs.include`. This will reduce the scanning time and cost, specifically on Google Cloud Storage. + +::: + +:::caution + +If you are ingesting datasets from Google Cloud Storage, we recommend running the ingestion on a server in the same region to avoid high egress costs. + +::: + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +#### Supported file types + +Supported file types are as follows: + +- CSV +- TSV +- JSONL +- JSON +- Parquet +- Apache Avro + +Schemas for Parquet and Avro files are extracted as provided. + +Schemas for schemaless formats (CSV, TSV, JSONL, JSON) are inferred. For CSV, TSV and JSONL files, we consider the first 100 rows by default, which can be controlled via the `max_rows` recipe parameter (see [below](#config-details)) +JSON file schemas are inferred on the basis of the entire file (given the difficulty in extracting only the first few objects of the file), which may impact performance. +We are working on using iterator-based JSON parsers to avoid reading in the entire JSON object. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.gcs.gcs_source.GCSSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/gcs/gcs_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Google Cloud Storage, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/glue.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/glue.md new file mode 100644 index 00000000..dfe35065 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/glue.md @@ -0,0 +1,960 @@ +--- +sidebar_position: 32 +title: Glue +slug: /generated/ingestion/sources/glue +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Glue + +## Overview + +Glue is a data platform used to store and query analytical or operational data. Learn more in the [official Glue documentation](https://aws.amazon.com/glue/). + +The DataHub integration for Glue covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +:::tip +If you also have files in S3 that you'd like to ingest, we recommend you use Glue's built-in data catalog. See here for a quick guide on how to set up a crawler on Glue and ingest the outputs with DataHub. +::: + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| -------------------- | --------------------------------------------------------- | ------------------ | +| `"glue"` | [Data Platform](../../metamodel/entities/dataPlatform.md) | | +| Glue Database | [Container](../../metamodel/entities/container.md) | Subtype `Database` | +| Glue Table | [Dataset](../../metamodel/entities/dataset.md) | Subtype `Table` | +| Glue Job | [Data Flow](../../metamodel/entities/dataFlow.md) | | +| Glue Job Transform | [Data Job](../../metamodel/entities/dataJob.md) | | +| Glue Job Data source | [Dataset](../../metamodel/entities/dataset.md) | | +| Glue Job Data sink | [Dataset](../../metamodel/entities/dataset.md) | | + + +## Module `glue` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database. | +| Column-level Lineage | ✅ | Support via the `emit_s3_lineage` config field. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default. | + +### Overview + +The `glue` module ingests metadata from Glue into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin extracts the following: + +- Tables in the Glue catalog +- Column types associated with each table +- Table metadata, such as owner, description and parameters +- Jobs and their component transformations, data sources, and data sinks + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +#### IAM permissions + +For ingesting datasets, the following IAM permissions are required: + +``` +{ + "Effect": "Allow", + "Action": [ + "glue:GetDatabases", + "glue:GetTables" + ], + "Resource": [ + "arn:aws:glue:$region-id:$account-id:catalog", + "arn:aws:glue:$region-id:$account-id:database/*", + "arn:aws:glue:$region-id:$account-id:table/*" + ] +} +``` + +For ingesting jobs (extract_transforms: True), the following additional permissions are required: + +``` +{ + "Effect": "Allow", + "Action": [ + "glue:GetDataflowGraph", + "glue:GetJobs", + "s3:GetObject", + ], + "Resource": "*" +} +``` + +For profiling datasets, the following additional permissions are required: + +``` + { + "Effect": "Allow", + "Action": [ + "glue:GetPartitions", + ], + "Resource": "*" +} +``` + +#### Glue Cross-account Access + +Glue ingestion supports cross-account access and lineage by allowing you to specify the target AWS account's Glue catalog using the `catalog_id` parameter in the ingestion recipe. This enables ingestion of Glue metadata from different AWS accounts, supporting cross-account lineage scenarios. You must ensure the correct IAM roles and permissions are set up for cross-account access. + +Example: There are 2 AWS accounts A and B, A has shared metadata with B. Account A has Glue table - tableA. If you ingest account A using Glue it will create dataset tableA in DataHub. If you want to ingest tableA via account B you can pass `catalog_id` parameter in recipe with A's catalog id. + +Ingestion without platform instance parameter + +- If both catalogs are ingested without platform instance parameter, DataHub should be able to understand that the database and tables are same +- DataHub will create single entity for table tableA +- It should show lineage between Glue and S3. You have to ingest S3 as separate source (https://docs.datahub.com/docs/generated/ingestion/sources/s3) + +- Ingestion with platform instance parameter + +- It will create separate entities for tableA as it will have different URN path +- It should show lineage between Glue and S3 + + +### Install the Plugin +```shell +pip install 'acryl-datahub[glue]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: glue + config: + # Coordinates + aws_region: "my-aws-region" + +sink: + # sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
aws_access_key_id
One of string, null
| AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_advanced_config
object
| Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html). | +|
aws_endpoint_url
One of string, null
| The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.
Default: None
| +|
aws_profile
One of string, null
| The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.
Default: None
| +|
aws_proxy
One of string, null
| A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.
Default: None
| +|
aws_region
One of string, null
| AWS region code.
Default: None
| +|
aws_retry_mode
Enum
| One of: "legacy", "standard", "adaptive"
Default: standard
| +|
aws_retry_num
integer
| Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.
Default: 5
| +|
aws_secret_access_key
One of string(password), null
| AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
aws_session_token
One of string(password), null
| AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.
Default: None
| +|
catalog_id
One of string, null
| The aws account id where the target glue catalog lives. If None, datahub will ingest glue in aws caller's account.
Default: None
| +|
emit_s3_lineage
boolean
| Whether to emit S3-to-Glue lineage.
Default: False
| +|
extract_delta_schema_from_parameters
One of boolean, null
| If enabled, delta schemas can be alternatively fetched from table parameters.
Default: False
| +|
extract_lakeformation_tags
One of boolean, null
| When True, extracts Lake Formation tags directly assigned to Glue tables/databases. Note: Tags inherited from databases or other parent resources are excluded.
Default: False
| +|
extract_owners
One of boolean, null
| When enabled, extracts ownership from Glue table property and overwrites existing owners (DATAOWNER). When disabled, ownership is left empty for datasets. Expects a corpGroup urn, a corpuser urn or only the identifier part for the latter. Not used in the normal course of AWS Glue operations.
Default: True
| +|
extract_transforms
One of boolean, null
| Whether to extract Glue transform jobs.
Default: True
| +|
glue_s3_lineage_direction
string
| If `upstream`, S3 is upstream to Glue. If `downstream` S3 is downstream to Glue.
Default: upstream
| +|
ignore_resource_links
One of boolean, null
| If set to True, ignore database resource links.
Default: False
| +|
ignore_unsupported_connectors
One of boolean, null
| Whether to ignore unsupported connectors. If disabled, an error will be raised.
Default: True
| +|
include_column_lineage
boolean
| When enabled, column-level lineage will be extracted from the s3.
Default: True
| +|
platform
string
| The platform to use for the dataset URNs. Must be one of ['glue', 'athena'].
Default: glue
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
read_timeout
number
| The timeout for reading from the connection (in seconds).
Default: 60
| +|
use_s3_bucket_tags
One of boolean, null
| If an S3 Buckets Tags should be created for the Tables ingested by Glue. Please Note that this will not apply tags to any folders ingested, only the files.
Default: False
| +|
use_s3_object_tags
One of boolean, null
| If an S3 Objects Tags should be created for the Tables ingested by Glue.
Default: False
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
aws_role
One of string, array, null
| AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).
Default: None
| +|
aws_role.union
One of string, AwsAssumeRoleConfig
| | +|
aws_role.union.RoleArn ❓
string
| ARN of the role to assume. | +|
aws_role.union.ExternalId
One of string, null
| External ID to use when assuming the role.
Default: None
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GlueProfilingConfig
| | +|
profiling.column_count
One of string, null
| The parameter name for column count in glue table.
Default: None
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.max
One of string, null
| The parameter name for the max value of a column.
Default: None
| +|
profiling.mean
One of string, null
| The parameter name for the mean value of a column.
Default: None
| +|
profiling.median
One of string, null
| The parameter name for the median value of a column.
Default: None
| +|
profiling.min
One of string, null
| The parameter name for the min value of a column.
Default: None
| +|
profiling.null_count
One of string, null
| The parameter name for the count of null values in a column.
Default: None
| +|
profiling.null_proportion
One of string, null
| The parameter name for the proportion of null values in a column.
Default: None
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.row_count
One of string, null
| The parameter name for row count in glue table.
Default: None
| +|
profiling.stdev
One of string, null
| The parameter name for the standard deviation of a column.
Default: None
| +|
profiling.unique_count
One of string, null
| The parameter name for the count of unique value in a column.
Default: None
| +|
profiling.unique_proportion
One of string, null
| The parameter name for the proportion of unique values in a column.
Default: None
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.partition_patterns
AllowDenyPattern
| A class to store allow deny regexes | +|
profiling.partition_patterns.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling.partition_patterns.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
profiling.partition_patterns.allow.string
string
| | +|
profiling.partition_patterns.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
profiling.partition_patterns.deny.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "AwsAssumeRoleConfig": { + "additionalProperties": true, + "properties": { + "RoleArn": { + "description": "ARN of the role to assume.", + "title": "Rolearn", + "type": "string" + }, + "ExternalId": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "External ID to use when assuming the role.", + "title": "Externalid" + } + }, + "required": [ + "RoleArn" + ], + "title": "AwsAssumeRoleConfig", + "type": "object" + }, + "GlueProfilingConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "row_count": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for row count in glue table.", + "title": "Row Count" + }, + "column_count": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for column count in glue table.", + "title": "Column Count" + }, + "unique_count": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the count of unique value in a column.", + "title": "Unique Count" + }, + "unique_proportion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the proportion of unique values in a column.", + "title": "Unique Proportion" + }, + "null_count": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the count of null values in a column.", + "title": "Null Count" + }, + "null_proportion": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the proportion of null values in a column.", + "title": "Null Proportion" + }, + "min": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the min value of a column.", + "title": "Min" + }, + "max": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the max value of a column.", + "title": "Max" + }, + "mean": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the mean value of a column.", + "title": "Mean" + }, + "median": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the median value of a column.", + "title": "Median" + }, + "stdev": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The parameter name for the standard deviation of a column.", + "title": "Stdev" + }, + "partition_patterns": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for filtering partitions for profile. The pattern should be a string like: \"{'key':'value'}\"." + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + } + }, + "title": "GlueProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "aws_access_key_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS access key ID. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Access Key Id" + }, + "aws_secret_access_key": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS secret access key. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Secret Access Key" + }, + "aws_session_token": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS session token. Can be auto-detected, see [the AWS boto3 docs](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) for details.", + "title": "Aws Session Token" + }, + "aws_role": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/AwsAssumeRoleConfig" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS roles to assume. If using the string format, the role ARN can be specified directly. If using the object format, the role can be specified in the RoleArn field and additional available arguments are the same as [boto3's STS.Client.assume_role](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html?highlight=assume_role#STS.Client.assume_role).", + "title": "Aws Role" + }, + "aws_profile": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The [named profile](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html) to use from AWS credentials. Falls back to default profile if not specified and no access keys provided. Profiles are configured in ~/.aws/credentials or ~/.aws/config.", + "title": "Aws Profile" + }, + "aws_region": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "AWS region code.", + "title": "Aws Region" + }, + "aws_endpoint_url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The AWS service endpoint. This is normally [constructed automatically](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html), but can be overridden here.", + "title": "Aws Endpoint Url" + }, + "aws_proxy": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A set of proxy configs to use with AWS. See the [botocore.config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html) docs for details.", + "title": "Aws Proxy" + }, + "aws_retry_num": { + "default": 5, + "description": "Number of times to retry failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "title": "Aws Retry Num", + "type": "integer" + }, + "aws_retry_mode": { + "default": "standard", + "description": "Retry mode to use for failed AWS requests. See the [botocore.retry](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/retries.html) docs for details.", + "enum": [ + "legacy", + "standard", + "adaptive" + ], + "title": "Aws Retry Mode", + "type": "string" + }, + "read_timeout": { + "default": 60, + "description": "The timeout for reading from the connection (in seconds).", + "title": "Read Timeout", + "type": "number" + }, + "aws_advanced_config": { + "additionalProperties": true, + "description": "Advanced AWS configuration options. These are passed directly to [botocore.config.Config](https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html).", + "title": "Aws Advanced Config", + "type": "object" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for databases to filter in ingestion." + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for tables to filter in ingestion." + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "" + }, + "platform": { + "default": "glue", + "description": "The platform to use for the dataset URNs. Must be one of ['glue', 'athena'].", + "title": "Platform", + "type": "string" + }, + "extract_owners": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "When enabled, extracts ownership from Glue table property and overwrites existing owners (DATAOWNER). When disabled, ownership is left empty for datasets. Expects a corpGroup urn, a corpuser urn or only the identifier part for the latter. Not used in the normal course of AWS Glue operations.", + "title": "Extract Owners" + }, + "extract_transforms": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to extract Glue transform jobs.", + "title": "Extract Transforms" + }, + "ignore_unsupported_connectors": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore unsupported connectors. If disabled, an error will be raised.", + "title": "Ignore Unsupported Connectors" + }, + "emit_s3_lineage": { + "default": false, + "description": "Whether to emit S3-to-Glue lineage.", + "title": "Emit S3 Lineage", + "type": "boolean" + }, + "glue_s3_lineage_direction": { + "default": "upstream", + "description": "If `upstream`, S3 is upstream to Glue. If `downstream` S3 is downstream to Glue.", + "title": "Glue S3 Lineage Direction", + "type": "string" + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "regex patterns for tables to filter to assign domain_key. ", + "title": "Domain", + "type": "object" + }, + "catalog_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The aws account id where the target glue catalog lives. If None, datahub will ingest glue in aws caller's account.", + "title": "Catalog Id" + }, + "ignore_resource_links": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If set to True, ignore database resource links.", + "title": "Ignore Resource Links" + }, + "use_s3_bucket_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If an S3 Buckets Tags should be created for the Tables ingested by Glue. Please Note that this will not apply tags to any folders ingested, only the files.", + "title": "Use S3 Bucket Tags" + }, + "use_s3_object_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If an S3 Objects Tags should be created for the Tables ingested by Glue.", + "title": "Use S3 Object Tags" + }, + "extract_lakeformation_tags": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "When True, extracts Lake Formation tags directly assigned to Glue tables/databases. Note: Tags inherited from databases or other parent resources are excluded.", + "title": "Extract Lakeformation Tags" + }, + "profiling": { + "$ref": "#/$defs/GlueProfilingConfig", + "description": "Configs to ingest data profiles from glue table" + }, + "extract_delta_schema_from_parameters": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If enabled, delta schemas can be alternatively fetched from table parameters.", + "title": "Extract Delta Schema From Parameters" + }, + "include_column_lineage": { + "default": true, + "description": "When enabled, column-level lineage will be extracted from the s3.", + "title": "Include Column Lineage", + "type": "boolean" + } + }, + "title": "GlueSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Compatibility + +To capture lineage across Glue jobs and databases, a requirements must be met – otherwise the AWS API is unable to report any lineage. The job must be created in Glue Studio with the "Generate classic script" option turned on (this option can be accessed in the "Script" tab). Any custom scripts that do not have the proper annotations will not have reported lineage. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.aws.glue.GlueSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Glue, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/grafana.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/grafana.md new file mode 100644 index 00000000..2e59fd5a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/grafana.md @@ -0,0 +1,600 @@ +--- +sidebar_position: 34 +title: Grafana +slug: /generated/ingestion/sources/grafana +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Grafana + +## Overview + +Grafana is a business intelligence and analytics platform. Learn more in the [official Grafana documentation](https://grafana.com/). + +The DataHub integration for Grafana covers BI entities such as dashboards, charts, datasets, and related ownership context. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| --------------------------- | --------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `"grafana"` | [Data Platform](../../metamodel/entities/dataPlatform.md) | | +| Grafana Folder | [Container](../../metamodel/entities/container.md) | Subtype `Folder` | +| Grafana Dashboard | [Container](../../metamodel/entities/container.md) | Subtype `Dashboard` | +| Grafana Panel/Visualization | [Chart](../../metamodel/entities/chart.md) | Various types mapped based on panel type (e.g., graph → LINE, pie → PIE) | +| Grafana Data Source | [Dataset](../../metamodel/entities/dataset.md) | Created for each panel's data source | +| Dashboard Owner | [Corp User](../../metamodel/entities/corpuser.md) | Dashboard creator assigned as TECHNICAL_OWNER; email suffix removal configurable via `remove_email_suffix` | +| Dashboard Tags | [Tag](../../metamodel/entities/tag.md) | Supports both simple tags and key:value tags | + + +## Module `grafana` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Column-level Lineage | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default. | +| Extract Ownership | ✅ | Enabled by default. | +| Extract Tags | ✅ | Enabled by default. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default. | + +### Overview + +The `grafana` module ingests metadata from Grafana into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +#### Compatibility + +Supports any Grafana instance accessible via API. Extracts column-level lineage from parseable SQL queries in data sources. + +For optimal SQL lineage extraction: + +- Configure database/schema information in data source connection settings +- Set `connection_to_platform_map` to match your data sources + +#### Extracted Metadata Scope + +The connector extracts metadata from Grafana APIs with support for: + +- Folder and dashboard container hierarchy +- Panel and visualization entities (chart modeling) +- Data source references for dataset linking +- Dashboard ownership and tags +- Optional table/column lineage from parseable SQL-based panels + +### Prerequisites + +The Grafana source supports two extraction modes based on your permission level: + +#### Enhanced Mode (Default) + +For full metadata extraction including lineage, containers, and detailed panel information: + +1. A running Grafana instance +2. A service account token with **Admin permissions** to: + - Read dashboards and folders + - Access data source configurations + - View user information + - Access detailed dashboard metadata + - Read panel configurations and transformations + +#### Basic Mode (Limited Permissions) + +For users with limited permissions who only need basic dashboard metadata: + +1. A running Grafana instance +2. A service account token with **Viewer permissions** to: + - Read dashboards (via `/api/search` endpoint) + - Basic dashboard metadata access + +To enable basic mode, set `basic_mode: true` in your configuration. This provides backwards compatibility with the original simple connector behavior. + +**Note:** Basic mode extracts only dashboard entities without folder hierarchy, panel details, lineage information, or schema metadata. It's recommended to use enhanced mode when possible for complete metadata extraction. + +#### Configuration Examples + +Enhanced Mode (Default): + +```yaml +source: + type: grafana + config: + url: "https://grafana.company.com" + service_account_token: "your_admin_token" + # basic_mode: false # Default - full extraction +``` + +Basic Mode (Limited Permissions): + +```yaml +source: + type: grafana + config: + url: "https://grafana.company.com" + service_account_token: "your_viewer_token" + basic_mode: true # Enable basic mode for limited permissions +``` + + +### Install the Plugin +```shell +pip install 'acryl-datahub[grafana]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: grafana + config: + # Coordinates + platform_instance: production # optional + env: PROD # optional + url: https://grafana.company.com + service_account_token: ${GRAFANA_SERVICE_ACCOUNT_TOKEN} + + # SSL verification for HTTPS connections + verify_ssl: true # optional, default is true + + # Ownership configuration + ingest_owners: true # optional, default is true - extract dashboard ownership + remove_email_suffix: true # optional, default is true - remove email suffix like @acryl.io + + # Source type mapping for lineage + connection_to_platform_map: + postgres: + platform: postgres + database: grafana # optional + database_schema: grafana # optional + platform_instance: database_2 # optional + env: PROD # optional + mysql_uid_1: # Grafana datasource UID + platform: mysql + platform_instance: database_1 # optional + database: my_database # optional +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
service_account_token ✅
string(password)
| Service account token for Grafana | +|
url ✅
string
| Grafana URL in the format http://your-grafana-instance with no trailing slash | +|
basic_mode
boolean
| Enable basic extraction mode for users with limited permissions. In basic mode, only dashboard metadata is extracted without detailed panel information, lineage, or folder hierarchy. This requires only basic dashboard read permissions.
Default: False
| +|
include_column_lineage
boolean
| Whether to extract column-level lineage from SQL queries. Only applicable when include_lineage is enabled.
Default: True
| +|
include_lineage
boolean
| Whether to extract lineage between charts and data sources. When enabled, the source will parse SQL queries and datasource configurations to build lineage relationships.
Default: True
| +|
ingest_owners
boolean
| Whether to ingest dashboard ownership information
Default: True
| +|
ingest_tags
boolean
| Whether to ingest dashboard and chart tags
Default: True
| +|
page_size
integer
| Number of items to fetch per API call when paginating through folders and dashboards
Default: 100
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
platform_instance_map
One of string, null
| A holder for platform -> platform_instance mappings to generate correct dataset urns
Default: None
| +|
remove_email_suffix
boolean
| Remove Grafana user email suffix for example, @acryl.io, when assigning ownership.
Default: True
| +|
skip_text_panels
boolean
| Whether to skip text panels during ingestion. Text panels don't contain data visualizations and may not be relevant for data lineage.
Default: False
| +|
verify_ssl
boolean
| Whether to verify SSL certificates when connecting to Grafana
Default: True
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
connection_to_platform_map
map(str,PlatformConnectionConfig)
| Platform connection configuration for mapping Grafana datasources to their actual platforms. | +|
connection_to_platform_map.`key`.platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
connection_to_platform_map.`key`.platform ❓
string
| The platform name (e.g., 'postgres', 'mysql', 'snowflake') | +|
connection_to_platform_map.`key`.database
One of string, null
| Default database name
Default: None
| +|
connection_to_platform_map.`key`.database_schema
One of string, null
| Default schema name
Default: None
| +|
connection_to_platform_map.`key`.env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
dashboard_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
dashboard_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
folder_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
folder_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Stateful ingestion configuration
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "PlatformConnectionConfig": { + "additionalProperties": false, + "description": "Platform connection configuration for mapping Grafana datasources to their actual platforms.", + "properties": { + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform": { + "description": "The platform name (e.g., 'postgres', 'mysql', 'snowflake')", + "title": "Platform", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Default database name", + "title": "Database" + }, + "database_schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Default schema name", + "title": "Database Schema" + } + }, + "required": [ + "platform" + ], + "title": "PlatformConnectionConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Configuration for Grafana source", + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Stateful ingestion configuration" + }, + "platform_instance_map": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A holder for platform -> platform_instance mappings to generate correct dataset urns", + "title": "Platform Instance Map" + }, + "url": { + "description": "Grafana URL in the format http://your-grafana-instance with no trailing slash", + "title": "Url", + "type": "string" + }, + "service_account_token": { + "description": "Service account token for Grafana", + "format": "password", + "title": "Service Account Token", + "type": "string", + "writeOnly": true + }, + "verify_ssl": { + "default": true, + "description": "Whether to verify SSL certificates when connecting to Grafana", + "title": "Verify Ssl", + "type": "boolean" + }, + "page_size": { + "default": 100, + "description": "Number of items to fetch per API call when paginating through folders and dashboards", + "title": "Page Size", + "type": "integer" + }, + "basic_mode": { + "default": false, + "description": "Enable basic extraction mode for users with limited permissions. In basic mode, only dashboard metadata is extracted without detailed panel information, lineage, or folder hierarchy. This requires only basic dashboard read permissions.", + "title": "Basic Mode", + "type": "boolean" + }, + "dashboard_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex pattern to filter dashboards for ingestion" + }, + "folder_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex pattern to filter folders for ingestion" + }, + "ingest_tags": { + "default": true, + "description": "Whether to ingest dashboard and chart tags", + "title": "Ingest Tags", + "type": "boolean" + }, + "ingest_owners": { + "default": true, + "description": "Whether to ingest dashboard ownership information", + "title": "Ingest Owners", + "type": "boolean" + }, + "remove_email_suffix": { + "default": true, + "description": "Remove Grafana user email suffix for example, @acryl.io, when assigning ownership.", + "title": "Remove Email Suffix", + "type": "boolean" + }, + "skip_text_panels": { + "default": false, + "description": "Whether to skip text panels during ingestion. Text panels don't contain data visualizations and may not be relevant for data lineage.", + "title": "Skip Text Panels", + "type": "boolean" + }, + "include_lineage": { + "default": true, + "description": "Whether to extract lineage between charts and data sources. When enabled, the source will parse SQL queries and datasource configurations to build lineage relationships.", + "title": "Include Lineage", + "type": "boolean" + }, + "include_column_lineage": { + "default": true, + "description": "Whether to extract column-level lineage from SQL queries. Only applicable when include_lineage is enabled.", + "title": "Include Column Lineage", + "type": "boolean" + }, + "connection_to_platform_map": { + "additionalProperties": { + "$ref": "#/$defs/PlatformConnectionConfig" + }, + "description": "Map of Grafana datasource types/UIDs to platform connection configs for lineage extraction", + "title": "Connection To Platform Map", + "type": "object" + } + }, + "required": [ + "url", + "service_account_token" + ], + "title": "GrafanaSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Lineage + +The Grafana source can extract lineage information between charts and their data sources. You can control lineage extraction using these configuration options: + +```yaml +source: + type: grafana + config: + url: "https://grafana.company.com" + service_account_token: "your_token" + + # Lineage extraction (default: true) + include_lineage: true + + # Column-level lineage from SQL queries (default: true) + # Only applicable when include_lineage is true + include_column_lineage: true + + # Platform mappings for lineage extraction + connection_to_platform_map: + postgres_datasource_uid: + platform: postgres + platform_instance: my_postgres + env: PROD + database: analytics + database_schema: public +``` + +**Lineage Features:** + +- **Dataset-level lineage**: Links charts to their underlying data sources +- **Column-level lineage**: Extracts field-to-field relationships from SQL queries +- **Platform mapping**: Maps Grafana data sources to their actual platforms for accurate lineage +- **SQL parsing**: Supports parsing of SQL queries for detailed lineage extraction + +**Performance Note:** Lineage extraction can be disabled (`include_lineage: false`) to improve ingestion performance when lineage information is not needed. + +#### Ownership + +The Grafana source extracts dashboard ownership from the dashboard creator and assigns them as a Technical Owner. + +```yaml +source: + type: grafana + config: + url: "https://grafana.company.com" + service_account_token: "your_token" + + # Ownership extraction (default: true) + ingest_owners: true + + # Email suffix removal like @acryl.io (default: true) + remove_email_suffix: true +``` + +**Ownership Features:** + +- **Technical Owner assignment**: Dashboard creators are automatically assigned as Technical Owners +- **Email suffix control**: Configure how user email addresses are converted to DataHub user URNs via `remove_email_suffix` +- **Disable ownership**: Set `ingest_owners: false` to skip ownership extraction entirely + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.grafana.grafana_source.GrafanaSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/grafana/grafana_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Grafana, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hana.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hana.md new file mode 100644 index 00000000..664de8f7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hana.md @@ -0,0 +1,1059 @@ +--- +sidebar_position: 73 +title: SAP HANA +slug: /generated/ingestion/sources/hana +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# SAP HANA + +## Overview + +Hana is a DataHub utility or metadata-focused integration. Learn more in the [official Hana documentation](https://github.com/SAP/sqlalchemy-hana). + +The DataHub integration for Hana covers metadata entities and operational objects relevant to this connector. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `hana` +![Testing](https://img.shields.io/badge/support%20status-testing-lightgrey) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_column_lineage`. Supported for types - View. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default to get lineage for views via `include_view_lineage`. Supported for types - View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `hana` module ingests metadata from Hana into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +The implementation uses the [SQLAlchemy Dialect for SAP HANA](https://github.com/SAP/sqlalchemy-hana). The SQLAlchemy Dialect for SAP HANA is an open-source project hosted at GitHub that is actively maintained by SAP SE, and is not part of a licensed SAP HANA edition or option. It is provided under the terms of the project license. Please notice that sqlalchemy-hana isn't an official SAP product and isn't covered by SAP support. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[hana]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: hana + config: + # Coordinates + host_port: localhost:39041 + database: dbname + + # Credentials + username: ${HANA_USER} + password: ${HANA_PASS} + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
host_port
string
|
Default: localhost:39041
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
scheme
string
|
Default: hana+hdbcli
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schemas to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "default": "localhost:39041", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "scheme": { + "default": "hana+hdbcli", + "title": "Scheme", + "type": "string" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + } + }, + "title": "HanaConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +Under the hood, [SQLAlchemy Dialect for SAP HANA](https://github.com/SAP/sqlalchemy-hana) uses the SAP HANA Python Driver hdbcli. Therefore it is compatible with HANA or HANA express versions since HANA SPS 2. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.hana.HanaSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/hana.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for SAP HANA, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hex.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hex.md new file mode 100644 index 00000000..852571dc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hex.md @@ -0,0 +1,431 @@ +--- +sidebar_position: 35 +title: Hex +slug: /generated/ingestion/sources/hex +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Hex + +## Overview + +Hex is a business intelligence and analytics platform. Learn more in the [official Hex documentation](https://hex.tech/). + +The DataHub integration for Hex covers BI entities such as dashboards, charts, datasets, and related ownership context. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Hex Concept | DataHub Concept | Notes | +| ----------- | ----------------------------------------------------------------------------------------- | ------------------- | +| `"hex"` | [Data Platform](/docs/generated/metamodel/entities/dataplatform/) | | +| Workspace | [Container](/docs/generated/metamodel/entities/container/) | | +| Project | [Dashboard](/docs/generated/metamodel/entities/dashboard/) | Subtype `Project` | +| Component | [Dashboard](/docs/generated/metamodel/entities/dashboard/) | Subtype `Component` | +| Collection | [Tag](/docs/generated/metamodel/entities/Tag/) | | + +Other Hex concepts are not mapped to DataHub entities yet. + + +## Module `hex` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. | +| Dataset Usage | ✅ | Supported by default. Supported for types - Project. | +| Descriptions | ✅ | Supported by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Extract Ownership | ✅ | Supported by default. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | + +### Overview + +The `hex` module ingests metadata from Hex into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +#### Workspace Name + +Find your workspace name in your Hex home page URL: + +``` +https://app.hex.tech/ +``` + +Example: In `https://app.hex.tech/acryl-partnership`, the workspace name is `acryl-partnership`. + +#### Authentication + +Requires a Hex API Bearer token ([documentation](https://learn.hex.tech/docs/api/api-overview)). + +**Token options:** + +- **Workspace Token** (recommended): Read-only token sufficient for ingestion +- **PAT (Personal Access Token)**: Ingests with user's permissions + + +### Install the Plugin +```shell +pip install 'acryl-datahub[hex]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: hex + config: + workspace_name: # Hex workspace name. You can find this name in your Hex home page URL: https://app.hex.tech/ + token: # Your PAT or Workspace token + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
token ✅
string(password)
| Hex API token; either PAT or Workflow token - https://learn.hex.tech/docs/api/api-overview#authentication | +|
workspace_name ✅
string
| Hex workspace name. You can find this name in your Hex home page URL: https://app.hex.tech/ | +|
base_url
string
| Hex API base URL. For most Hex users, this will be https://app.hex.tech/api/v1. Single-tenant app users should replace this with the URL they use to access Hex.
Default: https://app.hex.tech/api/v1
| +|
categories_as_tags
boolean
| Emit Hex Category as tags
Default: True
| +|
collections_as_tags
boolean
| Emit Hex Collections as tags
Default: True
| +|
datahub_page_size
integer
| Number of items to fetch per DataHub API call.
Default: 100
| +|
include_components
boolean
| Include Hex Components in the ingestion
Default: True
| +|
include_lineage
boolean
| Include Hex lineage, being fetched from DataHub. See "Limitations" section in the docs for more details about the limitations of this feature.
Default: True
| +|
lineage_end_time
One of string(date-time), null
| Latest date of lineage to consider. Default: Current time in UTC. You can specify absolute time like '2023-01-01' or relative time like '-1 day' or '-1d'.
Default: None
| +|
lineage_start_time
One of string(date-time), null
| Earliest date of lineage to consider. Default: 1 day before lineage end time. You can specify absolute time like '2023-01-01' or relative time like '-7 days' or '-7d'.
Default: None
| +|
page_size
integer
| Number of items to fetch per Hex API call.
Default: 100
| +|
patch_metadata
boolean
| Emit metadata as patch events
Default: False
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
set_ownership_from_email
boolean
| Set ownership identity from owner/creator email
Default: True
| +|
status_as_tag
boolean
| Emit Hex Status as tags
Default: True
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
component_title_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
component_title_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
project_title_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
project_title_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Configuration for stateful ingestion and stale metadata removal.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for stateful ingestion and stale metadata removal." + }, + "workspace_name": { + "description": "Hex workspace name. You can find this name in your Hex home page URL: https://app.hex.tech/", + "title": "Workspace Name", + "type": "string" + }, + "token": { + "description": "Hex API token; either PAT or Workflow token - https://learn.hex.tech/docs/api/api-overview#authentication", + "format": "password", + "title": "Token", + "type": "string", + "writeOnly": true + }, + "base_url": { + "default": "https://app.hex.tech/api/v1", + "description": "Hex API base URL. For most Hex users, this will be https://app.hex.tech/api/v1. Single-tenant app users should replace this with the URL they use to access Hex.", + "title": "Base Url", + "type": "string" + }, + "include_components": { + "default": true, + "description": "Include Hex Components in the ingestion", + "title": "Include Components", + "type": "boolean" + }, + "page_size": { + "default": 100, + "description": "Number of items to fetch per Hex API call.", + "title": "Page Size", + "type": "integer" + }, + "patch_metadata": { + "default": false, + "description": "Emit metadata as patch events", + "title": "Patch Metadata", + "type": "boolean" + }, + "collections_as_tags": { + "default": true, + "description": "Emit Hex Collections as tags", + "title": "Collections As Tags", + "type": "boolean" + }, + "status_as_tag": { + "default": true, + "description": "Emit Hex Status as tags", + "title": "Status As Tag", + "type": "boolean" + }, + "categories_as_tags": { + "default": true, + "description": "Emit Hex Category as tags", + "title": "Categories As Tags", + "type": "boolean" + }, + "project_title_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex pattern for project titles to filter in ingestion." + }, + "component_title_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex pattern for component titles to filter in ingestion." + }, + "set_ownership_from_email": { + "default": true, + "description": "Set ownership identity from owner/creator email", + "title": "Set Ownership From Email", + "type": "boolean" + }, + "include_lineage": { + "default": true, + "description": "Include Hex lineage, being fetched from DataHub. See \"Limitations\" section in the docs for more details about the limitations of this feature.", + "title": "Include Lineage", + "type": "boolean" + }, + "lineage_start_time": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Earliest date of lineage to consider. Default: 1 day before lineage end time. You can specify absolute time like '2023-01-01' or relative time like '-7 days' or '-7d'.", + "title": "Lineage Start Time" + }, + "lineage_end_time": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Latest date of lineage to consider. Default: Current time in UTC. You can specify absolute time like '2023-01-01' or relative time like '-1 day' or '-1d'.", + "title": "Lineage End Time" + }, + "datahub_page_size": { + "default": 100, + "description": "Number of items to fetch per DataHub API call.", + "title": "Datahub Page Size", + "type": "integer" + } + }, + "required": [ + "workspace_name", + "token" + ], + "title": "HexSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +Currently, the [Hex API](https://learn.hex.tech/docs/api/api-reference) has some limitations that affect the completeness of the extracted metadata: + +1. **Projects and Components Relationship**: The API does not support fetching the many-to-many relationship between Projects and their Components. + +2. **Metadata Access**: There is no direct method to retrieve metadata for Collections, Status, or Categories. This information is only available indirectly through references within Projects and Components. + +Please keep these limitations in mind when working with the Hex connector. + +For the **Dataset - Hex Project lineage**, the connector relies on the +[_Hex query metadata_](https://learn.hex.tech/docs/explore-data/cells/sql-cells/sql-cells-introduction#query-metadata) feature. +Therefore, in order to extract lineage information, the required setup must include: + +- A separated warehouse ingestor (_eg_ BigQuery, Snowflake, Redshift, ...) with `use_queries_v2` enabled in order to fetch Queries. + This will ingest the queries into DataHub as `Query` entities and the ones triggered by Hex will include the corresponding _Hex query metadata_. +- A DataHub server with version >= SaaS `0.3.10` or > OSS `1.0.0` so the `Query` entities are properly indexed by source (Hex in this case) and so fetched and processed by the Hex ingestor in order to emit the Dataset - Project lineage. + +Please note: + +- Lineage is only captured for scheduled executions of the Project. +- In cases where queries are handled by [`hextoolkit`](https://learn.hex.tech/tutorials/connect-to-data/using-the-hextoolkit), _Hex query metadata_ is not injected, which prevents capturing lineage. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.hex.hex.HexSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/hex/hex.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Hex, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive-metastore.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive-metastore.md new file mode 100644 index 00000000..f7fb66ec --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive-metastore.md @@ -0,0 +1,3355 @@ +--- +sidebar_position: 37 +title: Hive Metastore +slug: /generated/ingestion/sources/hive-metastore +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Hive Metastore +There are 2 sources that provide integration with Hive Metastore + + + + + + + + + + +
Source ModuleDocumentation
+ +`hive-metastore` + + + + + +Source that extracts metadata from Hive Metastore via SQL or Thrift connection. + +Implementation notes: +- Uses HiveDataFetcher abstraction (SQLAlchemyDataFetcher or ThriftDataFetcher based on connection_type) +- Uses SqlParsingAggregator for extracting view lineage from view definitions +- Supports Presto/Trino view parsing from TABLE_PARAMS +- Implements stateful ingestion with StaleEntityRemovalHandler +- Complex type handling for struct/map/array schema fields + [Read more...](#module-hive-metastore) + + +
+ +`presto-on-hive` + + + + + +Source that extracts metadata from Hive Metastore via SQL or Thrift connection. + +Implementation notes: +- Uses HiveDataFetcher abstraction (SQLAlchemyDataFetcher or ThriftDataFetcher based on connection_type) +- Uses SqlParsingAggregator for extracting view lineage from view definitions +- Supports Presto/Trino view parsing from TABLE_PARAMS +- Implements stateful ingestion with StaleEntityRemovalHandler +- Complex type handling for struct/map/array schema fields + [Read more...](#module-presto-on-hive) + + +
+ + +## Overview + +Hive Metastore is a data platform used to store and query analytical or operational data. Learn more in the [official Hive Metastore documentation](https://hive.apache.org/). + +The DataHub integration for Hive Metastore covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `hive-metastore` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Catalog, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ❌ | Not Supported. | +| Column-level Lineage | ✅ | Enabled by default for views via `include_view_lineage`, and to storage via `include_column_lineage` when storage lineage is enabled. Supported for types - Table, View. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ❌ | Not Supported. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default for views via `include_view_lineage`, and to upstream/downstream storage via `emit_storage_lineage`. Supported for types - Table, View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `hive-metastore` module ingests metadata from Hive Metastore into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +- Extracts metadata from Hive Metastore. +- Supports two connection methods selected via connection_type: + - sql: Direct connection to HMS backend database (MySQL/PostgreSQL) + - thrift: Connection to HMS Thrift API with Kerberos support +- Features: + - Table and view metadata extraction + - Schema field types including complex types (struct, map, array) + - Storage lineage to S3, HDFS, Azure, GCS + - View lineage via SQL parsing + - Stateful ingestion for stale entity removal + +#### Related Documentation + +- [Hive Metastore Configuration](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/hive-metastore_recipe.yml) - Configuration examples +- [Hive Connector](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/hive) - Alternative connector via HiveServer2 +- [SQLAlchemy Documentation](https://docs.sqlalchemy.org/) - Underlying database connection library + +### Prerequisites + +The Hive Metastore connector supports two connection modes: + +1. **SQL Mode (Default)**: Connects directly to the Hive metastore database (MySQL, PostgreSQL, etc.) +2. **Thrift Mode**: Connects to Hive Metastore via the Thrift API (port 9083), with Kerberos support + +Choose your connection mode based on your environment: + +| Feature | SQL Mode (default) | Thrift Mode | +| ------------------ | -------------------------------- | -------------------------------- | +| **Use when** | Direct database access available | Only HMS Thrift API accessible | +| **Authentication** | Database credentials | Kerberos/SASL or unauthenticated | +| **Port** | Database port (3306/5432) | Thrift port (9083) | +| **Dependencies** | Database drivers | `pymetastore`, `thrift-sasl` | + +**Requirements:** + +1. **Database Access**: Direct read access to the Hive metastore database (MySQL or PostgreSQL) + +2. **Network Access**: Access to metastore database on configured port + +3. **Database Driver**: Install the appropriate Python driver: + + ```bash + # For PostgreSQL metastore + pip install 'acryl-datahub[hive]' psycopg2-binary + + # For MySQL metastore + pip install 'acryl-datahub[hive]' PyMySQL + ``` + +4. **Metastore Schema**: Typically `public` (PostgreSQL) or database name (MySQL) + +#### Required Database Permissions + +The database user account used by DataHub needs read-only access to the Hive metastore tables. + +##### PostgreSQL Metastore + +```sql +-- Create a dedicated read-only user for DataHub +CREATE USER datahub_user WITH PASSWORD 'secure_password'; + +-- Grant connection privileges +GRANT CONNECT ON DATABASE metastore TO datahub_user; + +-- Grant schema usage +GRANT USAGE ON SCHEMA public TO datahub_user; + +-- Grant SELECT on metastore tables +GRANT SELECT ON ALL TABLES IN SCHEMA public TO datahub_user; + +-- Grant SELECT on future tables (for metastore upgrades) +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO datahub_user; +``` + +##### MySQL Metastore + +```sql +-- Create a dedicated read-only user for DataHub +CREATE USER 'datahub_user'@'%' IDENTIFIED BY 'secure_password'; + +-- Grant SELECT privileges on metastore database +GRANT SELECT ON metastore.* TO 'datahub_user'@'%'; + +-- Apply changes +FLUSH PRIVILEGES; +``` + +##### Required Metastore Tables + +DataHub queries the following metastore tables: + +| Table | Purpose | +| ---------------- | --------------------------------------------- | +| `DBS` | Database/schema information | +| `TBLS` | Table metadata | +| `TABLE_PARAMS` | Table properties (including view definitions) | +| `SDS` | Storage descriptor (location, format) | +| `COLUMNS_V2` | Column metadata | +| `PARTITION_KEYS` | Partition information | +| `SERDES` | Serialization/deserialization information | + +**Recommendation**: Grant `SELECT` on all metastore tables to ensure compatibility with different Hive versions and for future DataHub enhancements. + +#### Authentication + +##### PostgreSQL + +**Standard Connection**: + +```yaml +source: + type: hive-metastore + config: + host_port: metastore-db.company.com:5432 + database: metastore + username: datahub_user + password: ${METASTORE_PASSWORD} + scheme: "postgresql+psycopg2" +``` + +**SSL Connection**: + +```yaml +source: + type: hive-metastore + config: + host_port: metastore-db.company.com:5432 + database: metastore + username: datahub_user + password: ${METASTORE_PASSWORD} + scheme: "postgresql+psycopg2" + options: + connect_args: + sslmode: require + sslrootcert: /path/to/ca-cert.pem +``` + +##### MySQL + +**Standard Connection**: + +```yaml +source: + type: hive-metastore + config: + host_port: metastore-db.company.com:3306 + database: metastore + username: datahub_user + password: ${METASTORE_PASSWORD} + scheme: "mysql+pymysql" # Default if not specified +``` + +**SSL Connection**: + +```yaml +source: + type: hive-metastore + config: + host_port: metastore-db.company.com:3306 + database: metastore + username: datahub_user + password: ${METASTORE_PASSWORD} + scheme: "mysql+pymysql" + options: + connect_args: + ssl: + ca: /path/to/ca-cert.pem + cert: /path/to/client-cert.pem + key: /path/to/client-key.pem +``` + +##### Amazon RDS (PostgreSQL or MySQL) + +For AWS RDS-hosted metastore databases: + +```yaml +source: + type: hive-metastore + config: + host_port: metastore.abc123.us-east-1.rds.amazonaws.com:5432 + database: metastore + username: datahub_user + password: ${RDS_PASSWORD} + scheme: "postgresql+psycopg2" # or 'mysql+pymysql' + options: + connect_args: + sslmode: require # RDS requires SSL +``` + +##### Azure Database for PostgreSQL/MySQL + +```yaml +source: + type: hive-metastore + config: + host_port: metastore-server.postgres.database.azure.com:5432 + database: metastore + username: datahub_user@metastore-server # Note: Azure requires @server-name suffix + password: ${AZURE_DB_PASSWORD} + scheme: "postgresql+psycopg2" + options: + connect_args: + sslmode: require +``` + + +### Install the Plugin +```shell +pip install 'acryl-datahub[hive-metastore]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +# ============================================================================= +# SQL Mode (Default) - Direct database connection +# ============================================================================= +source: + type: hive-metastore + config: + # Hive metastore DB connection + host_port: localhost:5432 + database: metastore + + # specify the schema where metastore tables reside + schema_pattern: + allow: + - "^public" + + # credentials + username: user # optional + password: pass # optional + + #scheme: 'postgresql+psycopg2' # set this if metastore db is using postgres + #scheme: 'mysql+pymysql' # set this if metastore db is using mysql, default if unset + + # Filter databases using pattern-based filtering + #database_pattern: + # allow: + # - "^db1$" + # deny: + # - "^test_.*" + + # Storage Lineage Configuration (Optional) + # Enables lineage between Hive tables and their underlying storage locations + #emit_storage_lineage: false # Set to true to enable storage lineage + #hive_storage_lineage_direction: upstream # Direction: 'upstream' (storage -> Hive) or 'downstream' (Hive -> storage) + #include_column_lineage: true # Set to false to disable column-level lineage + #storage_platform_instance: "prod-hdfs" # Optional: platform instance for storage URNs + +sink: + # sink configs + +# ============================================================================= +# Thrift Mode - HMS Thrift API connection (use when database access unavailable) +# ============================================================================= +# Use this mode when: +# - You cannot access the metastore database directly +# - Only the HMS Thrift API (port 9083) is accessible +# - Your environment requires Kerberos authentication +# +# Prerequisites: +# - pip install 'acryl-datahub[hive-metastore]' +# - For Kerberos: pip install thrift-sasl pyhive[hive-pure-sasl] +# - For Kerberos: Run 'kinit' before ingestion to obtain ticket +# +# source: +# type: hive-metastore +# config: +# # ========================================================================= +# # Connection Settings (Required) +# # ========================================================================= +# connection_type: thrift # Enable Thrift mode (default is 'sql') +# host_port: hms.company.com:9083 # HMS Thrift API endpoint +# +# # ========================================================================= +# # Authentication - Kerberos/SASL (Optional) +# # ========================================================================= +# # Enable if HMS requires Kerberos authentication +# # Prerequisite: Run 'kinit -kt /path/to/keytab user@REALM' before ingestion +# use_kerberos: true +# +# # Kerberos service principal name (typically 'hive') +# # Check your HMS principal: klist -k /etc/hive/hive.keytab +# kerberos_service_name: hive +# +# # Override hostname for Kerberos principal (use with load balancers) +# # Set this if connecting via LB but Kerberos principal uses actual hostname +# # kerberos_hostname_override: hms-master.company.com +# +# # ========================================================================= +# # Connection Tuning (Optional) +# # ========================================================================= +# # timeout_seconds: 60 # Connection timeout (default: 60) +# # max_retries: 3 # Retry attempts for transient failures (default: 3) +# +# # ========================================================================= +# # HMS 3.x Catalog Support (Optional) +# # ========================================================================= +# # For HMS 3.x with multi-catalog support (e.g., Spark catalog) +# # catalog_name: spark_catalog +# # include_catalog_name_in_ids: true # Include catalog in dataset URNs +# +# # ========================================================================= +# # Filtering (Pattern-based only - WHERE clauses NOT supported) +# # ========================================================================= +# database_pattern: +# allow: +# - "^prod_.*" # Allow databases starting with 'prod_' +# - "^analytics$" # Allow exact match 'analytics' +# deny: +# - "^test_.*" # Deny databases starting with 'test_' +# - ".*_staging$" # Deny databases ending with '_staging' +# +# table_pattern: +# allow: +# - ".*" # Allow all tables by default +# deny: +# - "^tmp_.*" # Deny temporary tables +# +# # ========================================================================= +# # Storage Lineage (Optional - works same as SQL mode) +# # ========================================================================= +# emit_storage_lineage: true +# hive_storage_lineage_direction: upstream # or 'downstream' +# include_column_lineage: true +# # storage_platform_instance: "prod-hdfs" +# +# # ========================================================================= +# # Platform Instance (Optional - for multi-cluster environments) +# # ========================================================================= +# # platform_instance: "prod-hive" +# +# # ========================================================================= +# # Stateful Ingestion (Optional - for incremental updates) +# # ========================================================================= +# # stateful_ingestion: +# # enabled: true +# # remove_stale_metadata: true +# +# sink: +# type: datahub-rest +# config: +# server: http://localhost:8080 + +# ============================================================================= +# Thrift Mode - Minimal Example (No Kerberos) +# ============================================================================= +# source: +# type: hive-metastore +# config: +# connection_type: thrift +# host_port: hms.company.com:9083 +# use_kerberos: false +# +# sink: +# type: datahub-rest +# config: +# server: http://localhost:8080 + +# ============================================================================= +# Thrift Mode - Kerberos with Load Balancer +# ============================================================================= +# source: +# type: hive-metastore +# config: +# connection_type: thrift +# host_port: hms-lb.company.com:9083 # Load balancer address +# use_kerberos: true +# kerberos_service_name: hive +# kerberos_hostname_override: hms-master.company.com # Actual HMS hostname +# +# sink: +# type: datahub-rest +# config: +# server: http://localhost:8080 +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
catalog_name
One of string, null
| Catalog name for HMS 3.x multi-catalog deployments. Only for connection_type='thrift'.
Default: None
| +|
connection_type
Enum
| One of: "sql", "thrift" | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
emit_storage_lineage
boolean
| Whether to emit storage-to-Hive lineage. When enabled, creates lineage relationships between Hive tables and their underlying storage locations (S3, Azure, GCS, HDFS, etc.).
Default: False
| +|
enable_properties_merge
boolean
| Merge properties with existing server data instead of overwriting.
Default: True
| +|
hive_storage_lineage_direction
Enum
| One of: "upstream", "downstream" | +|
host_port
string
| Host and port. For SQL: database port (3306/5432). For Thrift: HMS Thrift port (9083).
Default: localhost:3306
| +|
include_catalog_name_in_ids
boolean
| Add catalog name to dataset URNs. Example: `urn:li:dataset:(urn:li:dataPlatform:hive,catalog.db.table,PROD)`
Default: False
| +|
include_column_lineage
boolean
| When enabled along with emit_storage_lineage, column-level lineage will be extracted between Hive table columns and storage location fields.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Extract lineage from Hive views by parsing view definitions.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
ingestion_job_id
string
|
Default:
| +|
kerberos_hostname_override
One of string, null
| Override hostname for Kerberos principal construction. Use when connecting through a load balancer. Only for connection_type='thrift'.
Default: None
| +|
kerberos_qop
string
| Kerberos Quality of Protection (QOP) for SASL authentication. Options: 'auth' (authentication only), 'auth-int' (authentication + integrity), 'auth-conf' (authentication + confidentiality/encryption). Must match the server's hadoop.rpc.protection setting. Only for connection_type='thrift'.
Default: auth
| +|
kerberos_service_name
string
| Kerberos service name for the HMS principal. Only for connection_type='thrift'.
Default: hive
| +|
metastore_db_name
One of string, null
| Name of the Hive metastore's database (usually: metastore). For backward compatibility, if not provided, the database field will be used. If both 'database' and 'metastore_db_name' are set, 'database' is used for filtering.
Default: None
| +|
mode
Enum
| One of: "hive", "presto", "presto-on-hive", "trino" | +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
schemas_where_clause_suffix
string
| DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.
Default:
| +|
simplify_nested_field_paths
boolean
| Simplify v2 field paths to v1. Falls back to v2 for Union/Array types.
Default: False
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
storage_platform_instance
One of string, null
| Platform instance for the storage system (e.g., 'my-s3-instance'). Used when generating URNs for storage datasets.
Default: None
| +|
tables_where_clause_suffix
string
| DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.
Default:
| +|
timeout_seconds
integer
| Connection timeout in seconds. Only for connection_type='thrift'.
Default: 60
| +|
use_catalog_subtype
boolean
| Use 'Catalog' (True) or 'Database' (False) as container subtype.
Default: True
| +|
use_dataset_pascalcase_subtype
boolean
| Use 'Table'/'View' (True) or 'table'/'view' (False) as dataset subtype.
Default: False
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
use_kerberos
boolean
| Whether to use Kerberos/SASL authentication. Only for connection_type='thrift'.
Default: False
| +|
username
One of string, null
| username
Default: None
| +|
views_where_clause_suffix
string
| DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.
Default:
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Configuration for stateful ingestion and stale entity removal.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "HiveMetastoreConfigMode": { + "description": "Mode for metadata extraction.", + "enum": [ + "hive", + "presto", + "presto-on-hive", + "trino" + ], + "title": "HiveMetastoreConfigMode", + "type": "string" + }, + "HiveMetastoreConnectionType": { + "description": "Connection type for HiveMetastoreSource.", + "enum": [ + "sql", + "thrift" + ], + "title": "HiveMetastoreConnectionType", + "type": "string" + }, + "LineageDirection": { + "description": "Direction of lineage relationship between storage and Hive", + "enum": [ + "upstream", + "downstream" + ], + "title": "LineageDirection", + "type": "string" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Configuration for Hive Metastore source.\n\nSupports two connection types:\n- sql: Direct database access (MySQL/PostgreSQL) to HMS backend\n- thrift: HMS Thrift API with Kerberos support", + "properties": { + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schemas to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for stateful ingestion and stale entity removal." + }, + "emit_storage_lineage": { + "default": false, + "description": "Whether to emit storage-to-Hive lineage. When enabled, creates lineage relationships between Hive tables and their underlying storage locations (S3, Azure, GCS, HDFS, etc.).", + "title": "Emit Storage Lineage", + "type": "boolean" + }, + "hive_storage_lineage_direction": { + "$ref": "#/$defs/LineageDirection", + "default": "upstream", + "description": "Direction of storage lineage. If 'upstream', storage is treated as upstream to Hive (data flows from storage to Hive). If 'downstream', storage is downstream to Hive (data flows from Hive to storage)." + }, + "include_column_lineage": { + "default": true, + "description": "When enabled along with emit_storage_lineage, column-level lineage will be extracted between Hive table columns and storage location fields.", + "title": "Include Column Lineage", + "type": "boolean" + }, + "storage_platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Platform instance for the storage system (e.g., 'my-s3-instance'). Used when generating URNs for storage datasets.", + "title": "Storage Platform Instance" + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Extract lineage from Hive views by parsing view definitions.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "default": "localhost:3306", + "description": "Host and port. For SQL: database port (3306/5432). For Thrift: HMS Thrift port (9083).", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "connection_type": { + "$ref": "#/$defs/HiveMetastoreConnectionType", + "default": "sql", + "description": "Connection method: 'sql' for direct database access (MySQL/PostgreSQL), 'thrift' for HMS Thrift API with optional Kerberos support." + }, + "views_where_clause_suffix": { + "default": "", + "description": "DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.", + "title": "Views Where Clause Suffix", + "type": "string" + }, + "tables_where_clause_suffix": { + "default": "", + "description": "DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.", + "title": "Tables Where Clause Suffix", + "type": "string" + }, + "schemas_where_clause_suffix": { + "default": "", + "description": "DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.", + "title": "Schemas Where Clause Suffix", + "type": "string" + }, + "metastore_db_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the Hive metastore's database (usually: metastore). For backward compatibility, if not provided, the database field will be used. If both 'database' and 'metastore_db_name' are set, 'database' is used for filtering.", + "title": "Metastore Db Name" + }, + "use_kerberos": { + "default": false, + "description": "Whether to use Kerberos/SASL authentication. Only for connection_type='thrift'.", + "title": "Use Kerberos", + "type": "boolean" + }, + "kerberos_service_name": { + "default": "hive", + "description": "Kerberos service name for the HMS principal. Only for connection_type='thrift'.", + "title": "Kerberos Service Name", + "type": "string" + }, + "kerberos_hostname_override": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Override hostname for Kerberos principal construction. Use when connecting through a load balancer. Only for connection_type='thrift'.", + "title": "Kerberos Hostname Override" + }, + "kerberos_qop": { + "default": "auth", + "description": "Kerberos Quality of Protection (QOP) for SASL authentication. Options: 'auth' (authentication only), 'auth-int' (authentication + integrity), 'auth-conf' (authentication + confidentiality/encryption). Must match the server's hadoop.rpc.protection setting. Only for connection_type='thrift'.", + "title": "Kerberos Qop", + "type": "string" + }, + "timeout_seconds": { + "default": 60, + "description": "Connection timeout in seconds. Only for connection_type='thrift'.", + "title": "Timeout Seconds", + "type": "integer" + }, + "catalog_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Catalog name for HMS 3.x multi-catalog deployments. Only for connection_type='thrift'.", + "title": "Catalog Name" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for databases to filter." + }, + "mode": { + "$ref": "#/$defs/HiveMetastoreConfigMode", + "default": "hive", + "description": "Platform mode for metadata. Valid options: ['hive', 'presto', 'presto-on-hive', 'trino']" + }, + "use_catalog_subtype": { + "default": true, + "description": "Use 'Catalog' (True) or 'Database' (False) as container subtype.", + "title": "Use Catalog Subtype", + "type": "boolean" + }, + "use_dataset_pascalcase_subtype": { + "default": false, + "description": "Use 'Table'/'View' (True) or 'table'/'view' (False) as dataset subtype.", + "title": "Use Dataset Pascalcase Subtype", + "type": "boolean" + }, + "include_catalog_name_in_ids": { + "default": false, + "description": "Add catalog name to dataset URNs. Example: urn:li:dataset:(urn:li:dataPlatform:hive,catalog.db.table,PROD)", + "title": "Include Catalog Name In Ids", + "type": "boolean" + }, + "enable_properties_merge": { + "default": true, + "description": "Merge properties with existing server data instead of overwriting.", + "title": "Enable Properties Merge", + "type": "boolean" + }, + "simplify_nested_field_paths": { + "default": false, + "description": "Simplify v2 field paths to v1. Falls back to v2 for Union/Array types.", + "title": "Simplify Nested Field Paths", + "type": "boolean" + }, + "ingestion_job_id": { + "default": "", + "title": "Ingestion Job Id", + "type": "string" + } + }, + "title": "HiveMetastore", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Thrift Connection Mode + +Use `connection_type: thrift` when you cannot access the metastore database directly but have access to the HMS Thrift API (typically port 9083). This is common in: + +- Kerberized Hadoop clusters where database access is restricted +- Cloud-managed Hive services that only expose the Thrift API +- Environments with strict network segmentation + +##### Thrift Mode Prerequisites + +Before using Thrift mode, ensure: + +1. **Network Access**: The machine running DataHub ingestion can reach HMS on port 9083 +2. **HMS Service Running**: The Hive Metastore service is running and accepting Thrift connections +3. **For Kerberos**: A valid Kerberos ticket is available (see Kerberos section below) + +**Verify connectivity**: + +```bash +# Test network connectivity to HMS +telnet hms.company.com 9083 + +# For Kerberos environments, verify ticket +klist +``` + +##### Thrift Mode Dependencies + +```bash +# Install with Thrift support +pip install 'acryl-datahub[hive-metastore]' + +# For Kerberos authentication, also install: +pip install thrift-sasl pyhive[hive-pure-sasl] +``` + +##### Thrift Configuration Options + +| Option | Type | Default | Required | Description | +| ----------------------------- | --------- | ------- | ---------------- | ------------------------------------------------------------------------------ | +| `connection_type` | string | `sql` | Yes (for Thrift) | Set to `thrift` to enable Thrift mode | +| `host_port` | string | - | Yes | HMS host and port (e.g., `hms.company.com:9083`) | +| `use_kerberos` | boolean | `false` | No | Enable Kerberos/SASL authentication | +| `kerberos_service_name` | string | `hive` | No | Kerberos service principal name | +| `kerberos_hostname_override` | string | - | No | Override hostname for Kerberos principal (for load balancers) | +| `kerberos_qop` | string | `auth` | No | Kerberos Quality of Protection: `auth`, `auth-int`, or `auth-conf` (see below) | +| `timeout_seconds` | int | `60` | No | Connection timeout in seconds | +| `max_retries` | int | `3` | No | Maximum retry attempts for transient failures | +| `catalog_name` | string | - | No | HMS 3.x catalog name (e.g., `spark_catalog`) | +| `include_catalog_name_in_ids` | boolean | `false` | No | Include catalog in dataset URNs | +| `database_pattern` | AllowDeny | - | No | Filter databases by regex pattern | +| `table_pattern` | AllowDeny | - | No | Filter tables by regex pattern | + +**Note**: SQL `WHERE` clause options (`tables_where_clause_suffix`, `views_where_clause_suffix`, `schemas_where_clause_suffix`) have been **deprecated** for security reasons (SQL injection risk) and are no longer supported. Use `database_pattern` and `table_pattern` instead. + +##### Basic Thrift Configuration + +```yaml +source: + type: hive-metastore + config: + connection_type: thrift + host_port: hms.company.com:9083 +``` + +##### Thrift with Kerberos Authentication + +Ensure you have a valid Kerberos ticket (`kinit -kt /path/to/keytab user@REALM`) before running ingestion: + +```yaml +source: + type: hive-metastore + config: + connection_type: thrift + host_port: hms.company.com:9083 + use_kerberos: true + kerberos_service_name: hive # Change if HMS uses different principal + # kerberos_hostname_override: hms-internal.company.com # If using load balancer + # catalog_name: spark_catalog # For HMS 3.x multi-catalog + # kerberos_qop: auth-conf # For Kerberos QOP authentication + integrity + encryption + database_pattern: + allow: + - "^prod_.*" +``` + +##### Kerberos Quality of Protection (QOP) + +If your Hive Metastore is configured with `hadoop.rpc.protection` set to `integrity` or `privacy`, you must configure the matching QOP level: + +| `hadoop.rpc.protection` | `kerberos_qop` | Description | +| ----------------------- | -------------- | --------------------------------------- | +| `authentication` | `auth` | Authentication only (default) | +| `integrity` | `auth-int` | Authentication + integrity checking | +| `privacy` | `auth-conf` | Authentication + integrity + encryption | + +##### Thrift Mode Limitations + +- **No Presto/Trino view lineage**: View SQL parsing requires SQL mode +- **No WHERE clause filtering**: Use `database_pattern`/`table_pattern` instead +- **Kerberos ticket required**: Must have valid ticket before running (not embedded in config) +- **HMS version compatibility**: Tested with HMS 2.x and 3.x + +#### Storage Lineage + +The Hive Metastore connector supports the same storage lineage features as the Hive connector, with enhanced performance due to direct database access. + +##### Quick Start + +Enable storage lineage with minimal configuration: + +```yaml +source: + type: hive-metastore + config: + host_port: metastore-db.company.com:5432 + database: metastore + username: datahub_user + password: ${METASTORE_PASSWORD} + scheme: "postgresql+psycopg2" + + # Enable storage lineage + emit_storage_lineage: true +``` + +##### Configuration Options + +Storage lineage is controlled by the same parameters as the Hive connector: + +| Parameter | Type | Default | Description | +| -------------------------------- | ------- | ------------ | --------------------------------------------------------------------------- | +| `emit_storage_lineage` | boolean | `false` | Master toggle to enable/disable storage lineage | +| `hive_storage_lineage_direction` | string | `"upstream"` | Direction: `"upstream"` (storage → Hive) or `"downstream"` (Hive → storage) | +| `include_column_lineage` | boolean | `true` | Enable column-level lineage from storage paths to Hive columns | +| `storage_platform_instance` | string | `None` | Platform instance for storage URNs (e.g., `"prod-s3"`, `"dev-hdfs"`) | + +##### Supported Storage Platforms + +All storage platforms supported by the Hive connector are also supported here: + +- Amazon S3 (`s3://`, `s3a://`, `s3n://`) +- HDFS (`hdfs://`) +- Google Cloud Storage (`gs://`) +- Azure Blob Storage (`wasb://`, `wasbs://`) +- Azure Data Lake (`adl://`, `abfs://`, `abfss://`) +- Databricks File System (`dbfs://`) +- Local File System (`file://`) + +See the sections above for complete configuration details. + +#### Presto and Trino View Support + +A key advantage of the Hive Metastore connector is its ability to extract metadata from Presto and Trino views that are stored in the metastore. + +##### How It Works + +1. **View Detection**: The connector identifies views by checking the `TABLE_PARAMS` table for Presto/Trino view definitions. + +2. **View Parsing**: Presto/Trino view JSON is parsed to extract: + + - Original SQL text + - Referenced tables + - Column metadata and types + +3. **Lineage Extraction**: SQL is parsed using `sqlglot` to create table-to-view lineage. + +4. **Storage Lineage Integration**: If storage lineage is enabled, the connector also creates lineage from storage → tables → views. + +##### Configuration + +Presto/Trino view support is automatically enabled when ingesting from a metastore that contains Presto/Trino views. No additional configuration is required. + +##### Example + +```yaml +source: + type: hive-metastore + config: + host_port: metastore-db.company.com:5432 + database: metastore + username: datahub_user + password: ${METASTORE_PASSWORD} + scheme: "postgresql+psycopg2" + + # Enable storage lineage for complete lineage chain + emit_storage_lineage: true +``` + +This configuration will create complete lineage: + +``` +S3 Bucket → Hive Table → Presto View +``` + +##### Limitations + +- **Presto/Trino Version**: The connector supports Presto 0.200+ and Trino view formats +- **Complex SQL**: Very complex SQL with non-standard syntax may have incomplete lineage +- **Cross-Database References**: Lineage is extracted for references within the same Hive metastore + +#### Schema Filtering + +For large metastore deployments with many databases, use filtering to limit ingestion scope: + +##### Database Filtering + +```yaml +source: + type: hive-metastore + config: + # ... connection config ... + + # Only ingest from specific databases + schema_pattern: + allow: + - "^production_.*" # All databases starting with production_ + - "analytics" # Specific database + deny: + - ".*_test$" # Exclude test databases +``` + +##### Table Filtering with SQL + +For filtering by database name, use pattern-based filtering: + +```yaml +source: + type: hive-metastore + config: + # ... connection config ... + + # Filter to specific databases using regex patterns + database_pattern: + allow: + - "^production_db$" + - "^analytics_db$" + deny: + - "^test_.*" + - ".*_staging$" +``` + +**Note**: The deprecated `*_where_clause_suffix` options have been removed for security reasons. Use `database_pattern` and `table_pattern` for filtering. + +#### Performance Considerations + +##### Advantages Over HiveServer2 Connector + +The Hive Metastore connector is significantly faster than the Hive connector because: + +1. **Direct Database Access**: No HiveServer2 overhead +2. **Batch Queries**: Fetches all metadata in optimized SQL queries +3. **No Query Execution**: Doesn't run Hive queries to extract metadata +4. **Parallel Processing**: Can process multiple databases concurrently + +**Performance Comparison** (approximate): + +- **10 databases, 1000 tables**: ~2 minutes (Metastore) vs ~15 minutes (HiveServer2) +- **100 databases, 10,000 tables**: ~15 minutes (Metastore) vs ~2 hours (HiveServer2) + +##### Optimization Tips + +1. **Database Connection Pooling**: The connector uses SQLAlchemy's default connection pooling. For very large deployments, consider tuning pool size: + + ```yaml + options: + pool_size: 10 + max_overflow: 20 + ``` + +2. **Schema Filtering**: Use `schema_pattern` to limit scope and reduce query time. + +3. **Stateful Ingestion**: Enable to only process changes: + + ```yaml + stateful_ingestion: + enabled: true + remove_stale_metadata: true + ``` + +4. **Disable Column Lineage**: If not needed: + ```yaml + emit_storage_lineage: true + include_column_lineage: false # Faster + ``` + +##### Network Considerations + +- **Latency**: Low latency to the metastore database is important +- **Bandwidth**: Minimal bandwidth required (only metadata, no data transfer) +- **Connection Limits**: Ensure metastore database can handle additional read connections + +#### Platform Instances + +When ingesting from multiple metastores (e.g., different clusters or environments), use `platform_instance`: + +```yaml +source: + type: hive-metastore + config: + host_port: prod-metastore-db.company.com:5432 + database: metastore + platform_instance: "prod-hive" +``` + +**Best Practice**: Combine with `storage_platform_instance`: + +```yaml +source: + type: hive-metastore + config: + platform_instance: "prod-hive" # Hive tables + storage_platform_instance: "prod-hdfs" # Storage locations + emit_storage_lineage: true +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +##### Metastore Schema Compatibility + +- **Hive Versions**: Tested with Hive 1.x, 2.x, and 3.x metastore schemas +- **Schema Variations**: Different Hive versions may have slightly different metastore schemas +- **Custom Tables**: If your organization has added custom metastore tables, they won't be processed + +##### Database Support + +- **Supported**: PostgreSQL, MySQL, MariaDB +- **Not Supported**: Oracle, MSSQL (may work but untested) +- **Derby**: Not recommended (embedded metastore, typically single-user) + +##### View Lineage Parsing + +- **Simple SQL**: Fully supported with accurate lineage +- **Complex SQL**: Best-effort parsing; some edge cases may have incomplete lineage +- **Non-standard SQL**: Presto/Trino-specific functions may not be fully parsed + +##### Permissions Limitations + +- **Read-Only**: The connector only needs SELECT permissions +- **No Write Operations**: Never requires INSERT, UPDATE, or DELETE +- **Metastore Locks**: Read operations don't acquire metastore locks + +##### Storage Lineage Limitations + +Same as the Hive connector: + +- Only tables with defined storage locations have lineage +- Temporary tables are not supported +- Partition-level lineage is aggregated at table level + +### Troubleshooting + +1. **Large Column Lists**: Tables with 500+ columns may be slow to process due to metastore query complexity. + +2. **View Definition Encoding**: Some older Hive versions store view definitions in non-UTF-8 encoding, which may cause parsing issues. + +3. **Case Sensitivity**: + + - PostgreSQL metastore: Case-sensitive identifiers + - MySQL metastore: Case-insensitive by default + - DataHub automatically lowercases URNs for consistency + +4. **Concurrent Metastore Writes**: If the metastore is being actively modified during ingestion, some metadata may be inconsistent. + +#### Connection Issues + +**Problem**: `Could not connect to metastore database` + +**Solutions**: + +- Verify `host_port`, `database`, and `scheme` are correct +- Check network connectivity: `telnet ` +- Verify firewall rules allow connections +- For PostgreSQL: Check `pg_hba.conf` allows connections from your IP +- For MySQL: Check `bind-address` in `my.cnf` + +#### Authentication Failures + +**Problem**: `Authentication failed` or `Access denied` + +**Solutions**: + +- Verify username and password are correct +- Check user has CONNECT/LOGIN privileges +- For Azure: Ensure username includes `@server-name` suffix +- Review database logs for detailed error messages + +#### Missing Tables + +**Problem**: Not all tables appear in DataHub + +**Solutions**: + +- Verify database user has SELECT on all metastore tables +- Check if tables are filtered by `schema_pattern`, `database_pattern`, or `table_pattern` +- Query metastore directly to verify tables exist: + +```sql + SELECT d.name as db_name, t.tbl_name as table_name, t.tbl_type + FROM TBLS t + JOIN DBS d ON t.db_id = d.db_id + WHERE d.name = 'your_database'; +``` + +#### Presto/Trino Views Not Appearing + +**Problem**: Views defined in Presto/Trino don't show up + +**Solutions**: + +- Check view definitions exist in metastore: + +```sql + SELECT d.name as db_name, t.tbl_name as view_name, tp.param_value + FROM TBLS t + JOIN DBS d ON t.db_id = d.db_id + JOIN TABLE_PARAMS tp ON t.tbl_id = tp.tbl_id + WHERE t.tbl_type = 'VIRTUAL_VIEW' + AND tp.param_key = 'presto_view' + LIMIT 10; +``` + +- +- Review ingestion logs for parsing errors +- Verify view JSON is valid + +#### Storage Lineage Not Appearing + +**Problem**: No storage lineage relationships visible + +**Solutions**: + +- Verify `emit_storage_lineage: true` is set +- Check tables have storage locations in metastore: +- + +```sql + SELECT d.name as db_name, t.tbl_name as table_name, s.location + FROM TBLS t + JOIN DBS d ON t.db_id = d.db_id + JOIN SDS s ON t.sd_id = s.sd_id + WHERE s.location IS NOT NULL + LIMIT 10; +``` + +- Review logs for "Failed to parse storage location" warnings +- See the "Storage Lineage" section above for troubleshooting tips + +#### Slow Ingestion + +**Problem**: Ingestion takes too long + +**Solutions**: + +- Use schema filtering to reduce scope +- Enable stateful ingestion to only process changes +- Check database query performance (may need indexes on metastore tables) +- Ensure low latency network connection to metastore database +- Consider disabling column lineage if not needed + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.hive.hive_metastore_source.HiveMetastoreSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/hive/hive_metastore_source.py) + + + +## Module `presto-on-hive` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Catalog, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ❌ | Not Supported. | +| Column-level Lineage | ✅ | Enabled by default for views via `include_view_lineage`, and to storage via `include_column_lineage` when storage lineage is enabled. Supported for types - Table, View. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ❌ | Not Supported. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default for views via `include_view_lineage`, and to upstream/downstream storage via `emit_storage_lineage`. Supported for types - Table, View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `presto-on-hive` module ingests metadata for Presto deployments that use Hive Metastore-backed catalogs. + +### Prerequisites + +- Connectivity to the target Presto deployment and backing metastore. +- Credentials and permissions to read catalog/schema/table metadata and optional query metadata. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[presto-on-hive]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: presto-on-hive + config: + host_port: "localhost:8080" + database: "hive" + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
catalog_name
One of string, null
| Catalog name for HMS 3.x multi-catalog deployments. Only for connection_type='thrift'.
Default: None
| +|
connection_type
Enum
| One of: "sql", "thrift" | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
emit_storage_lineage
boolean
| Whether to emit storage-to-Hive lineage. When enabled, creates lineage relationships between Hive tables and their underlying storage locations (S3, Azure, GCS, HDFS, etc.).
Default: False
| +|
enable_properties_merge
boolean
| Merge properties with existing server data instead of overwriting.
Default: True
| +|
hive_storage_lineage_direction
Enum
| One of: "upstream", "downstream" | +|
host_port
string
| Host and port. For SQL: database port (3306/5432). For Thrift: HMS Thrift port (9083).
Default: localhost:3306
| +|
include_catalog_name_in_ids
boolean
| Add catalog name to dataset URNs. Example: `urn:li:dataset:(urn:li:dataPlatform:hive,catalog.db.table,PROD)`
Default: False
| +|
include_column_lineage
boolean
| When enabled along with emit_storage_lineage, column-level lineage will be extracted between Hive table columns and storage location fields.
Default: True
| +|
include_table_location_lineage
boolean
| If the source supports it, include table lineage to the underlying storage location.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Extract lineage from Hive views by parsing view definitions.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
ingestion_job_id
string
|
Default:
| +|
kerberos_hostname_override
One of string, null
| Override hostname for Kerberos principal construction. Use when connecting through a load balancer. Only for connection_type='thrift'.
Default: None
| +|
kerberos_qop
string
| Kerberos Quality of Protection (QOP) for SASL authentication. Options: 'auth' (authentication only), 'auth-int' (authentication + integrity), 'auth-conf' (authentication + confidentiality/encryption). Must match the server's hadoop.rpc.protection setting. Only for connection_type='thrift'.
Default: auth
| +|
kerberos_service_name
string
| Kerberos service name for the HMS principal. Only for connection_type='thrift'.
Default: hive
| +|
metastore_db_name
One of string, null
| Name of the Hive metastore's database (usually: metastore). For backward compatibility, if not provided, the database field will be used. If both 'database' and 'metastore_db_name' are set, 'database' is used for filtering.
Default: None
| +|
mode
Enum
| One of: "hive", "presto", "presto-on-hive", "trino" | +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
schemas_where_clause_suffix
string
| DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.
Default:
| +|
simplify_nested_field_paths
boolean
| Simplify v2 field paths to v1. Falls back to v2 for Union/Array types.
Default: False
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
storage_platform_instance
One of string, null
| Platform instance for the storage system (e.g., 'my-s3-instance'). Used when generating URNs for storage datasets.
Default: None
| +|
tables_where_clause_suffix
string
| DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.
Default:
| +|
timeout_seconds
integer
| Connection timeout in seconds. Only for connection_type='thrift'.
Default: 60
| +|
use_catalog_subtype
boolean
| Use 'Catalog' (True) or 'Database' (False) as container subtype.
Default: True
| +|
use_dataset_pascalcase_subtype
boolean
| Use 'Table'/'View' (True) or 'table'/'view' (False) as dataset subtype.
Default: False
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
use_kerberos
boolean
| Whether to use Kerberos/SASL authentication. Only for connection_type='thrift'.
Default: False
| +|
username
One of string, null
| username
Default: None
| +|
views_where_clause_suffix
string
| DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.
Default:
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
schema_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
schema_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Configuration for stateful ingestion and stale entity removal.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "HiveMetastoreConfigMode": { + "description": "Mode for metadata extraction.", + "enum": [ + "hive", + "presto", + "presto-on-hive", + "trino" + ], + "title": "HiveMetastoreConfigMode", + "type": "string" + }, + "HiveMetastoreConnectionType": { + "description": "Connection type for HiveMetastoreSource.", + "enum": [ + "sql", + "thrift" + ], + "title": "HiveMetastoreConnectionType", + "type": "string" + }, + "LineageDirection": { + "description": "Direction of lineage relationship between storage and Hive", + "enum": [ + "upstream", + "downstream" + ], + "title": "LineageDirection", + "type": "string" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Configuration for Hive Metastore source.\n\nSupports two connection types:\n- sql: Direct database access (MySQL/PostgreSQL) to HMS backend\n- thrift: HMS Thrift API with Kerberos support", + "properties": { + "schema_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for schemas to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Configuration for stateful ingestion and stale entity removal." + }, + "emit_storage_lineage": { + "default": false, + "description": "Whether to emit storage-to-Hive lineage. When enabled, creates lineage relationships between Hive tables and their underlying storage locations (S3, Azure, GCS, HDFS, etc.).", + "title": "Emit Storage Lineage", + "type": "boolean" + }, + "hive_storage_lineage_direction": { + "$ref": "#/$defs/LineageDirection", + "default": "upstream", + "description": "Direction of storage lineage. If 'upstream', storage is treated as upstream to Hive (data flows from storage to Hive). If 'downstream', storage is downstream to Hive (data flows from Hive to storage)." + }, + "include_column_lineage": { + "default": true, + "description": "When enabled along with emit_storage_lineage, column-level lineage will be extracted between Hive table columns and storage location fields.", + "title": "Include Column Lineage", + "type": "boolean" + }, + "storage_platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Platform instance for the storage system (e.g., 'my-s3-instance'). Used when generating URNs for storage datasets.", + "title": "Storage Platform Instance" + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_table_location_lineage": { + "default": true, + "description": "If the source supports it, include table lineage to the underlying storage location.", + "title": "Include Table Location Lineage", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Extract lineage from Hive views by parsing view definitions.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "default": "localhost:3306", + "description": "Host and port. For SQL: database port (3306/5432). For Thrift: HMS Thrift port (9083).", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "connection_type": { + "$ref": "#/$defs/HiveMetastoreConnectionType", + "default": "sql", + "description": "Connection method: 'sql' for direct database access (MySQL/PostgreSQL), 'thrift' for HMS Thrift API with optional Kerberos support." + }, + "views_where_clause_suffix": { + "default": "", + "description": "DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.", + "title": "Views Where Clause Suffix", + "type": "string" + }, + "tables_where_clause_suffix": { + "default": "", + "description": "DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.", + "title": "Tables Where Clause Suffix", + "type": "string" + }, + "schemas_where_clause_suffix": { + "default": "", + "description": "DEPRECATED: This option has been deprecated for security reasons and will be removed in a future release. Use 'database_pattern' instead.", + "title": "Schemas Where Clause Suffix", + "type": "string" + }, + "metastore_db_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the Hive metastore's database (usually: metastore). For backward compatibility, if not provided, the database field will be used. If both 'database' and 'metastore_db_name' are set, 'database' is used for filtering.", + "title": "Metastore Db Name" + }, + "use_kerberos": { + "default": false, + "description": "Whether to use Kerberos/SASL authentication. Only for connection_type='thrift'.", + "title": "Use Kerberos", + "type": "boolean" + }, + "kerberos_service_name": { + "default": "hive", + "description": "Kerberos service name for the HMS principal. Only for connection_type='thrift'.", + "title": "Kerberos Service Name", + "type": "string" + }, + "kerberos_hostname_override": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Override hostname for Kerberos principal construction. Use when connecting through a load balancer. Only for connection_type='thrift'.", + "title": "Kerberos Hostname Override" + }, + "kerberos_qop": { + "default": "auth", + "description": "Kerberos Quality of Protection (QOP) for SASL authentication. Options: 'auth' (authentication only), 'auth-int' (authentication + integrity), 'auth-conf' (authentication + confidentiality/encryption). Must match the server's hadoop.rpc.protection setting. Only for connection_type='thrift'.", + "title": "Kerberos Qop", + "type": "string" + }, + "timeout_seconds": { + "default": 60, + "description": "Connection timeout in seconds. Only for connection_type='thrift'.", + "title": "Timeout Seconds", + "type": "integer" + }, + "catalog_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Catalog name for HMS 3.x multi-catalog deployments. Only for connection_type='thrift'.", + "title": "Catalog Name" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for databases to filter." + }, + "mode": { + "$ref": "#/$defs/HiveMetastoreConfigMode", + "default": "hive", + "description": "Platform mode for metadata. Valid options: ['hive', 'presto', 'presto-on-hive', 'trino']" + }, + "use_catalog_subtype": { + "default": true, + "description": "Use 'Catalog' (True) or 'Database' (False) as container subtype.", + "title": "Use Catalog Subtype", + "type": "boolean" + }, + "use_dataset_pascalcase_subtype": { + "default": false, + "description": "Use 'Table'/'View' (True) or 'table'/'view' (False) as dataset subtype.", + "title": "Use Dataset Pascalcase Subtype", + "type": "boolean" + }, + "include_catalog_name_in_ids": { + "default": false, + "description": "Add catalog name to dataset URNs. Example: urn:li:dataset:(urn:li:dataPlatform:hive,catalog.db.table,PROD)", + "title": "Include Catalog Name In Ids", + "type": "boolean" + }, + "enable_properties_merge": { + "default": true, + "description": "Merge properties with existing server data instead of overwriting.", + "title": "Enable Properties Merge", + "type": "boolean" + }, + "simplify_nested_field_paths": { + "default": false, + "description": "Simplify v2 field paths to v1. Falls back to v2 for Union/Array types.", + "title": "Simplify Nested Field Paths", + "type": "boolean" + }, + "ingestion_job_id": { + "default": "", + "title": "Ingestion Job Id", + "type": "string" + } + }, + "title": "HiveMetastore", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported metadata, lineage, and usage features for this module. + +### Limitations + +- Coverage depends on metadata visibility in both Presto and the Hive Metastore integration. +- Lineage and usage fidelity depend on query access and parsing compatibility. + +### Troubleshooting + +- Verify catalog connectivity and metadata permissions in Presto. +- Confirm metastore access and namespace/table visibility for the ingestion principal. +- Check ingestion logs for query parsing errors or permission-denied responses. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.hive.hive_metastore_source.HiveMetastoreSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/hive/hive_metastore_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Hive Metastore, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive.md new file mode 100644 index 00000000..842d086d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/hive.md @@ -0,0 +1,1459 @@ +--- +sidebar_position: 36 +title: Hive +slug: /generated/ingestion/sources/hive +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Hive + +## Overview + +Hive is a data platform used to store and query analytical or operational data. Learn more in the [official Hive documentation](https://hive.apache.org/). + +The DataHub integration for Hive covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `hive` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - Database, Schema. | +| [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md) | ✅ | Optionally enabled via `classification.enabled`. | +| Column-level Lineage | ✅ | Enabled by default for views via `include_view_column_lineage`, and to storage via `include_column_lineage` when storage lineage is enabled. Supported for types - Table, View. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ✅ | Supported via the `domain` config field. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default for views via `include_view_lineage`, and to upstream/downstream storage via `emit_storage_lineage`. Supported for types - Table, View. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `hive` module ingests metadata from Hive into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin extracts the following: + +- Metadata for databases, schemas, and tables +- Column types associated with each table +- Detailed table and storage information +- Table, row, and column statistics via optional SQL profiling. + +#### Related Documentation + +- [Hive Source Configuration](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/hive_recipe.yml) - Configuration examples +- [Hive Metastore Connector](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/hive-metastore) - Alternative connector for direct metastore access +- [PyHive Documentation](https://github.com/dropbox/PyHive) - Underlying connection library + +### Prerequisites + +1. **Network Access**: Access to HiveServer2 on port 10000 (or 10001 for TLS) + +2. **User Account**: Hive user with read permissions on target databases and tables + +3. **Dependencies**: Install PyHive connectivity: + ```bash + pip install 'acryl-datahub[hive]' + ``` + +#### Required Permissions + +The Hive user account used by DataHub needs the following permissions: + +##### Minimum Permissions (Metadata Only) + +```sql +-- Grant SELECT on all databases you want to ingest +GRANT SELECT ON DATABASE TO USER ; + +-- Grant SELECT on tables/views for schema extraction +GRANT SELECT ON TABLE .* TO USER ; +``` + +##### Additional Permissions for Storage Lineage + +If you plan to enable storage lineage, the connector needs to read table location information: + +```sql +-- Grant DESCRIBE on tables to read storage locations +GRANT SELECT ON .* TO USER ; +``` + +##### Recommendations + +- **Read-only Access**: DataHub only needs read permissions. Never grant `INSERT`, `UPDATE`, `DELETE`, or `DROP` privileges. +- **Database Filtering**: If you only need to ingest specific databases, use the `database` config parameter to limit scope and reduce the permissions required. + +#### Authentication + +The Hive connector supports multiple authentication methods through PyHive. Configure authentication using the recipe parameters described below. + +##### Basic Authentication (Username/Password) + +The simplest authentication method using a username and password: + +```yaml +source: + type: hive + config: + host_port: hive.company.com:10000 + username: datahub_user + password: ${HIVE_PASSWORD} # Use environment variables for sensitive data +``` + +##### LDAP Authentication + +For LDAP-based authentication: + +```yaml +source: + type: hive + config: + host_port: hive.company.com:10000 + username: datahub_user + password: ${LDAP_PASSWORD} + options: + connect_args: + auth: LDAP +``` + +##### Kerberos Authentication + +For Kerberos-secured Hive clusters: + +```yaml +source: + type: hive + config: + host_port: hive.company.com:10000 + options: + connect_args: + auth: KERBEROS + kerberos_service_name: hive +``` + +**Requirements**: + +- Valid Kerberos ticket (use `kinit` before running ingestion) +- Kerberos configuration file (`/etc/krb5.conf` or specified via `KRB5_CONFIG` environment variable) +- PyKerberos or requests-kerberos package installed + +##### TLS/SSL Connection + +For secure connections over HTTPS: + +```yaml +source: + type: hive + config: + host_port: hive.company.com:10001 + scheme: "hive+https" + username: datahub_user + password: ${HIVE_PASSWORD} + options: + connect_args: + auth: BASIC +``` + +##### Azure HDInsight + +For Microsoft Azure HDInsight clusters: + +```yaml +source: + type: hive + config: + host_port: .azurehdinsight.net:443 + scheme: "hive+https" + username: admin + password: ${HDINSIGHT_PASSWORD} + options: + connect_args: + http_path: "/hive2" + auth: BASIC +``` + +##### Databricks (via PyHive) + +For Databricks clusters using the Hive connector: + +```yaml +source: + type: hive + config: + host_port: :443 + scheme: "databricks+pyhive" + username: token # or your Databricks username + password: ${DATABRICKS_TOKEN} # Personal access token or password + options: + connect_args: + http_path: "sql/protocolv1/o/xxxyyyzzzaaasa/1234-567890-hello123" +``` + +**Note**: For comprehensive Databricks support, consider using the dedicated [Databricks Unity Catalog](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/databricks) connector instead, which provides enhanced features. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[hive]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: hive + config: + # Coordinates + host_port: localhost:10000 + database: DemoDatabase # optional, if not specified, ingests from all databases + + # Credentials + username: user # optional + password: pass # optional + + # For more details on authentication, see the PyHive docs: + # https://github.com/dropbox/PyHive#passing-session-configuration. + # LDAP, Kerberos, etc. are supported using connect_args, which can be + # added under the `options` config parameter. + #options: + # connect_args: + # auth: KERBEROS + # kerberos_service_name: hive + #scheme: 'hive+http' # set this if Thrift should use the HTTP transport + #scheme: 'hive+https' # set this if Thrift should use the HTTP with SSL transport + #scheme: 'sparksql' # set this for Spark Thrift Server + + # Storage Lineage Configuration (Optional) + # Enables lineage between Hive tables and their underlying storage locations + #emit_storage_lineage: false # Set to true to enable storage lineage + #hive_storage_lineage_direction: upstream # Direction: 'upstream' (storage -> Hive) or 'downstream' (Hive -> storage) + #include_column_lineage: true # Set to false to disable column-level lineage + #storage_platform_instance: "prod-s3" # Optional: platform instance for storage URNs + +sink: + # sink configs + +# --------------------------------------------------------- +# Recipe (Azure HDInsight) +# Connecting to Microsoft Azure HDInsight using TLS. +# --------------------------------------------------------- + +source: + type: hive + config: + # Coordinates + host_port: .azurehdinsight.net:443 + + # Credentials + username: admin + password: password + + # Options + options: + connect_args: + http_path: "/hive2" + auth: BASIC + +sink: + # sink configs + +# --------------------------------------------------------- +# Recipe (Databricks) +# Ensure that databricks-dbapi is installed. If not, use ```pip install databricks-dbapi``` to install. +# Use the ```http_path``` from your Databricks cluster in the following recipe. +# See (https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url) for instructions to find ```http_path```. +# --------------------------------------------------------- + +source: + type: hive + config: + host_port: :443 + username: token / username + password: / password + scheme: 'databricks+pyhive' + + options: + connect_args: + http_path: 'sql/protocolv1/o/xxxyyyzzzaaasa/1234-567890-hello123' + +sink: + # sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
host_port ✅
string
| host URL | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
database
One of string, null
| database (catalog)
Default: None
| +|
emit_storage_lineage
boolean
| Whether to emit storage-to-Hive lineage. When enabled, creates lineage relationships between Hive tables and their underlying storage locations (S3, Azure, GCS, HDFS, etc.).
Default: False
| +|
hive_storage_lineage_direction
Enum
| One of: "upstream", "downstream" | +|
include_column_lineage
boolean
| When enabled along with emit_storage_lineage, column-level lineage will be extracted between Hive table columns and storage location fields.
Default: True
| +|
include_tables
boolean
| Whether tables should be ingested.
Default: True
| +|
include_view_column_lineage
boolean
| Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.
Default: True
| +|
include_view_lineage
boolean
| Populates view->view and table->view lineage using DataHub's sql parser.
Default: True
| +|
include_views
boolean
| Whether views should be ingested.
Default: True
| +|
incremental_lineage
boolean
| When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.
Default: False
| +|
options
object
| Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`. | +|
password
One of string(password), null
| password
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
sqlalchemy_uri
One of string, null
| URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.
Default: None
| +|
storage_platform_instance
One of string, null
| Platform instance for the storage system (e.g., 'my-s3-instance'). Used when generating URNs for storage datasets.
Default: None
| +|
use_file_backed_cache
boolean
| Whether to use a file backed cache for the view definitions.
Default: True
| +|
username
One of string, null
| username
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
database_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
database_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
profile_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
profile_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
view_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
view_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification
ClassificationConfig
| | +|
classification.enabled
boolean
| Whether classification should be used to auto-detect glossary terms
Default: False
| +|
classification.info_type_to_term
map(str,string)
| | +|
classification.max_workers
integer
| Number of worker processes to use for classification. Set to 1 to disable.
Default: 10
| +|
classification.sample_size
integer
| Number of sample values used for classification.
Default: 100
| +|
classification.classifiers
array
| Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.
Default: [{'type': 'datahub', 'config': None}]
| +|
classification.classifiers.DynamicTypedClassifierConfig
DynamicTypedClassifierConfig
| | +|
classification.classifiers.DynamicTypedClassifierConfig.type ❓
string
| The type of the classifier to use. For DataHub, use `datahub` | +|
classification.classifiers.DynamicTypedClassifierConfig.config
One of object, null
| The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.
Default: None
| +|
classification.column_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.column_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
classification.table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
classification.table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
GEProfilingConfig
| | +|
profiling.catch_exceptions
boolean
|
Default: True
| +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.field_sample_values_limit
integer
| Upper limit for number of sample values to collect for all columns.
Default: 20
| +|
profiling.include_field_distinct_count
boolean
| Whether to profile for the number of distinct values for each column.
Default: True
| +|
profiling.include_field_distinct_value_frequencies
boolean
| Whether to profile for distinct value frequencies.
Default: False
| +|
profiling.include_field_histogram
boolean
| Whether to profile for the histogram for numeric fields.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_mean_value
boolean
| Whether to profile for the mean value of numeric columns.
Default: True
| +|
profiling.include_field_median_value
boolean
| Whether to profile for the median value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.include_field_quantiles
boolean
| Whether to profile for the quantiles of numeric columns.
Default: False
| +|
profiling.include_field_sample_values
boolean
| Whether to profile for the sample values for all columns.
Default: True
| +|
profiling.include_field_stddev_value
boolean
| Whether to profile for the standard deviation of numeric columns.
Default: True
| +|
profiling.limit
One of integer, null
| Max number of documents to profile. By default, profiles all documents.
Default: None
| +|
profiling.max_number_of_fields_to_profile
One of integer, null
| A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.
Default: None
| +|
profiling.max_workers
integer
| Number of worker threads to use for profiling. Set to 1 to disable.
Default: 50
| +|
profiling.method
Enum
| One of: "ge", "sqlalchemy"
Default: ge
| +|
profiling.offset
One of integer, null
| Offset in documents to profile. By default, uses no offset.
Default: None
| +|
profiling.partition_datetime
One of string(date-time), null
| If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.
Default: None
| +|
profiling.partition_profiling_enabled
boolean
| Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.
Default: True
| +|
profiling.profile_external_tables
boolean
| Whether to profile external tables. Only Snowflake and Redshift supports this.
Default: False
| +|
profiling.profile_if_updated_since_days
One of number, null
| Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.
Default: None
| +|
profiling.profile_nested_fields
boolean
| Whether to profile complex types like structs, arrays and maps.
Default: False
| +|
profiling.profile_table_level_only
boolean
| Whether to perform profiling at table-level only, or include column-level profiling as well.
Default: False
| +|
profiling.profile_table_row_count_estimate_only
boolean
| Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL.
Default: False
| +|
profiling.profile_table_row_limit
One of integer, null
| Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.
Default: 5000000
| +|
profiling.profile_table_size_limit
One of integer, null
| Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.
Default: 5
| +|
profiling.query_combiner_enabled
boolean
| *This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.
Default: True
| +|
profiling.report_dropped_profiles
boolean
| Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.
Default: False
| +|
profiling.sample_size
integer
| Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.
Default: 10000
| +|
profiling.turn_off_expensive_profiling_metrics
boolean
| Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.
Default: False
| +|
profiling.use_sampling
boolean
| Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.tags_to_ignore_sampling
One of array, null
| Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.
Default: None
| +|
profiling.tags_to_ignore_sampling.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "ClassificationConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether classification should be used to auto-detect glossary terms", + "title": "Enabled", + "type": "boolean" + }, + "sample_size": { + "default": 100, + "description": "Number of sample values used for classification.", + "title": "Sample Size", + "type": "integer" + }, + "max_workers": { + "default": 10, + "description": "Number of worker processes to use for classification. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "column_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format." + }, + "info_type_to_term": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Optional mapping to provide glossary term identifier for info type", + "title": "Info Type To Term", + "type": "object" + }, + "classifiers": { + "default": [ + { + "type": "datahub", + "config": null + } + ], + "description": "Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance.", + "items": { + "$ref": "#/$defs/DynamicTypedClassifierConfig" + }, + "title": "Classifiers", + "type": "array" + } + }, + "title": "ClassificationConfig", + "type": "object" + }, + "DynamicTypedClassifierConfig": { + "additionalProperties": false, + "properties": { + "type": { + "description": "The type of the classifier to use. For DataHub, use `datahub`", + "title": "Type", + "type": "string" + }, + "config": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "default": null, + "description": "The configuration required for initializing the classifier. If not specified, uses defaults for classifer type.", + "title": "Config" + } + }, + "required": [ + "type" + ], + "title": "DynamicTypedClassifierConfig", + "type": "object" + }, + "GEProfilingConfig": { + "additionalProperties": false, + "properties": { + "method": { + "default": "ge", + "description": "Profiling method to use. Options: `ge` (Great Expectations) or `sqlalchemy` (custom SQLAlchemy-based profiler). The SQLAlchemy profiler has no GE dependency and provides the same functionality.", + "enum": [ + "ge", + "sqlalchemy" + ], + "title": "Method", + "type": "string" + }, + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + }, + "limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Max number of documents to profile. By default, profiles all documents.", + "title": "Limit" + }, + "offset": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Offset in documents to profile. By default, uses no offset.", + "title": "Offset" + }, + "profile_table_level_only": { + "default": false, + "description": "Whether to perform profiling at table-level only, or include column-level profiling as well.", + "title": "Profile Table Level Only", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_distinct_count": { + "default": true, + "description": "Whether to profile for the number of distinct values for each column.", + "title": "Include Field Distinct Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "include_field_mean_value": { + "default": true, + "description": "Whether to profile for the mean value of numeric columns.", + "title": "Include Field Mean Value", + "type": "boolean" + }, + "include_field_median_value": { + "default": true, + "description": "Whether to profile for the median value of numeric columns.", + "title": "Include Field Median Value", + "type": "boolean" + }, + "include_field_stddev_value": { + "default": true, + "description": "Whether to profile for the standard deviation of numeric columns.", + "title": "Include Field Stddev Value", + "type": "boolean" + }, + "include_field_quantiles": { + "default": false, + "description": "Whether to profile for the quantiles of numeric columns.", + "title": "Include Field Quantiles", + "type": "boolean" + }, + "include_field_distinct_value_frequencies": { + "default": false, + "description": "Whether to profile for distinct value frequencies.", + "title": "Include Field Distinct Value Frequencies", + "type": "boolean" + }, + "include_field_histogram": { + "default": false, + "description": "Whether to profile for the histogram for numeric fields.", + "title": "Include Field Histogram", + "type": "boolean" + }, + "include_field_sample_values": { + "default": true, + "description": "Whether to profile for the sample values for all columns.", + "title": "Include Field Sample Values", + "type": "boolean" + }, + "max_workers": { + "default": 50, + "description": "Number of worker threads to use for profiling. Set to 1 to disable.", + "title": "Max Workers", + "type": "integer" + }, + "report_dropped_profiles": { + "default": false, + "description": "Whether to report datasets or dataset columns which were not profiled. Set to `True` for debugging purposes.", + "title": "Report Dropped Profiles", + "type": "boolean" + }, + "turn_off_expensive_profiling_metrics": { + "default": false, + "description": "Whether to turn off expensive profiling or not. This turns off profiling for quantiles, distinct_value_frequencies, histogram & sample_values. This also limits maximum number of fields being profiled to 10.", + "title": "Turn Off Expensive Profiling Metrics", + "type": "boolean" + }, + "field_sample_values_limit": { + "default": 20, + "description": "Upper limit for number of sample values to collect for all columns.", + "title": "Field Sample Values Limit", + "type": "integer" + }, + "max_number_of_fields_to_profile": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A positive integer that specifies the maximum number of columns to profile for any table. `None` implies all columns. The cost of profiling goes up significantly as the number of columns to profile goes up.", + "title": "Max Number Of Fields To Profile" + }, + "profile_if_updated_since_days": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Profile table only if it has been updated since these many number of days. If set to `null`, no constraint of last modified time for tables to profile. Supported only in `snowflake` and `BigQuery`.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery" + ] + }, + "title": "Profile If Updated Since Days" + }, + "profile_table_size_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5, + "description": "Profile tables only if their size is less than specified GBs. If set to `null`, no limit on the size of tables to profile. Supported only in `Snowflake`, `BigQuery` and `Databricks`. Supported for `Oracle` based on calculated size from gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "unity-catalog", + "oracle" + ] + }, + "title": "Profile Table Size Limit" + }, + "profile_table_row_limit": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": 5000000, + "description": "Profile tables only if their row count is less than specified count. If set to `null`, no limit on the row count of tables to profile. Supported only in `Snowflake`, `BigQuery`. Supported for `Oracle` based on gathered stats.", + "schema_extra": { + "supported_sources": [ + "snowflake", + "bigquery", + "oracle" + ] + }, + "title": "Profile Table Row Limit" + }, + "profile_table_row_count_estimate_only": { + "default": false, + "description": "Use an approximate query for row count. This will be much faster but slightly less accurate. Only supported for Postgres and MySQL. ", + "schema_extra": { + "supported_sources": [ + "postgres", + "mysql" + ] + }, + "title": "Profile Table Row Count Estimate Only", + "type": "boolean" + }, + "query_combiner_enabled": { + "default": true, + "description": "*This feature is still experimental and can be disabled if it causes issues.* Reduces the total number of queries issued and speeds up profiling by dynamically combining SQL queries where possible.", + "title": "Query Combiner Enabled", + "type": "boolean" + }, + "catch_exceptions": { + "default": true, + "description": "", + "title": "Catch Exceptions", + "type": "boolean" + }, + "partition_profiling_enabled": { + "default": true, + "description": "Whether to profile partitioned tables. Only BigQuery and Aws Athena supports this. If enabled, latest partition data is used for profiling.", + "schema_extra": { + "supported_sources": [ + "athena", + "bigquery" + ] + }, + "title": "Partition Profiling Enabled", + "type": "boolean" + }, + "partition_datetime": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "If specified, profile only the partition which matches this datetime. If not specified, profile the latest partition. Only Bigquery supports this.", + "schema_extra": { + "supported_sources": [ + "bigquery" + ] + }, + "title": "Partition Datetime" + }, + "use_sampling": { + "default": true, + "description": "Whether to profile column level stats on sample of table. Only BigQuery and Snowflake support this. If enabled, profiling is done on rows sampled from table. Sampling is not done for smaller tables. ", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Use Sampling", + "type": "boolean" + }, + "sample_size": { + "default": 10000, + "description": "Number of rows to be sampled from table for column level profiling.Applicable only if `use_sampling` is set to True.", + "schema_extra": { + "supported_sources": [ + "bigquery", + "snowflake" + ] + }, + "title": "Sample Size", + "type": "integer" + }, + "profile_external_tables": { + "default": false, + "description": "Whether to profile external tables. Only Snowflake and Redshift supports this.", + "schema_extra": { + "supported_sources": [ + "redshift", + "snowflake" + ] + }, + "title": "Profile External Tables", + "type": "boolean" + }, + "tags_to_ignore_sampling": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Fixed list of tags to ignore sampling. If not specified, tables will be sampled based on `use_sampling`.", + "title": "Tags To Ignore Sampling" + }, + "profile_nested_fields": { + "default": false, + "description": "Whether to profile complex types like structs, arrays and maps. ", + "title": "Profile Nested Fields", + "type": "boolean" + } + }, + "title": "GEProfilingConfig", + "type": "object" + }, + "LineageDirection": { + "description": "Direction of lineage relationship between storage and Hive", + "enum": [ + "upstream", + "downstream" + ], + "title": "LineageDirection", + "type": "string" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "emit_storage_lineage": { + "default": false, + "description": "Whether to emit storage-to-Hive lineage. When enabled, creates lineage relationships between Hive tables and their underlying storage locations (S3, Azure, GCS, HDFS, etc.).", + "title": "Emit Storage Lineage", + "type": "boolean" + }, + "hive_storage_lineage_direction": { + "$ref": "#/$defs/LineageDirection", + "default": "upstream", + "description": "Direction of storage lineage. If 'upstream', storage is treated as upstream to Hive (data flows from storage to Hive). If 'downstream', storage is downstream to Hive (data flows from Hive to storage)." + }, + "include_column_lineage": { + "default": true, + "description": "When enabled along with emit_storage_lineage, column-level lineage will be extracted between Hive table columns and storage location fields.", + "title": "Include Column Lineage", + "type": "boolean" + }, + "storage_platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Platform instance for the storage system (e.g., 'my-s3-instance'). Used when generating URNs for storage datasets.", + "title": "Storage Platform Instance" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion. Specify regex to match the entire table name in database.schema.table format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "view_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for views to filter in ingestion. Note: Defaults to table_pattern if not specified. Specify regex to match the entire view name in database.schema.view format. e.g. to match all views starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.*'" + }, + "classification": { + "$ref": "#/$defs/ClassificationConfig", + "default": { + "enabled": false, + "sample_size": 100, + "max_workers": 10, + "table_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "column_pattern": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "info_type_to_term": {}, + "classifiers": [ + { + "config": null, + "type": "datahub" + } + ] + }, + "description": "For details, refer to [Classification](../../../../metadata-ingestion/docs/dev_guides/classification.md)." + }, + "incremental_lineage": { + "default": false, + "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", + "title": "Incremental Lineage", + "type": "boolean" + }, + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "options": { + "additionalProperties": true, + "description": "Any options specified here will be passed to [SQLAlchemy.create_engine](https://docs.sqlalchemy.org/en/14/core/engines.html#sqlalchemy.create_engine) as kwargs. To set connection arguments in the URL, specify them under `connect_args`.", + "title": "Options", + "type": "object" + }, + "profile_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns to filter tables (or specific columns) for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered." + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "Attach domains to databases, schemas or tables during ingestion using regex patterns. Domain key can be a guid like *urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba* or a string like \"Marketing\".) If you provide strings, then datahub will attempt to resolve this name to a guid, and will error out if this fails. There can be multiple domain keys specified.", + "title": "Domain", + "type": "object" + }, + "include_views": { + "default": true, + "description": "Whether views should be ingested.", + "title": "Include Views", + "type": "boolean" + }, + "include_tables": { + "default": true, + "description": "Whether tables should be ingested.", + "title": "Include Tables", + "type": "boolean" + }, + "include_view_lineage": { + "default": true, + "description": "Populates view->view and table->view lineage using DataHub's sql parser.", + "title": "Include View Lineage", + "type": "boolean" + }, + "include_view_column_lineage": { + "default": true, + "description": "Populates column-level lineage for view->view and table->view lineage using DataHub's sql parser. Requires `include_view_lineage` to be enabled.", + "title": "Include View Column Lineage", + "type": "boolean" + }, + "use_file_backed_cache": { + "default": true, + "description": "Whether to use a file backed cache for the view definitions.", + "title": "Use File Backed Cache", + "type": "boolean" + }, + "profiling": { + "$ref": "#/$defs/GEProfilingConfig", + "default": { + "method": "ge", + "enabled": false, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + }, + "limit": null, + "offset": null, + "profile_table_level_only": false, + "include_field_null_count": true, + "include_field_distinct_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "include_field_mean_value": true, + "include_field_median_value": true, + "include_field_stddev_value": true, + "include_field_quantiles": false, + "include_field_distinct_value_frequencies": false, + "include_field_histogram": false, + "include_field_sample_values": true, + "max_workers": 50, + "report_dropped_profiles": false, + "turn_off_expensive_profiling_metrics": false, + "field_sample_values_limit": 20, + "max_number_of_fields_to_profile": null, + "profile_if_updated_since_days": null, + "profile_table_size_limit": 5, + "profile_table_row_limit": 5000000, + "profile_table_row_count_estimate_only": false, + "query_combiner_enabled": true, + "catch_exceptions": true, + "partition_profiling_enabled": true, + "partition_datetime": null, + "use_sampling": true, + "sample_size": 10000, + "profile_external_tables": false, + "tags_to_ignore_sampling": null, + "profile_nested_fields": false + } + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "username", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "password", + "title": "Password" + }, + "host_port": { + "description": "host URL", + "title": "Host Port", + "type": "string" + }, + "database": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "database (catalog)", + "title": "Database" + }, + "sqlalchemy_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "URI of database to connect to. See https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls. Takes precedence over other connection parameters.", + "title": "Sqlalchemy Uri" + }, + "database_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for databases to filter in ingestion." + } + }, + "required": [ + "host_port" + ], + "title": "HiveConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Storage Lineage + +DataHub can extract lineage between Hive tables and their underlying storage locations (S3, Azure Blob, HDFS, GCS, etc.). This feature creates relationships showing data flow from raw storage to Hive tables. + +##### Quick Start + +Enable storage lineage with minimal configuration: + +```yaml +source: + type: hive + config: + host_port: hive.company.com:10000 + username: datahub_user + password: ${HIVE_PASSWORD} + + # Enable storage lineage + emit_storage_lineage: true +``` + +This will: + +- Extract storage locations from Hive table metadata +- Create lineage from storage (S3, HDFS, etc.) to Hive tables +- Include column-level lineage by default + +##### Configuration Options + +Storage lineage behavior is controlled by four parameters: + +| Parameter | Type | Default | Description | +| -------------------------------- | ------- | ------------ | --------------------------------------------------------------------------- | +| `emit_storage_lineage` | boolean | `false` | Master toggle to enable/disable storage lineage | +| `hive_storage_lineage_direction` | string | `"upstream"` | Direction: `"upstream"` (storage → Hive) or `"downstream"` (Hive → storage) | +| `include_column_lineage` | boolean | `true` | Enable column-level lineage from storage paths to Hive columns | +| `storage_platform_instance` | string | `None` | Platform instance for storage URNs (e.g., `"prod-s3"`, `"dev-hdfs"`) | + +##### Supported Storage Platforms + +The connector automatically detects and creates lineage for: + +- **Amazon S3**: `s3://`, `s3a://`, `s3n://` +- **HDFS**: `hdfs://` +- **Google Cloud Storage**: `gs://` +- **Azure Blob Storage**: `wasb://`, `wasbs://` +- **Azure Data Lake (Gen1)**: `adl://` +- **Azure Data Lake (Gen2)**: `abfs://`, `abfss://` +- **Databricks File System**: `dbfs://` +- **Local File System**: `file://` or absolute paths + +#### Platform Instances + +When ingesting from multiple Hive environments (e.g., production, staging, development), use `platform_instance` to distinguish them: + +```yaml +source: + type: hive + config: + host_port: prod-hive.company.com:10000 + platform_instance: "prod-hive" +``` + +This creates URNs like: + +``` +urn:li:dataset:(urn:li:dataPlatform:hive,database.table,prod-hive) +``` + +**Best Practice**: Combine with `storage_platform_instance` for complete environment isolation: + +```yaml +source: + type: hive + config: + platform_instance: "prod-hive" # Hive environment + storage_platform_instance: "prod-s3" # Storage environment + emit_storage_lineage: true +``` + +#### Large Hive Deployments + +For Hive clusters with thousands of tables, consider: + +1. **Database Filtering**: Limit ingestion to specific databases: + + ```yaml + database: "production_db" # Only ingest one database + ``` + +2. **Incremental Ingestion**: Use DataHub's stateful ingestion to only process changes: + + ```yaml + stateful_ingestion: + enabled: true + remove_stale_metadata: true + ``` + +3. **Disable Column Lineage**: If not needed, disable to improve performance: + + ```yaml + emit_storage_lineage: true + include_column_lineage: false # Faster ingestion + ``` + +4. **Connection Pooling**: The connector uses a single connection by default. For better performance with large schemas, ensure your HiveServer2 has sufficient resources. + +#### Network Latency + +- If DataHub is running far from your Hive cluster, expect slower metadata extraction +- Consider running ingestion from a machine closer to your Hive infrastructure +- Use scheduled ingestion during off-peak hours for large deployments + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +#### Hive Version Support + +- **Supported Versions**: Hive 1.x, 2.x, and 3.x +- **HiveServer2 Required**: The connector connects to HiveServer2, not the metastore database directly +- For direct metastore access, use the [Hive Metastore](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/hive-metastore) connector instead + +#### View Definitions + +- **Simple Views**: Fully supported with SQL lineage extraction +- **Complex Views**: Views with complex SQL (CTEs, subqueries) are supported via SQL parsing +- **Presto/Trino Views**: Not directly accessible via this connector. Use the [Hive Metastore](https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/hive-metastore) connector for Presto/Trino view support + +#### Storage Lineage Limitations + +- **Location Required**: Only tables with defined storage locations (`LOCATION` clause) will have storage lineage +- **External Tables**: Best supported (always have explicit locations) +- **Managed Tables**: Lineage created for default warehouse locations +- **Temporary Tables**: Not supported (no persistent storage location) + +#### Partitioned Tables + +- Partition information is extracted and included in schema metadata +- Partition-level storage lineage is aggregated at the table level +- Individual partition lineage is not currently supported + +#### Authentication Limitations + +- **No Proxy Support**: Direct connection to HiveServer2 required +- **Token-Based Auth**: Not natively supported (use Kerberos or basic auth) +- **Multi-Factor Authentication**: Not supported + +### Troubleshooting + +1. **Session Timeout**: Long-running ingestion may hit HiveServer2 session timeouts. Configure `hive.server2.session.timeout` appropriately on the Hive side. + +2. **Large Schemas**: Tables with 1000+ columns may be slow to ingest due to schema extraction overhead. + +3. **Case Sensitivity**: + - Hive is case-insensitive by default + - DataHub automatically lowercases URNs for consistency +4. **View Lineage Parsing**: Complex views using non-standard SQL may not have complete lineage extracted. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.sql.hive.hive_source.HiveSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/hive/hive_source.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Hive, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/iceberg.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/iceberg.md new file mode 100644 index 00000000..dc13f0af --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/iceberg.md @@ -0,0 +1,894 @@ +--- +sidebar_position: 39 +title: Iceberg +slug: /generated/ingestion/sources/iceberg +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Iceberg + +## Overview + +Apache Iceberg is a data platform used to store and query analytical or operational data. Learn more in the [official Apache Iceberg documentation](https://iceberg.apache.org/). + +The DataHub integration for Apache Iceberg covers core metadata entities such as datasets/tables/views, schema fields, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `iceberg` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ✅ | Optionally enabled via configuration. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Domains](../../../domains.md) | ❌ | Currently not supported. | +| Extract Ownership | ✅ | Automatically ingests ownership information from table properties based on `user_ownership_property` and `group_ownership_property`. | +| Partition Support | ❌ | Currently not supported. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Optionally enabled via configuration, an Iceberg instance represents the catalog name where the table is stored. | + +### Overview + +The `iceberg` module ingests metadata from Iceberg into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[iceberg]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: iceberg + config: + catalog: + my_catalog: + type: "glue" + s3.region: "us-west-2" + region_name: "us-west-2" + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
catalog ✅
map(str,object)
| | +|
group_ownership_property
One of string, null
| Iceberg table property to look for a `CorpGroup` owner. Can only hold a single group value. If property has no value, no owner information will be emitted.
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
processing_threads
integer
| How many threads will be processing tables
Default: 1
| +|
user_ownership_property
One of string, null
| Iceberg table property to look for a `CorpUser` owner. Can only hold a single user value. If property has no value, no owner information will be emitted.
Default: owner
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
namespace_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
namespace_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
table_pattern
AllowDenyPattern
| A class to store allow deny regexes | +|
table_pattern.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
profiling
IcebergProfilingConfig
| | +|
profiling.enabled
boolean
| Whether profiling should be done.
Default: False
| +|
profiling.include_field_max_value
boolean
| Whether to profile for the max value of numeric columns.
Default: True
| +|
profiling.include_field_min_value
boolean
| Whether to profile for the min value of numeric columns.
Default: True
| +|
profiling.include_field_null_count
boolean
| Whether to profile for the number of nulls for each column.
Default: True
| +|
profiling.operation_config
OperationConfig
| | +|
profiling.operation_config.lower_freq_profile_enabled
boolean
| Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.
Default: False
| +|
profiling.operation_config.profile_date_of_month
One of integer, null
| Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
profiling.operation_config.profile_day_of_week
One of integer, null
| Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.
Default: None
| +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
| Iceberg Stateful Ingestion Config.
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "IcebergProfilingConfig": { + "additionalProperties": false, + "properties": { + "enabled": { + "default": false, + "description": "Whether profiling should be done.", + "title": "Enabled", + "type": "boolean" + }, + "include_field_null_count": { + "default": true, + "description": "Whether to profile for the number of nulls for each column.", + "title": "Include Field Null Count", + "type": "boolean" + }, + "include_field_min_value": { + "default": true, + "description": "Whether to profile for the min value of numeric columns.", + "title": "Include Field Min Value", + "type": "boolean" + }, + "include_field_max_value": { + "default": true, + "description": "Whether to profile for the max value of numeric columns.", + "title": "Include Field Max Value", + "type": "boolean" + }, + "operation_config": { + "$ref": "#/$defs/OperationConfig", + "description": "Experimental feature. To specify operation configs." + } + }, + "title": "IcebergProfilingConfig", + "type": "object" + }, + "OperationConfig": { + "additionalProperties": false, + "properties": { + "lower_freq_profile_enabled": { + "default": false, + "description": "Whether to do profiling at lower freq or not. This does not do any scheduling just adds additional checks to when not to run profiling.", + "title": "Lower Freq Profile Enabled", + "type": "boolean" + }, + "profile_day_of_week": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 0 to 6 for day of week (both inclusive). 0 is Monday and 6 is Sunday. If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Day Of Week" + }, + "profile_date_of_month": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Number between 1 to 31 for date of month (both inclusive). If not specified, defaults to Nothing and this field does not take affect.", + "title": "Profile Date Of Month" + } + }, + "title": "OperationConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Iceberg Stateful Ingestion Config." + }, + "catalog": { + "additionalProperties": { + "additionalProperties": true, + "type": "object" + }, + "description": "Catalog configuration where to find Iceberg tables. Only one catalog specification is supported. The format is the same as [pyiceberg's catalog configuration](https://py.iceberg.apache.org/configuration/), where the catalog name is specified as the object name and attributes are set as key-value pairs.", + "title": "Catalog", + "type": "object" + }, + "table_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for tables to filter in ingestion." + }, + "namespace_pattern": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "Regex patterns for namespaces to filter in ingestion." + }, + "user_ownership_property": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "owner", + "description": "Iceberg table property to look for a `CorpUser` owner. Can only hold a single user value. If property has no value, no owner information will be emitted.", + "title": "User Ownership Property" + }, + "group_ownership_property": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Iceberg table property to look for a `CorpGroup` owner. Can only hold a single group value. If property has no value, no owner information will be emitted.", + "title": "Group Ownership Property" + }, + "profiling": { + "$ref": "#/$defs/IcebergProfilingConfig", + "default": { + "enabled": false, + "include_field_null_count": true, + "include_field_min_value": true, + "include_field_max_value": true, + "operation_config": { + "lower_freq_profile_enabled": false, + "profile_date_of_month": null, + "profile_day_of_week": null + } + } + }, + "processing_threads": { + "default": 1, + "description": "How many threads will be processing tables", + "title": "Processing Threads", + "type": "integer" + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "description": "A map of domain names to allow/deny patterns for tables and namespaces. Domain key can be a guid like \"urn:li:domain:ec428203-ce86-4db3-985d-5a8ee6df32ba\" or a string like \"Engineering\". If multiple patterns match, at most one domain is assigned to the entity.", + "title": "Domain", + "type": "object" + } + }, + "required": [ + "catalog" + ], + "title": "IcebergSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Setting up connection to an Iceberg catalog + +There are multiple servers compatible with the Iceberg Catalog specification. DataHub's `iceberg` connector uses `pyiceberg` +library to extract metadata from them. The recipe for the source consists of 2 parts: + +1. `catalog` part which is passed as-is to the `pyiceberg` library and configures the connection and its details (i.e. authentication). + The name of catalog specified in the recipe has no consequence, it is just a formal requirement from the library. + Only one catalog will be considered for the ingestion. +2. The remaining configuration consists of parameters, such as `env` or `stateful_ingestion` which are standard + DataHub's ingestor configuration parameters and are described in the [Config Details](#config-details) chapter. + +This chapter showcases several examples of setting up connections to an Iceberg catalog, varying based on the underlying +implementation. Iceberg is designed to have catalog and warehouse separated, which is reflected in how we configure it. +It is especially visible when using Iceberg REST Catalog - which can use many blob storages +(AWS S3, Azure Blob Storage, MinIO) as a warehouse. + +Note that, for advanced users, it is possible to specify a custom catalog client implementation via `py-catalog-impl` +configuration option - refer to `pyiceberg` documentation on details. + +#### Glue catalog + S3 warehouse + +The minimal configuration for connecting to Glue catalog with S3 warehouse: + +```yaml +source: + type: "iceberg" + config: + catalog: + my_catalog: + type: "glue" + s3.region: "us-west-2" + region_name: "us-west-2" +``` + +Where `us-west-2` is the region from which you want to ingest. The above configuration will work assuming your pod or environment in which +you run your datahub CLI is already authenticated to AWS and has proper permissions granted (see below). If you need +to specify secrets directly, use the following configuration as the template: + +```yaml +source: + type: "iceberg" + config: + catalog: + demo: + type: "glue" + s3.region: "us-west-2" + s3.access-key-id: "${AWS_ACCESS_KEY_ID}" + s3.secret-access-key: "${AWS_SECRET_ACCESS_KEY}" + s3.session-token: "${AWS_SESSION_TOKEN}" + + aws_access_key_id: "${AWS_ACCESS_KEY_ID}" + aws_secret_access_key: "${AWS_SECRET_ACCESS_KEY}" + aws_session_token: "${AWS_SESSION_TOKEN}" + region_name: "us-west-2" +``` + +This example uses references to fill credentials (either from Secrets defined in Managed Ingestion or environmental variables). +It is possible (but not recommended due to security concerns) to provide those values in plaintext, directly in the recipe. + +##### Glue and S3 permissions required + +The role used by the ingestor for ingesting metadata from Glue Iceberg Catalog and S3 warehouse is: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["glue:GetDatabases", "glue:GetTables", "glue:GetTable"], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:ListBucket", "s3:GetObjectVersion"], + "Resource": [ + "arn:aws:s3:::", + "arn:aws:s3:::/*" + ] + } + ] +} +``` + +#### Iceberg REST Catalog + MinIO + +The following configuration assumes MinIO defines authentication using the `s3.*` prefix. Note the specification of `s3.endpoint`, assuming +MinIO listens on port `9000` at `minio-host`. The `uri` parameter points at Iceberg REST Catalog (IRC) endpoint (in this case `iceberg-catalog:8181`). + +```yaml +source: + type: "iceberg" + config: + catalog: + demo: + type: "rest" + uri: "http://iceberg-catalog:8181" + s3.access-key-id: "${AWS_ACCESS_KEY_ID}" + s3.secret-access-key: "${AWS_SECRET_ACCESS_KEY}" + s3.region: "eu-east-1" + s3.endpoint: "http://minio-host:9000" +``` + +#### Iceberg REST Catalog (with authentication) + S3 + +This example assumes IRC requires token authentication (via `Authorization` header). There are more options available, +see https://py.iceberg.apache.org/configuration/#rest-catalog for details. Moreover, the assumption here is that the +environment (i.e. pod) is already authenticated to perform actions against AWS S3. + +```yaml +source: + type: "iceberg" + config: + catalog: + demo: + type: "rest" + uri: "http://iceberg-catalog-uri" + token: "token-value" + s3.region: "us-west-2" +``` + +##### Special REST connection parameters for resiliency + +Unlike other parameters provided in the dictionary under the `catalog` key, `connection` parameter is a custom feature in +DataHub, allowing to inject connection resiliency parameters to the REST connection made by the ingestor. `connection` +allows for 2 parameters: + +- `timeout` is provided as amount of seconds, it needs to be whole number (or `null` to turn it off) +- `retry` is a complex object representing parameters used to create [urllib3 Retry object](https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#module-urllib3.util.retry). + There are many possible parameters, most important would be `total` (total retries) and `backoff_factor`. See the linked docs + for the details. + +```yaml +source: + type: "iceberg" + config: + catalog: + demo: + type: "rest" + uri: "http://iceberg-catalog-uri" + connection: + retry: + backoff_factor: 0.5 + total: 3 + timeout: 120 +``` + +#### Google BigLake REST Catalog + GCS warehouse + +DataHub supports ingesting metadata from [Google BigLake](https://cloud.google.com/bigquery/docs/biglake-intro) via the Iceberg REST Catalog API. +BigLake provides unified governance and security for data across data lakes and data warehouses. + +PyIceberg 0.9+ natively supports BigLake authentication using Google Cloud's Application Default Credentials (ADC). + +##### Prerequisites + +1. **GCP Project** with BigLake API enabled: + + ```bash + gcloud services enable biglake.googleapis.com --project=YOUR_PROJECT_ID + ``` + +2. **Service Account** with required permissions: + + - `biglake.catalogs.get` + - `biglake.tables.get` + - `biglake.tables.list` + - `biglake.databases.get` + - `biglake.databases.list` + - Storage Object Viewer (for GCS buckets containing Iceberg data) + +3. **BigLake Catalog** created in your GCP project: + ```bash + gcloud alpha biglake catalogs create CATALOG_NAME \ + --location=REGION \ + --project=PROJECT_ID + ``` + +##### Authentication Setup + +BigLake authentication uses Application Default Credentials (ADC). DataHub provides automatic OAuth scope fixing for seamless integration. + +**Setup:** + +```bash +export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json +export GCP_PROJECT_ID=your-project-id +export GCS_WAREHOUSE_BUCKET=your-bucket-name +``` + +##### Configuration + +```yaml +source: + type: iceberg + config: + catalog: + my_biglake_catalog: + type: rest + uri: https://biglake.googleapis.com/iceberg/v1/restcatalog + warehouse: gs://my-bucket + auth: + type: google + google: + scopes: + - https://www.googleapis.com/auth/cloud-platform + header.x-goog-user-project: my-project + header.X-Iceberg-Access-Delegation: "" # End-user credentials mode + connection: + timeout: 120 + retry: + total: 5 + backoff_factor: 0.3 +``` + +**Key Configuration Parameters:** + +- `auth.type: google` - Uses Application Default Credentials +- `auth.google.scopes` - OAuth scopes required for BigLake access +- `header.x-goog-user-project` - Specifies the GCP project for billing +- `header.X-Iceberg-Access-Delegation: ""` - Uses end-user credentials mode + +##### How Authentication Works + +When using `auth.type: google` with explicit scopes: + +1. **Discovers credentials** using Google Cloud's Application Default Credentials (ADC) chain: + + - **Environment Variable**: `GOOGLE_APPLICATION_CREDENTIALS` pointing to service account JSON (most common) + - **gcloud CLI**: Credentials from `gcloud auth application-default login` + - **GCE/GKE Metadata Server**: Automatic when running on Google Cloud infrastructure + - **Workload Identity**: Automatic when using GKE Workload Identity + +2. **Uses explicit OAuth scopes**: The `auth.google.scopes` configuration ensures the correct `cloud-platform` scope is used for BigLake access + + - PyIceberg passes the scopes directly to `google.auth.default()` + - Requires `google-auth` library to be installed (included in DataHub dependencies) + +3. **Handles token refresh**: Automatic token refresh with no manual management needed + +##### Using Managed Ingestion with Secrets + +For production environments using Managed Ingestion (via the DataHub UI), you can securely store your GCP service account credentials as DataHub secrets instead of using environment variables. + +**Step 1: Create a Secret in DataHub** + +1. Navigate to **Settings** > **Secrets** in the DataHub UI +2. Click **Create new secret** +3. Enter a name (e.g., `BIGLAKE_SERVICE_ACCOUNT_JSON`) +4. Paste the **entire contents** of your GCP service account JSON file as the value +5. Optionally add a description +6. Click **Create** + +**Step 2: Reference the Secret in Your Recipe** + +Use the `${SECRET_NAME}` syntax to reference your secret in the ingestion recipe: + +```yaml +source: + type: iceberg + config: + env: prod + catalog: + my_biglake_catalog: + type: rest + uri: https://biglake.googleapis.com/iceberg/v1/restcatalog + warehouse: gs://my-bucket + auth: + type: google + google: + credentials_json: ${BIGLAKE_SERVICE_ACCOUNT_JSON} + scopes: + - https://www.googleapis.com/auth/cloud-platform + header.x-goog-user-project: my-project + header.X-Iceberg-Access-Delegation: "" + +sink: + type: datahub-rest + config: + server: ${DATAHUB_GMS_URL} + token: ${DATAHUB_GMS_TOKEN} +``` + +The secret will be automatically resolved at runtime when the ingestion executes. + +**Alternative: Using Structured Credentials** + +You can also use individual secrets for each credential component, which provides better validation: + +```yaml +source: + type: iceberg + config: + catalog: + my_biglake_catalog: + type: rest + uri: https://biglake.googleapis.com/iceberg/v1/restcatalog + warehouse: gs://my-bucket + auth: + type: google + google: + credentials: + project_id: ${GCP_PROJECT_ID} + private_key_id: ${GCP_PRIVATE_KEY_ID} + private_key: ${GCP_PRIVATE_KEY} + client_email: ${GCP_CLIENT_EMAIL} + client_id: ${GCP_CLIENT_ID} + scopes: + - https://www.googleapis.com/auth/cloud-platform + header.x-goog-user-project: ${GCP_PROJECT_ID} + header.X-Iceberg-Access-Delegation: "" +``` + +Create these individual secrets in DataHub: + +- `GCP_PROJECT_ID` +- `GCP_PRIVATE_KEY_ID` +- `GCP_PRIVATE_KEY` (the private key value from your service account JSON) +- `GCP_CLIENT_EMAIL` +- `GCP_CLIENT_ID` + +**Step 3: Deploy via DataHub UI** + +1. Navigate to **Ingestion** > **Create new source** +2. Select **Iceberg** as the source type +3. Paste your recipe configuration with secret references +4. Configure a schedule (optional) +5. Click **Save and Run** + +##### Using Vended Credentials (Optional) + +**Important**: Vended credentials require your BigLake catalog to be configured with `CREDENTIAL_MODE_SERVICE_ACCOUNT`. Most BigLake catalogs use `CREDENTIAL_MODE_END_USER` by default, which **does not** support vended credentials. + +If you get an error stating "X-Iceberg-Access-Delegation header must not contain vended-credentials when credential mode is CREDENTIAL_MODE_END_USER", your catalog doesn't support this feature. Use the standard configuration with `header.X-Iceberg-Access-Delegation: ""` instead. + +For catalogs that support vended credentials, set `header.X-Iceberg-Access-Delegation: vended-credentials`: + +```yaml +source: + type: iceberg + config: + catalog: + my_biglake_catalog: + type: rest + uri: https://biglake.googleapis.com/iceberg/v1/restcatalog + warehouse: gs://my-bucket + auth: + type: google + google: + scopes: + - https://www.googleapis.com/auth/cloud-platform + header.x-goog-user-project: my-project + header.X-Iceberg-Access-Delegation: vended-credentials # Only for CREDENTIAL_MODE_SERVICE_ACCOUNT +``` + +**When to use vended credentials:** + +- Running ingestion from environments without direct GCS access +- Implementing fine-grained access control through BigLake +- Avoiding long-lived service account keys + +BigLake will generate short-lived service account token scoped to the specific tables being accessed. + +##### Troubleshooting + +**Error: "invalid_scope: Invalid OAuth scope or ID token audience provided"** + +This error occurs when OAuth scopes are not properly configured. To fix: + +- Ensure `auth.google.scopes` is set to `["https://www.googleapis.com/auth/cloud-platform"]` in your configuration +- Verify `google-auth` library is installed: `pip install google-auth` +- Check that `GOOGLE_APPLICATION_CREDENTIALS` points to a valid service account JSON file + +**Error: "X-Iceberg-Access-Delegation header must not contain vended-credentials when credential mode is CREDENTIAL_MODE_END_USER"** + +- Your BigLake catalog uses end-user credentials mode, which doesn't support vended credentials +- **Solution**: Use `header.X-Iceberg-Access-Delegation: ""` (empty string) in your configuration +- Vended credentials only work with `CREDENTIAL_MODE_SERVICE_ACCOUNT` + +**Error: "Authentication failed"** + +- Verify ADC is configured: `gcloud auth application-default print-access-token` +- Check service account has required permissions +- Ensure BigLake API is enabled: `gcloud services list --enabled | grep biglake` + +**Error: "Catalog not found"** + +- Verify catalog exists: `gcloud alpha biglake catalogs list --location=REGION --project=PROJECT` +- Check URI format: `https://biglake.googleapis.com/v1/projects/{PROJECT}/locations/{REGION}/catalogs/{CATALOG}` + +**Error: "Permission denied on GCS warehouse"** + +- Grant Storage Object Viewer role to service account: + ```bash + gsutil iam ch serviceAccount:SA_EMAIL:roles/storage.objectViewer gs://BUCKET_NAME + ``` + +**Error: "User project header required"** + +- Ensure `header.x-goog-user-project` is set to your GCP project ID + +#### SQL catalog + Azure DLS as the warehouse + +This example targets `Postgres` as the sql-type `Iceberg` catalog and uses Azure DLS as the warehouse. + +```yaml +source: + type: "iceberg" + config: + catalog: + demo: + type: sql + uri: postgresql+psycopg2://user:password@sqldatabase.postgres.database.azure.com:5432/icebergcatalog + adlfs.tenant-id: + adlfs.account-name: + adlfs.client-id: + adlfs.client-secret: +``` + +#### Concept Mapping + + + + +This ingestion source maps the following Source System Concepts to DataHub Concepts: + + + +| Source Concept | DataHub Concept | Notes | +| --------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `iceberg` | [Data Platform](docs/generated/metamodel/entities/dataPlatform.md) | | +| Table | [Dataset](docs/generated/metamodel/entities/dataset.md) | An Iceberg table is registered inside a catalog using a name, where the catalog is responsible for creating, dropping and renaming tables. Catalogs manage a collection of tables that are usually grouped into namespaces. The name of a table is mapped to a Dataset name. If a [Platform Instance](/docs/platform-instances/) is configured, it will be used as a prefix: `.my.namespace.table`. | +| [Table property](https://iceberg.apache.org/docs/latest/configuration/#table-properties) | [User (a.k.a CorpUser)](docs/generated/metamodel/entities/corpuser.md) | The value of a table property can be used as the name of a CorpUser owner. This table property name can be configured with the source option `user_ownership_property`. | +| [Table property](https://iceberg.apache.org/docs/latest/configuration/#table-properties) | CorpGroup | The value of a table property can be used as the name of a CorpGroup owner. This table property name can be configured with the source option `group_ownership_property`. | +| Table parent folders (excluding [warehouse catalog location](https://iceberg.apache.org/docs/latest/configuration/#catalog-properties)) | Container | Available in a future release | +| [Table schema](https://iceberg.apache.org/spec/#schemas-and-data-types) | SchemaField | Maps to the fields defined within the Iceberg table schema definition. | + +#### DataHub Iceberg REST Catalog + +DataHub also implements the Iceberg REST Catalog. See the [Iceberg Catalog documentation](docs/iceberg-catalog.md) for more details. + +#### Integration Details + +For advanced Iceberg behavior and tuning, refer to: + +- [Iceberg catalog properties](https://iceberg.apache.org/docs/latest/configuration/#catalog-properties) +- [Iceberg table properties](https://iceberg.apache.org/docs/latest/configuration/#table-properties) +- [Iceberg specification](https://iceberg.apache.org/spec/) +- [PyIceberg configuration](https://py.iceberg.apache.org/) + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + +#### Exceptions while increasing `processing_threads` + +Each processing thread will open several files/sockets to download manifest files from blob storage. If you experience +exceptions appearing when increasing `processing_threads` configuration parameter, try to increase limit of open +files (e.g. using `ulimit` in Linux). + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.iceberg.iceberg.IcebergSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Iceberg, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/json-schema.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/json-schema.md new file mode 100644 index 00000000..63a197a9 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/json-schema.md @@ -0,0 +1,286 @@ +--- +sidebar_position: 40 +title: JSON Schemas +slug: /generated/ingestion/sources/json-schema +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# JSON Schemas + +## Overview + +Json Schema is a storage and lakehouse platform. Learn more in the [official Json Schema documentation](https://json-schema.org/). + +The DataHub integration for Json Schema covers file/lakehouse metadata entities such as datasets, paths, and containers. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `json-schema` +![Incubating](https://img.shields.io/badge/support%20status-incubating-blue) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Descriptions | ✅ | Extracts descriptions at top level and field level. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | With stateful ingestion enabled, will remove entities from DataHub if they are no longer present in the source. | +| Extract Ownership | ❌ | Does not currently support extracting ownership. | +| Extract Tags | ❌ | Does not currently support extracting tags. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Supports platform instance via config. | +| Schema Metadata | ✅ | Extracts schemas, following references. | + +### Overview + +The `json-schema` module ingests metadata from Json Schema into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[json-schema]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +pipeline_name: json_schema_ingestion +source: + type: json-schema + config: + path: # e.g. https://json.schemastore.org/petstore-v1.0.json + platform: # e.g. schemaregistry + # platform_instance: + stateful_ingestion: + enabled: true # recommended to have this turned on + +# sink configs if needed +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
path ✅
One of string(file-path), string(directory-path), string(uri)
| Set this to a single file-path or a directory-path (for recursive traversal) or a remote url. e.g. https://json.schemastore.org/petstore-v1.0.json | +|
platform ✅
string
| Set this to a platform that you want all schemas to live under. e.g. schemaregistry / schemarepo etc. | +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
use_id_as_base_uri
boolean
| When enabled, uses the `$id` field in the json schema as the base uri for following references.
Default: False
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
uri_replace_pattern
One of URIReplacePattern, null
| Use this if URI-s need to be modified during reference resolution. Simple string match - replace capabilities are supported.
Default: None
| +|
uri_replace_pattern.match ❓
string
| Pattern to match on uri-s as part of reference resolution. See replace field | +|
uri_replace_pattern.replace ❓
string
| Pattern to replace with as part of reference resolution. See match field | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + }, + "URIReplacePattern": { + "additionalProperties": false, + "properties": { + "match": { + "description": "Pattern to match on uri-s as part of reference resolution. See replace field", + "title": "Match", + "type": "string" + }, + "replace": { + "description": "Pattern to replace with as part of reference resolution. See match field", + "title": "Replace", + "type": "string" + } + }, + "required": [ + "match", + "replace" + ], + "title": "URIReplacePattern", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "path": { + "anyOf": [ + { + "format": "file-path", + "type": "string" + }, + { + "format": "directory-path", + "type": "string" + }, + { + "format": "uri", + "minLength": 1, + "type": "string" + } + ], + "description": "Set this to a single file-path or a directory-path (for recursive traversal) or a remote url. e.g. https://json.schemastore.org/petstore-v1.0.json", + "title": "Path" + }, + "platform": { + "description": "Set this to a platform that you want all schemas to live under. e.g. schemaregistry / schemarepo etc.", + "title": "Platform", + "type": "string" + }, + "use_id_as_base_uri": { + "default": false, + "description": "When enabled, uses the `$id` field in the json schema as the base uri for following references.", + "title": "Use Id As Base Uri", + "type": "boolean" + }, + "uri_replace_pattern": { + "anyOf": [ + { + "$ref": "#/$defs/URIReplacePattern" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Use this if URI-s need to be modified during reference resolution. Simple string match - replace capabilities are supported." + } + }, + "required": [ + "path", + "platform" + ], + "title": "JsonSchemaSourceConfig", + "type": "object" +} +``` + + + + + +#### Configuration Notes + +- You must provide a `platform` field. Most organizations have custom project names for their schema repositories, so you can pick whatever name makes sense. For example, you might want to call your schema platform **schemaregistry**. After picking a custom platform, you can use the [put platform](../../../../docs/cli.md#put-platform) command to register your custom platform into DataHub. + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.schema.json_schema.JsonSchemaSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/schema/json_schema.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for JSON Schemas, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka-connect.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka-connect.md new file mode 100644 index 00000000..5b9719fc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka-connect.md @@ -0,0 +1,1375 @@ +--- +sidebar_position: 42 +title: Kafka Connect +slug: /generated/ingestion/sources/kafka-connect +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Kafka Connect + +## Overview + +Kafka Connect is a streaming or integration platform. Learn more in the [official Kafka Connect documentation](https://kafka.apache.org/documentation/#connect). + +The DataHub integration for Kafka Connect covers streaming/integration entities such as topics, connectors, pipelines, or jobs. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Source Concept | DataHub Concept | Notes | +| ------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | ----- | +| `"kafka-connect"` | [Data Platform](/docs/generated/metamodel/entities/dataplatform/) | | +| [Connector](https://kafka.apache.org/documentation/#connect_connectorsandtasks) | [DataFlow](/docs/generated/metamodel/entities/dataflow/) | | +| Kafka Topic | [Dataset](/docs/generated/metamodel/entities/dataset/) | | + + +## Module `kafka-connect` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Enabled by default. | +| Schema Metadata | ✅ | Enabled by default. | +| Table-Level Lineage | ✅ | Enabled by default. | + +### Overview + +The `kafka-connect` module ingests metadata from Kafka Connect into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +This plugin extracts the following: + +- Source and Sink Connectors in Kafka Connect as Data Pipelines +- For Source connectors - Data Jobs to represent lineage information between source dataset to Kafka topic per `{connector_name}:{source_dataset}` combination +- For Sink connectors - Data Jobs to represent lineage information between Kafka topic to destination dataset per `{connector_name}:{topic}` combination + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + +#### Java Runtime Dependency + +This source requires Java to be installed and available on the system for transform pipeline support (RegexRouter, etc.). The Java runtime is accessed via JPype to enable Java regex pattern matching that's compatible with Kafka Connect transforms. + +- **Python installations**: Install Java separately (e.g., `apt-get install openjdk-11-jre-headless` on Debian/Ubuntu) +- **Docker deployments**: Ensure your DataHub ingestion Docker image includes a Java runtime. The official DataHub images include Java by default. +- **Impact**: Without Java, transform pipeline features will be disabled and lineage accuracy may be reduced for connectors using transforms + +**Note for Docker users**: If you're building custom Docker images for DataHub ingestion, ensure a Java Runtime Environment (JRE) is included in your image to support full transform pipeline functionality. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[kafka-connect]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "kafka-connect" + config: + # Coordinates + connect_uri: "http://localhost:8083" + + # Credentials + username: admin + password: password + + # Optional + # Platform instance mapping to use when constructing URNs. + # Use if single instance of platform is referred across connectors. + platform_instance_map: + mysql: mysql_platform_instance + +sink: + # sink configs + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
cluster_name
One of string, null
| Cluster to ingest from.
Default: connect-cluster
| +|
confluent_cloud_cluster_id
One of string, null
| Confluent Cloud Kafka Connect cluster ID (e.g., 'lkc-abc123'). When specified along with confluent_cloud_environment_id, the connect_uri will be automatically constructed. This is the recommended approach for Confluent Cloud instead of manually constructing the full URI.
Default: None
| +|
confluent_cloud_environment_id
One of string, null
| Confluent Cloud environment ID (e.g., 'env-xyz123'). When specified along with confluent_cloud_cluster_id, the connect_uri will be automatically constructed. This is the recommended approach for Confluent Cloud instead of manually constructing the full URI.
Default: None
| +|
connect_to_platform_map
One of string, null
| Platform instance mapping when multiple instances for a platform is available. Entry for a platform should be in either `platform_instance_map` or `connect_to_platform_map`. e.g.`connect_to_platform_map: { "postgres-connector-finance-db": "postgres": "core_finance_instance" }`
Default: None
| +|
connect_uri
string
| URI to connect to.
Default: http://localhost:8083/
| +|
convert_lineage_urns_to_lowercase
boolean
| Whether to convert the urns of ingested lineage dataset to lowercase
Default: False
| +|
kafka_api_key
One of string, null
| Optional: Confluent Cloud Kafka API key for authenticating with Kafka REST API v3. If not specified, DataHub will reuse the Connect credentials (username/password) for Kafka API authentication. Only needed if you want to use separate credentials for the Kafka API.
Default: None
| +|
kafka_api_secret
One of string(password), null
| Optional: Confluent Cloud Kafka API secret for authenticating with Kafka REST API v3. If not specified, DataHub will reuse the Connect credentials (username/password) for Kafka API authentication. Only needed if you want to use separate credentials for the Kafka API.
Default: None
| +|
kafka_rest_endpoint
One of string, null
| Optional: Confluent Cloud Kafka REST API endpoint for comprehensive topic retrieval. Format: https://pkc-xxxxx.region.provider.confluent.cloud If not specified, DataHub automatically derives the endpoint from connector configurations (kafka.endpoint). When available, enables getting all topics from Kafka cluster for improved transform pipeline accuracy.
Default: None
| +|
password
One of string(password), null
| Kafka Connect password.
Default: None
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
platform_instance_map
One of string, null
| Platform instance mapping to use when constructing URNs. e.g.`platform_instance_map: { "hive": "warehouse" }`
Default: None
| +|
schema_resolver_expand_patterns
One of boolean, null
| Enable table pattern expansion using DataHub schema metadata. When use_schema_resolver=True, this controls whether to expand patterns like 'database.*' to actual table names by querying DataHub. Only applies when use_schema_resolver is enabled. Defaults to True when use_schema_resolver is enabled.
Default: None
| +|
schema_resolver_finegrained_lineage
One of boolean, null
| Enable fine-grained (column-level) lineage extraction using DataHub schema metadata. When use_schema_resolver=True, this controls whether to generate column-level lineage by matching schemas between source tables and Kafka topics. Only applies when use_schema_resolver is enabled. Defaults to True when use_schema_resolver is enabled.
Default: None
| +|
use_connect_topics_api
boolean
| Whether to use Kafka Connect API for topic retrieval and validation. This flag controls the environment-specific topic retrieval strategy:
**When True (default):** - **Self-hosted environments:** Uses runtime `/connectors/{name}/topics` API for accurate topic information - **Confluent Cloud:** Uses comprehensive Kafka REST API v3 to get all topics for transform pipeline, with config-based fallback
**When False:** Disables all API-based topic retrieval for both environments. Returns empty topic lists. Useful for air-gapped environments or when topic validation isn't needed for performance optimization.
Default: True
| +|
use_schema_resolver
boolean
| Use DataHub's schema metadata to enhance Kafka Connect connector lineage. When enabled (requires DataHub graph connection): 1) Expands table patterns (e.g., 'database.*') to actual tables using DataHub metadata 2) Generates fine-grained column-level lineage for Kafka Connect sources/sinks.

**Auto-enabled for Confluent Cloud:** This feature is automatically enabled for Confluent Cloud environments where DataHub graph connection is required. Set `use_schema_resolver: false` to disable.

**Prerequisite:** Source database tables must be ingested into DataHub before Kafka Connect ingestion for this feature to work. Without prior database ingestion, schema resolver will not find table metadata.
Default: False
| +|
username
One of string, null
| Kafka Connect username.
Default: None
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
connector_patterns
AllowDenyPattern
| A class to store allow deny regexes | +|
connector_patterns.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
connector_patterns.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
connector_patterns.allow.string
string
| | +|
connector_patterns.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
connector_patterns.deny.string
string
| | +|
generic_connectors
array
| Provide lineage graph for sources connectors other than Confluent JDBC Source Connector, Debezium Source Connector, and Mongo Source Connector
Default: []
| +|
generic_connectors.GenericConnectorConfig
GenericConnectorConfig
| | +|
generic_connectors.GenericConnectorConfig.connector_name ❓
string
| | +|
generic_connectors.GenericConnectorConfig.source_dataset ❓
string
| | +|
generic_connectors.GenericConnectorConfig.source_platform ❓
string
| | +|
provided_configs
One of array, null
| Provided Configurations
Default: None
| +|
provided_configs.ProvidedConfig
ProvidedConfig
| | +|
provided_configs.ProvidedConfig.path_key ❓
string
| | +|
provided_configs.ProvidedConfig.provider ❓
string
| | +|
provided_configs.ProvidedConfig.value ❓
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "GenericConnectorConfig": { + "additionalProperties": false, + "properties": { + "connector_name": { + "title": "Connector Name", + "type": "string" + }, + "source_dataset": { + "title": "Source Dataset", + "type": "string" + }, + "source_platform": { + "title": "Source Platform", + "type": "string" + } + }, + "required": [ + "connector_name", + "source_dataset", + "source_platform" + ], + "title": "GenericConnectorConfig", + "type": "object" + }, + "ProvidedConfig": { + "additionalProperties": false, + "properties": { + "provider": { + "title": "Provider", + "type": "string" + }, + "path_key": { + "title": "Path Key", + "type": "string" + }, + "value": { + "title": "Value", + "type": "string" + } + }, + "required": [ + "provider", + "path_key", + "value" + ], + "title": "ProvidedConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance_map": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Platform instance mapping to use when constructing URNs. e.g.`platform_instance_map: { \"hive\": \"warehouse\" }`", + "title": "Platform Instance Map" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "connect_uri": { + "default": "http://localhost:8083/", + "description": "URI to connect to.", + "title": "Connect Uri", + "type": "string" + }, + "username": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kafka Connect username.", + "title": "Username" + }, + "password": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Kafka Connect password.", + "title": "Password" + }, + "cluster_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "connect-cluster", + "description": "Cluster to ingest from.", + "title": "Cluster Name" + }, + "convert_lineage_urns_to_lowercase": { + "default": false, + "description": "Whether to convert the urns of ingested lineage dataset to lowercase", + "title": "Convert Lineage Urns To Lowercase", + "type": "boolean" + }, + "connector_patterns": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [], + "ignoreCase": true + }, + "description": "regex patterns for connectors to filter for ingestion." + }, + "provided_configs": { + "anyOf": [ + { + "items": { + "$ref": "#/$defs/ProvidedConfig" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Provided Configurations", + "title": "Provided Configs" + }, + "connect_to_platform_map": { + "anyOf": [ + { + "additionalProperties": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Platform instance mapping when multiple instances for a platform is available. Entry for a platform should be in either `platform_instance_map` or `connect_to_platform_map`. e.g.`connect_to_platform_map: { \"postgres-connector-finance-db\": \"postgres\": \"core_finance_instance\" }`", + "title": "Connect To Platform Map" + }, + "generic_connectors": { + "default": [], + "description": "Provide lineage graph for sources connectors other than Confluent JDBC Source Connector, Debezium Source Connector, and Mongo Source Connector", + "items": { + "$ref": "#/$defs/GenericConnectorConfig" + }, + "title": "Generic Connectors", + "type": "array" + }, + "use_connect_topics_api": { + "default": true, + "description": "Whether to use Kafka Connect API for topic retrieval and validation. This flag controls the environment-specific topic retrieval strategy: \n**When True (default):** - **Self-hosted environments:** Uses runtime `/connectors/{name}/topics` API for accurate topic information - **Confluent Cloud:** Uses comprehensive Kafka REST API v3 to get all topics for transform pipeline, with config-based fallback \n**When False:** Disables all API-based topic retrieval for both environments. Returns empty topic lists. Useful for air-gapped environments or when topic validation isn't needed for performance optimization.", + "title": "Use Connect Topics Api", + "type": "boolean" + }, + "kafka_rest_endpoint": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional: Confluent Cloud Kafka REST API endpoint for comprehensive topic retrieval. Format: https://pkc-xxxxx.region.provider.confluent.cloud If not specified, DataHub automatically derives the endpoint from connector configurations (kafka.endpoint). When available, enables getting all topics from Kafka cluster for improved transform pipeline accuracy.", + "title": "Kafka Rest Endpoint" + }, + "kafka_api_key": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional: Confluent Cloud Kafka API key for authenticating with Kafka REST API v3. If not specified, DataHub will reuse the Connect credentials (username/password) for Kafka API authentication. Only needed if you want to use separate credentials for the Kafka API.", + "title": "Kafka Api Key" + }, + "kafka_api_secret": { + "anyOf": [ + { + "format": "password", + "type": "string", + "writeOnly": true + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional: Confluent Cloud Kafka API secret for authenticating with Kafka REST API v3. If not specified, DataHub will reuse the Connect credentials (username/password) for Kafka API authentication. Only needed if you want to use separate credentials for the Kafka API.", + "title": "Kafka Api Secret" + }, + "confluent_cloud_environment_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Confluent Cloud environment ID (e.g., 'env-xyz123'). When specified along with confluent_cloud_cluster_id, the connect_uri will be automatically constructed. This is the recommended approach for Confluent Cloud instead of manually constructing the full URI.", + "title": "Confluent Cloud Environment Id" + }, + "confluent_cloud_cluster_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Confluent Cloud Kafka Connect cluster ID (e.g., 'lkc-abc123'). When specified along with confluent_cloud_environment_id, the connect_uri will be automatically constructed. This is the recommended approach for Confluent Cloud instead of manually constructing the full URI.", + "title": "Confluent Cloud Cluster Id" + }, + "use_schema_resolver": { + "default": false, + "description": "Use DataHub's schema metadata to enhance Kafka Connect connector lineage. When enabled (requires DataHub graph connection): 1) Expands table patterns (e.g., 'database.*') to actual tables using DataHub metadata 2) Generates fine-grained column-level lineage for Kafka Connect sources/sinks. \n\n**Auto-enabled for Confluent Cloud:** This feature is automatically enabled for Confluent Cloud environments where DataHub graph connection is required. Set `use_schema_resolver: false` to disable. \n\n**Prerequisite:** Source database tables must be ingested into DataHub before Kafka Connect ingestion for this feature to work. Without prior database ingestion, schema resolver will not find table metadata.", + "title": "Use Schema Resolver", + "type": "boolean" + }, + "schema_resolver_expand_patterns": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Enable table pattern expansion using DataHub schema metadata. When use_schema_resolver=True, this controls whether to expand patterns like 'database.*' to actual table names by querying DataHub. Only applies when use_schema_resolver is enabled. Defaults to True when use_schema_resolver is enabled.", + "title": "Schema Resolver Expand Patterns" + }, + "schema_resolver_finegrained_lineage": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Enable fine-grained (column-level) lineage extraction using DataHub schema metadata. When use_schema_resolver=True, this controls whether to generate column-level lineage by matching schemas between source tables and Kafka topics. Only applies when use_schema_resolver is enabled. Defaults to True when use_schema_resolver is enabled.", + "title": "Schema Resolver Finegrained Lineage" + } + }, + "title": "KafkaConnectSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Transform Pipeline Support + +**✅ Fully Supported:** + +- **Any combination of transforms**: RegexRouter, EventRouter, and non-routing transforms +- **Complex transform chains**: Multiple chained transforms automatically handled +- **Both environments**: Self-hosted and Confluent Cloud work identically +- **Future-proof**: New transform types automatically supported + +:::warning Considerations + +For connectors not listed in the supported connector table above, use the `generic_connectors` configuration to provide explicit lineage mappings. +Some advanced connector-specific features may not be fully supported +::: + +#### Environment-Specific Behavior + +**Environment Detection**: Automatically detects environment based on `connect_uri` patterns containing `confluent.cloud`. + +##### Self-hosted Kafka Connect + +- **Topic Discovery**: Uses runtime `/connectors/{name}/topics` API endpoint for maximum accuracy +- **Requirements**: Standard Kafka Connect REST API access +- **Fallback**: If runtime API fails, falls back to config-based derivation + +```yml +source: + type: kafka-connect + config: + # Self-hosted Kafka Connect cluster + connect_uri: "http://localhost:8083" + # use_connect_topics_api: true # Default - enables runtime topic discovery +``` + +##### Confluent Cloud + +- **Topic Discovery**: Uses comprehensive Kafka REST API v3 to get all topics, with automatic credential reuse +- **Transform Support**: Full support for all transform combinations via reverse pipeline strategy using actual cluster topics +- **Auto-derivation**: Automatically derives Kafka REST endpoint from connector configurations + +**Recommended approach using environment and cluster IDs:** + +```yml +source: + type: kafka-connect + config: + # Auto-construct URI from environment and cluster IDs (recommended) + confluent_cloud_environment_id: "env-xyz123" # Your Confluent Cloud environment ID + confluent_cloud_cluster_id: "lkc-abc456" # Your Kafka Connect cluster ID + + # Standard credentials for Kafka Connect API + username: "your-connect-api-key" # API key for Kafka Connect access + password: "your-connect-api-secret" # API secret for Kafka Connect access + + # Optional: Separate credentials for Kafka REST API (if different from Connect API) + kafka_api_key: "your-kafka-api-key" # API key for Kafka REST API access + kafka_api_secret: "your-kafka-api-secret" # API secret for Kafka REST API access + + # Optional: Dedicated Kafka REST endpoint for comprehensive topic retrieval + kafka_rest_endpoint: "https://pkc-xxxxx.region.provider.confluent.cloud" + + # use_connect_topics_api: true # Default - enables comprehensive topic retrieval +``` + +**Alternative approach using full URI (legacy):** + +```yml +source: + type: kafka-connect + config: + # Confluent Cloud Connect URI - automatically detected + connect_uri: "https://api.confluent.cloud/connect/v1/environments/env-123/clusters/lkc-abc456" + username: "your-connect-api-key" # API key for Kafka Connect + password: "your-connect-api-secret" # API secret for Kafka Connect + kafka_api_key: "your-kafka-api-key" # API key for Kafka REST API (if different) + kafka_api_secret: "your-kafka-api-secret" # API secret for Kafka REST API (if different) + + # Optional: Dedicated Kafka REST endpoint for comprehensive topic retrieval + kafka_rest_endpoint: "https://pkc-xxxxx.region.provider.confluent.cloud" +``` + +##### Configuration Control + +The `use_connect_topics_api` flag controls topic retrieval behavior: + +- **When `true` (default)**: Uses environment-specific topic discovery with full transform support +- **When `false`**: Disables all topic discovery for air-gapped environments or performance optimization + +#### Advanced Scenarios: Complex Transform Chains + +The new reverse transform pipeline strategy handles complex scenarios automatically: + +```yaml +### Example: EventRouter + RegexRouter chain +transforms: EventRouter,RegexRouter +transforms.EventRouter.type: io.debezium.transforms.outbox.EventRouter +transforms.RegexRouter.type: org.apache.kafka.connect.transforms.RegexRouter +transforms.RegexRouter.regex: "outbox\\.event\\.(.*)" +transforms.RegexRouter.replacement: "events.$1" +``` + +#### Advanced Scenarios: Fallback Options + +- If transform pipeline cannot determine mappings, DataHub falls back to simple topic-based lineage +- For unsupported connector types or complex custom scenarios, use `generic_connectors` configuration + +#### Advanced Scenarios: Performance Optimization + +- Set `use_connect_topics_api: false` to disable topic discovery in air-gapped environments +- Transform pipeline processing adds minimal overhead and improves lineage accuracy + +#### Supported Source Connectors + +| Connector Type | Self-hosted Support | Confluent Cloud Support | Topic Discovery Method | Lineage Extraction | +| -------------------------------------------------------------------------------- | ------------------- | ----------------------- | --------------------------- | ------------------------------ | +| **Platform JDBC Source**
`io.confluent.connect.jdbc.JdbcSourceConnector` | ✅ Full | ✅ Full | Runtime API / Config-based | Table → Topic mapping | +| **Cloud PostgreSQL CDC**
`PostgresCdcSource` | ✅ Full | ✅ Full | Runtime API / Config-based | Table → Topic mapping | +| **Cloud PostgreSQL CDC V2**
`PostgresCdcSourceV2` | ✅ Full | ✅ Full | Runtime API / Config-based | Table → Topic mapping | +| **Cloud MySQL Source**
`MySqlSource` | ✅ Full | ✅ Full | Runtime API / Config-based | Table → Topic mapping | +| **Cloud MySQL CDC**
`MySqlCdcSource` | ✅ Full | ✅ Full | Runtime API / Config-based | Table → Topic mapping | +| **Debezium MySQL**
`io.debezium.connector.mysql.MySqlConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Database → Topic CDC mapping | +| **Debezium PostgreSQL**
`io.debezium.connector.postgresql.PostgresConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Database → Topic CDC mapping | +| **Debezium SQL Server**
`io.debezium.connector.sqlserver.SqlServerConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Database → Topic CDC mapping | +| **Debezium Oracle**
`io.debezium.connector.oracle.OracleConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Database → Topic CDC mapping | +| **Debezium DB2**
`io.debezium.connector.db2.Db2Connector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Database → Topic CDC mapping | +| **Debezium MongoDB**
`io.debezium.connector.mongodb.MongoDbConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Collection → Topic CDC mapping | +| **Debezium Vitess**
`io.debezium.connector.vitess.VitessConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Table → Topic CDC mapping | +| **MongoDB Source**
`com.mongodb.kafka.connect.MongoSourceConnector` | ✅ Full | 🔧 Config Required | Runtime API / Manual config | Collection → Topic mapping | +| **Generic Connectors** | 🔧 Config Required | 🔧 Config Required | User-defined mapping | Custom lineage mapping | + +#### Supported Sink Connectors + +| Connector Type | Self-hosted Support | Confluent Cloud Support | Topic Discovery Method | Lineage Extraction | +| ------------------------------------------------------------------------------ | ------------------- | ----------------------- | -------------------------- | ------------------------- | +| **BigQuery Sink**
`com.wepay.kafka.connect.bigquery.BigQuerySinkConnector` | ✅ Full | ✅ Full | Runtime API / Config-based | Topic → Table mapping | +| **S3 Sink**
`io.confluent.connect.s3.S3SinkConnector` | ✅ Full | ✅ Full | Runtime API / Config-based | Topic → S3 object mapping | +| **Snowflake Sink**
`com.snowflake.kafka.connector.SnowflakeSinkConnector` | ✅ Full | ✅ Full | Runtime API / Config-based | Topic → Table mapping | +| **Cloud PostgreSQL Sink**
`PostgresSink` | ✅ Full | ✅ Full | Runtime API / Config-based | Topic → Table mapping | +| **Cloud MySQL Sink**
`MySqlSink` | ✅ Full | ✅ Full | Runtime API / Config-based | Topic → Table mapping | +| **Cloud Snowflake Sink**
`SnowflakeSink` | ✅ Full | ✅ Full | Runtime API / Config-based | Topic → Table mapping | +| **Debezium JDBC Sink**
`io.debezium.connector.jdbc.JdbcSinkConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Topic → Table mapping | +| **Confluent JDBC Sink**
`io.confluent.connect.jdbc.JdbcSinkConnector` | ✅ Full | ✅ Partial | Runtime API / Config-based | Topic → Table mapping | + +**Legend:** + +- ✅ **Full**: Complete lineage extraction with accurate topic discovery +- ✅ **Partial**: Lineage extraction supported but topic discovery may be limited (config-based only) +- 🔧 **Config Required**: Requires `generic_connectors` configuration for lineage mapping +- ❌ **Not supported**: Connector class is not used in this environment + +:::info +On JDBC Sink connectors in Confluent Cloud:\*\* `io.debezium.connector.jdbc.JdbcSinkConnector` and `io.confluent.connect.jdbc.JdbcSinkConnector` are not Confluent Cloud managed connectors — they can only appear as custom (self-managed) connectors deployed against a Confluent Cloud Kafka cluster. When present, DataHub supports lineage extraction for them, but with one limitation: the target platform (e.g. `postgres`, `mysql`, `oracle`, `mssql`) must be auto-detected from the `connection.url` field in the connector configuration. If `connection.url` is absent or uses an unrecognised JDBC scheme, platform detection will fail and a warning will be emitted. For Confluent Cloud managed JDBC sink connectors, use the dedicated `PostgresSink` or `MySqlSink` connector classes instead, which have explicit platform support. +::: + +#### Supported Transforms + +DataHub uses an **advanced transform pipeline strategy** that automatically handles complex transform chains by applying the complete pipeline to all topics and checking if results exist. This provides robust support for any combination of transforms. + +##### Topic Routing Transforms + +- **RegexRouter**: `org.apache.kafka.connect.transforms.RegexRouter` +- **Cloud RegexRouter**: `io.confluent.connect.cloud.transforms.TopicRegexRouter` +- **Debezium EventRouter**: `io.debezium.transforms.outbox.EventRouter` (Outbox pattern) + +##### Non-Topic Routing Transforms + +DataHub recognizes but passes through these transforms (they don't affect lineage): + +- InsertField, ReplaceField, MaskField, ValueToKey, HoistField, ExtractField +- SetSchemaMetadata, Flatten, Cast, HeadersFrom, TimestampConverter +- Filter, InsertHeader, DropHeaders, Drop, TombstoneHandler + +##### Transform Pipeline Strategy + +DataHub uses an improved **reverse transform pipeline approach** that: + +1. **Takes all actual topics** from the connector manifest/Kafka cluster +2. **Applies the complete transform pipeline** to each topic +3. **Checks if transformed results exist** in the actual topic list +4. **Creates lineage mappings** only for successful matches + +**Benefits:** + +- ✅ **Works with any transform combination** (single or chained transforms) +- ✅ **Handles complex scenarios** like EventRouter + RegexRouter chains +- ✅ **Uses actual topics as source of truth** (no prediction needed) +- ✅ **Future-proof** for new transform types +- ✅ **Works identically** for both self-hosted and Confluent Cloud environments + +#### How Lineage Inference Works with Transform Pipelines + +Kafka Connect connectors can apply transforms (like RegexRouter) that modify topic names before data reaches Kafka. DataHub's lineage inference analyzes these transform configurations to determine how topics are produced: + +1. **Configuration Analysis** - Extracts source tables from connector configuration (`table.include.list`, `database.include.list`) +2. **Transform Application** - Applies configured transforms (RegexRouter, EventRouter, etc.) to predict final topic names +3. **Topic Validation** - Validates predicted topics against actual cluster topics using Kafka REST API v3 +4. **Lineage Construction** - Maps source tables to validated topics, preserving schema information + +This approach works for both self-hosted and Confluent Cloud environments: + +- **Self-hosted**: Uses runtime `/connectors/{name}/topics` API for actual topics produced by each connector +- **Confluent Cloud**: Uses Kafka REST API v3 to get all cluster topics, then applies transform pipeline to match with connector config + +**Key Benefits**: + +- **90-95% accuracy** for Cloud connectors with transforms (significant improvement over previous config-only approach) +- **Full RegexRouter support** with Java regex compatibility +- **Complex transform chains** handled correctly +- **Schema preservation** maintains full table names with schema information + +**Configuration Options:** + +- **Environment/Cluster IDs (recommended)**: Use `confluent_cloud_environment_id` and `confluent_cloud_cluster_id` for automatic URI construction +- **Auto-derivation**: DataHub finds Kafka REST endpoint automatically from connector configs +- **Manual endpoint**: Specify `kafka_rest_endpoint` if auto-derivation doesn't work +- **Separate credentials (typical)**: Use `connect_api_key`/`connect_api_secret` for Connect API and `kafka_api_key`/`kafka_api_secret` for Kafka REST API +- **Legacy credentials**: Use `username`/`password` for Connect API (falls back for Kafka API if separate credentials not provided) + +#### Enhanced Topic Resolution for Source and Sink Connectors + +DataHub now provides intelligent topic resolution that works reliably across all environments, including Confluent Cloud where the Kafka Connect topics API is unavailable. + +##### How It Works + +**Source Connectors** (Debezium, Snowflake CDC, JDBC): + +- Always derive expected topics from connector configuration (`table.include.list`, `database.include.list`) +- Apply configured transforms (RegexRouter, EventRouter, etc.) to predict final topic names +- When Kafka API is available: Filter to only topics that exist in Kafka +- When Kafka API is unavailable (Confluent Cloud): Create lineages for all configured tables without filtering + +**Sink Connectors** (S3, Snowflake, BigQuery, JDBC): + +- Support both explicit topic lists (`topics` field) and regex patterns (`topics.regex` field) +- When `topics.regex` is used: + - Priority 1: Match against `manifest.topic_names` from Kafka API (if available) + - Priority 2: Query DataHub for Kafka topics and match pattern (if `use_schema_resolver` enabled) + - Priority 3: Warn user that pattern cannot be expanded + +##### Configuration Examples + +**Source Connector with Pattern Expansion:** + +```yml +# Debezium PostgreSQL source with wildcard tables +connector.config: + table.include.list: "public.analytics_.*" + # When Kafka API unavailable, DataHub will: + # 1. Query DataHub for all PostgreSQL tables matching pattern + # 2. Derive expected topic names (server.schema.table format) + # 3. Apply transforms if configured + # 4. Create lineages without Kafka validation +``` + +**Sink Connector with topics.regex (Confluent Cloud):** + +```yml +# S3 sink connector consuming from pattern-matched topics +connector.config: + topics.regex: "analytics\\..*" # Match topics like analytics.users, analytics.orders + # When Kafka API unavailable, DataHub will: + # 1. Query DataHub for all Kafka topics (requires use_schema_resolver: true) + # 2. Match topics against the regex pattern + # 3. Create lineages for matched topics +``` + +**Enable DataHub Topic Querying for Sink Connectors:** + +```yml +source: + type: kafka-connect + config: + connect_uri: "https://api.confluent.cloud/connect/v1/environments/env-123/clusters/lkc-abc456" + username: "your-connect-api-key" + password: "your-connect-api-secret" + + # Enable DataHub schema resolver for topic pattern expansion + use_schema_resolver: true # Required for topics.regex fallback + + # Configure graph connection for DataHub queries + datahub_gms_url: "http://localhost:8080" # Your DataHub GMS endpoint +``` + +##### Key Benefits + +1. **Confluent Cloud Support**: Both source and sink connectors work correctly with pattern-based configurations +2. **Config as Source of Truth**: Source connectors always derive topics from configuration, not from querying all tables in DataHub +3. **Smart Fallback**: Sink connectors can query DataHub for Kafka topics when Kafka API is unavailable +4. **Pattern Expansion**: Wildcards in `table.include.list` and `topics.regex` are properly expanded +5. **Transform Support**: All transforms (RegexRouter, EventRouter, etc.) are applied correctly + +##### When DataHub Topic Querying is Used + +DataHub will query for topics in these scenarios: + +**Source Connectors:** + +- When expanding wildcard patterns in `table.include.list` (e.g., `ANALYTICS.PUBLIC.*`) +- Queries source platform (PostgreSQL, MySQL, etc.) for tables matching the pattern + +**Sink Connectors:** + +- When `topics.regex` is used AND Kafka API is unavailable (Confluent Cloud) +- Queries DataHub's Kafka platform for topics matching the regex pattern +- Requires `use_schema_resolver: true` in configuration + +**Important Notes:** + +- DataHub never queries "all tables" to create lineages - config is always the source of truth +- Source connectors query source platforms (databases) to expand table patterns +- Sink connectors query Kafka platform to expand topic regex patterns +- Both require appropriate DataHub credentials and connectivity + +#### Using DataHub Schema Resolver for Pattern Expansion and Column-Level Lineage + +The Kafka Connect source can query DataHub for schema information to provide two capabilities: + +1. **Pattern Expansion** - Converts wildcard patterns like `database.*` into actual table names by querying DataHub +2. **Column-Level Lineage** - Generates field-level lineage by matching schemas between source tables and Kafka topics + +Both features require existing metadata in DataHub from your database and Kafka schema registry ingestion. + +##### Auto-Enabled for Confluent Cloud + +**Starting with the latest version**, `use_schema_resolver` is **automatically enabled** for Confluent Cloud environments to provide better defaults for enhanced lineage extraction. This gives you column-level lineage and pattern expansion out of the box! + +**Confluent Cloud (Auto-Enabled):** + +```yml +source: + type: kafka-connect + config: + # Confluent Cloud environment + confluent_cloud_environment_id: "env-xyz123" + confluent_cloud_cluster_id: "lkc-abc456" + username: "your-connect-api-key" + password: "your-connect-api-secret" + + # Schema resolver automatically enabled! ✓ + # use_schema_resolver: true (auto-enabled) + # schema_resolver_expand_patterns: true (auto-enabled) + # schema_resolver_finegrained_lineage: true (auto-enabled) +``` + +**To disable** (if you don't need these features): + +```yml +source: + type: kafka-connect + config: + confluent_cloud_environment_id: "env-xyz123" + confluent_cloud_cluster_id: "lkc-abc456" + use_schema_resolver: false # Explicitly disable auto-enable +``` + +**Self-hosted (Manual Enable Required):** + +```yml +source: + type: kafka-connect + config: + connect_uri: "http://localhost:8083" + + # Must explicitly enable for self-hosted + use_schema_resolver: true + + # DataHub connection + datahub_api: + server: "http://localhost:8080" +``` + +**Important Prerequisites:** + +> **⚠️ Source database tables must be ingested into DataHub BEFORE running Kafka Connect ingestion** +> +> The schema resolver queries DataHub for existing table metadata. If your source databases haven't been ingested yet, the feature will have no effect. Run database ingestion first! + +**Recommended Ingestion Order:** + +1. Ingest source databases (PostgreSQL, MySQL, Snowflake, etc.) → DataHub +2. Ingest Kafka schema registry (optional, for topic schemas) → DataHub +3. Run Kafka Connect ingestion → Enjoy enhanced lineage! + +##### Configuration Overview + +```yml +source: + type: kafka-connect + config: + connect_uri: "http://localhost:8083" + + # Enable DataHub schema querying (auto-enabled for Confluent Cloud) + use_schema_resolver: true + + # Control which features to use (both default to true when schema resolver enabled) + schema_resolver_expand_patterns: true # Expand wildcard patterns + schema_resolver_finegrained_lineage: true # Generate column-level lineage + + # DataHub connection (required when use_schema_resolver=true) + datahub_api: + server: "http://localhost:8080" + token: "your-datahub-token" # Optional +``` + +##### Pattern Expansion + +Converts wildcard patterns in connector configurations into actual table names by querying DataHub. + +**Example: MySQL Source with Wildcards** + +```yml +# Connector config contains pattern +connector.config: + table.include.list: "analytics.user_*" # Pattern: matches user_events, user_profiles, etc. + +# DataHub config +source: + type: kafka-connect + config: + use_schema_resolver: true + schema_resolver_expand_patterns: true +# Result: DataHub queries for MySQL tables matching "analytics.user_*" +# Finds: user_events, user_profiles, user_sessions +# Creates lineage: +# mysql.analytics.user_events -> kafka.server.analytics.user_events +# mysql.analytics.user_profiles -> kafka.server.analytics.user_profiles +# mysql.analytics.user_sessions -> kafka.server.analytics.user_sessions +``` + +**When to use:** + +- Connector configs have wildcard patterns (`database.*`, `schema.table_*`) +- You want accurate lineage without manually listing every table +- Source metadata exists in DataHub from database ingestion + +**When to skip:** + +- Connector configs use explicit table lists (no patterns) +- Source metadata not yet in DataHub +- Want faster ingestion without DataHub API calls + +**Configuration:** + +```yml +source: + type: kafka-connect + config: + use_schema_resolver: true + schema_resolver_expand_patterns: true # Enable pattern expansion + + + # If you only want column-level lineage but NOT pattern expansion: + # schema_resolver_expand_patterns: false +``` + +**Behavior without schema resolver:** +Patterns are treated as literal table names, resulting in potentially incorrect lineage. + +##### Column-Level Lineage + +Generates field-level lineage by matching column names between source tables and Kafka topics. + +**Example: PostgreSQL to Kafka CDC** + +```yml +source: + type: kafka-connect + config: + use_schema_resolver: true + schema_resolver_finegrained_lineage: true +# Source table schema in DataHub: +# postgres.public.users: [user_id, email, created_at, updated_at] + +# Kafka topic schema in DataHub: +# kafka.server.public.users: [user_id, email, created_at, updated_at] + +# Result: Column-level lineage created: +# postgres.public.users.user_id -> kafka.server.public.users.user_id +# postgres.public.users.email -> kafka.server.public.users.email +# postgres.public.users.created_at -> kafka.server.public.users.created_at +# postgres.public.users.updated_at -> kafka.server.public.users.updated_at +``` + +**Requirements:** + +- Source table schema exists in DataHub (from database ingestion) +- Kafka topic schema exists in DataHub (from schema registry or Kafka ingestion) +- Column names match between source and target (case-insensitive matching) + +**Benefits:** + +- **Impact Analysis**: See which fields are affected by schema changes +- **Data Tracing**: Track specific data elements through pipelines +- **Schema Understanding**: Visualize how data flows at the field level + +**ReplaceField Transform Support:** + +Column-level lineage respects ReplaceField transforms that filter or rename columns: + +```yml +# Connector excludes specific fields +connector.config: + transforms: "removeFields" + transforms.removeFields.type: "org.apache.kafka.connect.transforms.ReplaceField$Value" + transforms.removeFields.exclude: "internal_id,temp_column" +# DataHub behavior: +# Source schema: [user_id, email, internal_id, temp_column] +# After transform: [user_id, email] +# Column lineage created only for: user_id, email +``` + +**Configuration:** + +```yml +source: + type: kafka-connect + config: + use_schema_resolver: true + schema_resolver_finegrained_lineage: true # Enable column-level lineage + + + # If you only want pattern expansion but NOT column-level lineage: + # schema_resolver_finegrained_lineage: false +``` + +**Behavior without schema resolver:** +Only dataset-level lineage is created (e.g., `postgres.users -> kafka.users`), without field-level detail. + +##### Complete Configuration Example + +```yml +source: + type: kafka-connect + config: + # Kafka Connect cluster + connect_uri: "http://localhost:8083" + cluster_name: "production-connect" + + # Enable schema resolver features + use_schema_resolver: true + schema_resolver_expand_patterns: true # Expand wildcard patterns + schema_resolver_finegrained_lineage: true # Generate column-level lineage + + # DataHub connection + datahub_api: + server: "http://datahub.company.com" + token: "${DATAHUB_TOKEN}" + + # Platform instances (if using multiple) + platform_instance_map: + postgres: "prod-postgres" + kafka: "prod-kafka" +``` + +##### Performance Impact + +**API Calls per Connector:** + +- Pattern expansion: 1 GraphQL query per unique wildcard pattern +- Column-level lineage: 2 GraphQL queries (source schema + target schema) +- Results cached for ingestion run duration + +**Optimization:** + +```yml +# Minimal configuration - no schema resolver +source: + type: kafka-connect + config: + connect_uri: "http://localhost:8083" + # use_schema_resolver: false # Default - no DataHub queries + +# Pattern expansion only +source: + type: kafka-connect + config: + use_schema_resolver: true + schema_resolver_expand_patterns: true + schema_resolver_finegrained_lineage: false # Skip column lineage for faster ingestion + +# Column lineage only +source: + type: kafka-connect + config: + use_schema_resolver: true + schema_resolver_expand_patterns: false # Skip pattern expansion + schema_resolver_finegrained_lineage: true +``` + +**Best Practice:** +Run database and Kafka schema ingestion before Kafka Connect ingestion to pre-populate DataHub with schema metadata. + +##### Troubleshooting + +**"Pattern expansion found no matches for: analytics.\*"** + +Causes: + +- Source database metadata not in DataHub +- Pattern syntax doesn't match DataHub dataset names +- Platform instance mismatch + +Solutions: + +1. Run database ingestion first to populate DataHub +2. Verify pattern matches table naming in source system +3. Check `platform_instance_map` matches database ingestion config +4. Use explicit table list to bypass pattern expansion temporarily + +**"SchemaResolver not available: DataHub graph connection is not available"** + +Causes: + +- Missing `datahub_api` configuration +- DataHub GMS not accessible + +Solutions: + +```yml +source: + type: kafka-connect + config: + use_schema_resolver: true + datahub_api: + server: "http://localhost:8080" # Add DataHub GMS URL + token: "your-token" # Add if authentication enabled +``` + +**Column-level lineage not appearing** + +Check: + +1. Source table schema exists: Search for table in DataHub UI +2. Kafka topic schema exists: Search for topic in DataHub UI +3. Column names match (case differences are handled automatically) +4. Check ingestion logs for warnings about missing schemas + +**Slow ingestion with schema resolver enabled** + +Profile: + +- Check logs for "Schema resolver cache hits: X, misses: Y" +- High misses indicate missing metadata in DataHub + +Temporarily disable to compare: + +```yml +use_schema_resolver: false +``` + +#### Working with Platform Instances + +If you've multiple instances of kafka OR source/sink systems that are referred in your `kafka-connect` setup, you'd need to configure platform instance for these systems in `kafka-connect` recipe to generate correct lineage edges. You must have already set `platform_instance` in recipes of original source/sink systems. Refer the document [Working with Platform Instances](/docs/platform-instances) to understand more about this. + +There are two options available to declare source/sink system's `platform_instance` in `kafka-connect` recipe. If single instance of platform is used across all `kafka-connect` connectors, you can use `platform_instance_map` to specify platform_instance to use for a platform when constructing URNs for lineage. + +Example: + +```yml +# Map of platform name to platform instance +platform_instance_map: + snowflake: snowflake_platform_instance + mysql: mysql_platform_instance +``` + +If multiple instances of platform are used across `kafka-connect` connectors, you'd need to specify platform_instance to use for platform for every connector. + +##### Example - Multiple MySQL Source Connectors each reading from different mysql instance + +```yml +# Map of platform name to platform instance per connector +connect_to_platform_map: + mysql_connector1: + mysql: mysql_instance1 + + mysql_connector2: + mysql: mysql_instance2 +``` + +Here mysql_connector1 and mysql_connector2 are names of MySQL source connectors as defined in `kafka-connect` connector config. + +##### Example - Multiple MySQL Source Connectors each reading from difference mysql instance and writing to different kafka cluster + +```yml +connect_to_platform_map: + mysql_connector1: + mysql: mysql_instance1 + kafka: kafka_instance1 + + mysql_connector2: + mysql: mysql_instance2 + kafka: kafka_instance2 +``` + +You can also use combination of `platform_instance_map` and `connect_to_platform_map` in your recipe. Note that, the platform_instance specified for the connector in `connect_to_platform_map` will always take higher precedance even if platform_instance for same platform is set in `platform_instance_map`. + +If you do not use `platform_instance` in original source/sink recipes, you do not need to specify them in above configurations. + +Note that, you do not need to specify platform_instance for BigQuery. + +##### Example - Multiple BigQuery Sink Connectors each writing to different kafka cluster + +```yml +connect_to_platform_map: + bigquery_connector1: + kafka: kafka_instance1 + + bigquery_connector2: + kafka: kafka_instance2 +``` + +#### Provided Configurations from External Sources + +Kafka Connect supports pluggable configuration providers which can load configuration data from external sources at runtime. These values are not available to DataHub ingestion source through Kafka Connect APIs. If you are using such provided configurations to specify connection url (database, etc) in Kafka Connect connector configuration then you will need also add these in `provided_configs` section in recipe for DataHub to generate correct lineage. + +```yml +# Optional mapping of provider configurations if using +provided_configs: + - provider: env + path_key: MYSQL_CONNECTION_URL + value: jdbc:mysql://test_mysql:3306/librarydb +``` + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + +#### Air-gapped or Performance-Optimized Environments + +Disable topic discovery entirely for environments where API access is not available or not needed: + +```yml +source: + type: kafka-connect + config: + connect_uri: "http://localhost:8083" + use_connect_topics_api: false # Disables all topic discovery API calls +``` + +**Note**: When `use_connect_topics_api` is `false`, topic information will not be extracted, which may impact lineage accuracy but improves performance and works in air-gapped environments. + +#### Topic Discovery Issues + +**Problem**: Missing or incomplete topic information in lineage + +**Solutions**: + +1. **Verify Environment Detection**: + + ```bash + # Check logs for environment detection messages + # Self-hosted: "Detected self-hosted Kafka Connect - using runtime topics API" + # Confluent Cloud: "Detected Confluent Cloud - using comprehensive Kafka REST API topic retrieval" + ``` + +2. **Test API Connectivity**: + + ```bash + # For self-hosted - test topics API + curl -X GET "http://localhost:8083/connectors/{connector-name}/topics" + + # For Confluent Cloud - test Kafka REST API v3 + curl -X GET "https://pkc-xxxxx.region.provider.confluent.cloud/kafka/v3/clusters/{cluster-id}/topics" + ``` + +3. **Configuration Troubleshooting**: + ```yml + # Enable debug logging + source: + type: kafka-connect + config: + # ... other config ... + use_connect_topics_api: true # Ensure this is enabled (default) + ``` + +#### Environment-Specific Issues + +**Self-hosted Issues**: + +- **403/401 errors**: Check authentication credentials (`username`, `password`) +- **404 errors**: Verify Kafka Connect cluster is running and REST API is accessible +- **Empty topic lists**: Check if connectors are actually running and processing data + +**Confluent Cloud Issues**: + +- **Missing topics**: Verify connector configuration has proper source table fields (`table.include.list`, `query`) +- **Transform accuracy**: Check that RegexRouter patterns in connector config are valid Java regex +- **Complex transforms**: Now fully supported via forward transform pipeline with topic validation +- **Schema preservation**: Full schema information (e.g., `public.users`) is maintained through transform pipeline + +#### Performance Optimization + +If topic discovery is impacting performance: + +```yml +source: + type: kafka-connect + config: + connect_uri: "http://localhost:8083" + use_connect_topics_api: false # Disable for better performance (no topic info) +``` + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.kafka_connect.kafka_connect.KafkaConnectSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/kafka_connect/kafka_connect.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Kafka Connect, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka.md new file mode 100644 index 00000000..9c20b2e6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/kafka.md @@ -0,0 +1,732 @@ +--- +sidebar_position: 41 +title: Kafka +slug: /generated/ingestion/sources/kafka +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Kafka + +## Overview + +Kafka is a streaming or integration platform. Learn more in the [official Kafka documentation](https://kafka.apache.org/). + +The DataHub integration for Kafka covers streaming/integration entities such as topics, connectors, pipelines, or jobs. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------------------------- | ---------------------------- | ---------------------------------------------------------------- | +| Platform/account/project scope | Platform Instance, Container | Organizes assets within the platform context. | +| Core technical asset (for example table/view/topic/file) | Dataset | Primary ingested technical asset. | +| Schema fields / columns | SchemaField | Included when schema extraction is supported. | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | +| Dependencies and processing relationships | Lineage edges | Available when lineage extraction is supported and enabled. | + + +## Module `kafka` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Column-level Lineage | ❌ | Not supported. | +| [Data Profiling](../../../../metadata-ingestion/docs/dev_guides/sql_profiles.md) | ❌ | Not supported. | +| Descriptions | ✅ | Set dataset description to top level doc field for Avro schema. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| [Platform Instance](../../../platform-instances.md) | ✅ | For multiple Kafka clusters, use the platform_instance configuration. | +| Schema Metadata | ✅ | Schemas associated with each topic are extracted from the schema registry. Avro and Protobuf (certified), JSON (incubating). Schema references are supported. | +| Table-Level Lineage | ❌ | Not supported. If you use Kafka Connect, the kafka-connect source can generate lineage. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `kafka` module ingests metadata from Kafka into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +Extract Topics & Schemas from Apache Kafka or Confluent Cloud. + +This plugin extracts the following: + +- Topics from the Kafka broker +- Schemas associated with each topic from the schema registry (Avro, Protobuf and JSON schemas are supported) + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[kafka]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "kafka" + config: + platform_instance: "YOUR_CLUSTER_ID" + connection: + bootstrap: "broker:9092" + schema_registry_url: http://localhost:8081 + +sink: + # sink configs + + +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
convert_urns_to_lowercase
boolean
| Whether to convert dataset urns to lowercase.
Default: False
| +|
disable_topic_record_naming_strategy
boolean
| Disables the utilization of the TopicRecordNameStrategy for Schema Registry subjects. For more information, visit: https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#handling-differences-between-preregistered-and-client-derived-schemas:~:text=io.confluent.kafka.serializers.subject.TopicRecordNameStrategy
Default: False
| +|
enable_meta_mapping
boolean
| When enabled, applies the mappings that are defined through the meta_mapping directives.
Default: True
| +|
external_url_base
One of string, null
| Base URL for external platform (e.g. Aiven) where topics can be viewed. The topic name will be appended to this base URL.
Default: None
| +|
field_meta_mapping
object
| mapping rules that will be executed against field-level schema properties. Refer to the section below on meta automated mappings.
Default: {}
| +|
ignore_warnings_on_schema_type
boolean
| Disables warnings reported for non-AVRO/Protobuf value or key schemas if set.
Default: False
| +|
ingest_schemas_as_entities
boolean
| Enables ingesting schemas from schema registry as separate entities, in addition to the topics
Default: False
| +|
meta_mapping
object
| mapping rules that will be executed against top-level schema properties. Refer to the section below on meta automated mappings.
Default: {}
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
schema_registry_class
string
| The fully qualified implementation class(custom) that implements the KafkaSchemaRegistryBase interface.
Default: datahub.ingestion.source.confluent_schema_registry...
| +|
schema_tags_field
string
| The field name in the schema metadata that contains the tags to be added to the dataset.
Default: tags
| +|
strip_user_ids_from_email
boolean
| Whether or not to strip email id while adding owners using meta mappings.
Default: False
| +|
tag_prefix
string
| Prefix added to tags during ingestion.
Default:
| +|
topic_subject_map
map(str,string)
| | +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
connection
KafkaConsumerConnectionConfig
| Configuration class for holding connectivity information for Kafka consumers | +|
connection.bootstrap
string
|
Default: localhost:9092
| +|
connection.client_timeout_seconds
integer
| The request timeout used when interacting with the Kafka APIs.
Default: 60
| +|
connection.consumer_config
object
| Extra consumer config serialized as JSON. These options will be passed into Kafka's DeserializingConsumer. See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md . | +|
connection.schema_registry_config
object
| Extra schema registry config serialized as JSON. These options will be passed into Kafka's SchemaRegistryClient. https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html?#schemaregistryclient | +|
connection.schema_registry_url
string
| Schema registry URL. Can be overridden with KAFKA_SCHEMAREGISTRY_URL environment variable, or will use DATAHUB_GMS_BASE_PATH if not set. | +|
domain
map(str,AllowDenyPattern)
| A class to store allow deny regexes | +|
domain.`key`.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
domain.`key`.allow.string
string
| | +|
domain.`key`.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
domain.`key`.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
domain.`key`.deny.string
string
| | +|
topic_patterns
AllowDenyPattern
| A class to store allow deny regexes | +|
topic_patterns.ignoreCase
One of boolean, null
| Whether to ignore case sensitivity during pattern matching.
Default: True
| +|
topic_patterns.allow
array
| List of regex patterns to include in ingestion
Default: ['.*']
| +|
topic_patterns.allow.string
string
| | +|
topic_patterns.deny
array
| List of regex patterns to exclude from ingestion.
Default: []
| +|
topic_patterns.deny.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "AllowDenyPattern": { + "additionalProperties": false, + "description": "A class to store allow deny regexes", + "properties": { + "allow": { + "default": [ + ".*" + ], + "description": "List of regex patterns to include in ingestion", + "items": { + "type": "string" + }, + "title": "Allow", + "type": "array" + }, + "deny": { + "default": [], + "description": "List of regex patterns to exclude from ingestion.", + "items": { + "type": "string" + }, + "title": "Deny", + "type": "array" + }, + "ignoreCase": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": true, + "description": "Whether to ignore case sensitivity during pattern matching.", + "title": "Ignorecase" + } + }, + "title": "AllowDenyPattern", + "type": "object" + }, + "KafkaConsumerConnectionConfig": { + "additionalProperties": false, + "description": "Configuration class for holding connectivity information for Kafka consumers", + "properties": { + "bootstrap": { + "default": "localhost:9092", + "title": "Bootstrap", + "type": "string" + }, + "schema_registry_url": { + "description": "Schema registry URL. Can be overridden with KAFKA_SCHEMAREGISTRY_URL environment variable, or will use DATAHUB_GMS_BASE_PATH if not set.", + "title": "Schema Registry Url", + "type": "string" + }, + "schema_registry_config": { + "additionalProperties": true, + "description": "Extra schema registry config serialized as JSON. These options will be passed into Kafka's SchemaRegistryClient. https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html?#schemaregistryclient", + "title": "Schema Registry Config", + "type": "object" + }, + "client_timeout_seconds": { + "default": 60, + "description": "The request timeout used when interacting with the Kafka APIs.", + "title": "Client Timeout Seconds", + "type": "integer" + }, + "consumer_config": { + "additionalProperties": true, + "description": "Extra consumer config serialized as JSON. These options will be passed into Kafka's DeserializingConsumer. See https://docs.confluent.io/platform/current/clients/confluent-kafka-python/html/index.html#deserializingconsumer and https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md .", + "title": "Consumer Config", + "type": "object" + } + }, + "title": "KafkaConsumerConnectionConfig", + "type": "object" + }, + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "convert_urns_to_lowercase": { + "default": false, + "description": "Whether to convert dataset urns to lowercase.", + "title": "Convert Urns To Lowercase", + "type": "boolean" + }, + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "connection": { + "$ref": "#/$defs/KafkaConsumerConnectionConfig", + "default": { + "bootstrap": "localhost:9092", + "schema_registry_url": "http://localhost:8080/schema-registry/api/", + "schema_registry_config": {}, + "client_timeout_seconds": 60, + "consumer_config": {} + } + }, + "topic_patterns": { + "$ref": "#/$defs/AllowDenyPattern", + "default": { + "allow": [ + ".*" + ], + "deny": [ + "^_.*" + ], + "ignoreCase": true + } + }, + "domain": { + "additionalProperties": { + "$ref": "#/$defs/AllowDenyPattern" + }, + "default": {}, + "description": "A map of domain names to allow deny patterns. Domains can be urn-based (urn:li:domain:13ae4d85-d955-49fc-8474-9004c663a810) or bare (`13ae4d85-d955-49fc-8474-9004c663a810`).", + "title": "Domain", + "type": "object" + }, + "topic_subject_map": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Provides the mapping for the `key` and the `value` schemas of a topic to the corresponding schema registry subject name. Each entry of this map has the form `-key`:`` and `-value`:`` for the key and the value schemas associated with the topic, respectively. This parameter is mandatory when the [RecordNameStrategy](https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#how-the-naming-strategies-work) is used as the subject naming strategy in the kafka schema registry. NOTE: When provided, this overrides the default subject name resolution even when the `TopicNameStrategy` or the `TopicRecordNameStrategy` are used.", + "title": "Topic Subject Map", + "type": "object" + }, + "schema_registry_class": { + "default": "datahub.ingestion.source.confluent_schema_registry.ConfluentSchemaRegistry", + "description": "The fully qualified implementation class(custom) that implements the KafkaSchemaRegistryBase interface.", + "title": "Schema Registry Class", + "type": "string" + }, + "schema_tags_field": { + "default": "tags", + "description": "The field name in the schema metadata that contains the tags to be added to the dataset.", + "title": "Schema Tags Field", + "type": "string" + }, + "enable_meta_mapping": { + "default": true, + "description": "When enabled, applies the mappings that are defined through the meta_mapping directives.", + "title": "Enable Meta Mapping", + "type": "boolean" + }, + "meta_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against top-level schema properties. Refer to the section below on meta automated mappings.", + "title": "Meta Mapping", + "type": "object" + }, + "field_meta_mapping": { + "additionalProperties": true, + "default": {}, + "description": "mapping rules that will be executed against field-level schema properties. Refer to the section below on meta automated mappings.", + "title": "Field Meta Mapping", + "type": "object" + }, + "strip_user_ids_from_email": { + "default": false, + "description": "Whether or not to strip email id while adding owners using meta mappings.", + "title": "Strip User Ids From Email", + "type": "boolean" + }, + "tag_prefix": { + "default": "", + "description": "Prefix added to tags during ingestion.", + "title": "Tag Prefix", + "type": "string" + }, + "ignore_warnings_on_schema_type": { + "default": false, + "description": "Disables warnings reported for non-AVRO/Protobuf value or key schemas if set.", + "title": "Ignore Warnings On Schema Type", + "type": "boolean" + }, + "disable_topic_record_naming_strategy": { + "default": false, + "description": "Disables the utilization of the TopicRecordNameStrategy for Schema Registry subjects. For more information, visit: https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#handling-differences-between-preregistered-and-client-derived-schemas:~:text=io.confluent.kafka.serializers.subject.TopicRecordNameStrategy", + "title": "Disable Topic Record Naming Strategy", + "type": "boolean" + }, + "ingest_schemas_as_entities": { + "default": false, + "description": "Enables ingesting schemas from schema registry as separate entities, in addition to the topics", + "title": "Ingest Schemas As Entities", + "type": "boolean" + }, + "external_url_base": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Base URL for external platform (e.g. Aiven) where topics can be viewed. The topic name will be appended to this base URL.", + "title": "External Url Base" + } + }, + "title": "KafkaSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +:::note +Stateful Ingestion is available only when a Platform Instance is assigned to this source. +::: + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +#### Connecting to Confluent Cloud + +If using Confluent Cloud you can use a recipe like this. In this `consumer_config.sasl.username` and `consumer_config.sasl.password` are the API credentials that you get (in the Confluent UI) from your cluster -> Data Integration -> API Keys. `schema_registry_config.basic.auth.user.info` has API credentials for Confluent schema registry which you get (in Confluent UI) from Schema Registry -> API credentials. + +When creating API Key for the cluster ensure that the ACLs associated with the key are set like below. This is required for DataHub to read topic metadata from topics in Confluent Cloud. + +``` +Topic Name = * +Permission = ALLOW +Operation = DESCRIBE +Pattern Type = LITERAL +``` + +```yml +source: + type: "kafka" + config: + platform_instance: "YOUR_CLUSTER_ID" + connection: + bootstrap: "abc-defg.eu-west-1.aws.confluent.cloud:9092" + consumer_config: + security.protocol: "SASL_SSL" + sasl.mechanism: "PLAIN" + sasl.username: "${CLUSTER_API_KEY_ID}" + sasl.password: "${CLUSTER_API_KEY_SECRET}" + schema_registry_url: "https://abc-defgh.us-east-2.aws.confluent.cloud" + schema_registry_config: + basic.auth.user.info: "${REGISTRY_API_KEY_ID}:${REGISTRY_API_KEY_SECRET}" + +sink: + # sink configs +``` + +If you are trying to add domains to your topics you can use a configuration like below. + +```yml +source: + type: "kafka" + config: + # ...connection block + domain: + "urn:li:domain:13ae4d85-d955-49fc-8474-9004c663a810": + allow: + - ".*" + "urn:li:domain:d6ec9868-6736-4b1f-8aa6-fee4c5948f17": + deny: + - ".*" +``` + +Note that the `domain` in config above can be either an _urn_ or a domain _id_ (i.e. `urn:li:domain:13ae4d85-d955-49fc-8474-9004c663a810` or simply `13ae4d85-d955-49fc-8474-9004c663a810`). The Domain should exist in your DataHub instance before ingesting data into the Domain. To create a Domain on DataHub, check out the [Domains User Guide](/docs/domains/). + +If you are using a non-default subject naming strategy in the schema registry, such as [RecordNameStrategy](https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#how-the-naming-strategies-work), the mapping for the topic's key and value schemas to the schema registry subject names should be provided via `topic_subject_map` as shown in the configuration below. + +```yml +source: + type: "kafka" + config: + # ...connection block + # Defines the mapping for the key & value schemas associated with a topic & the subject name registered with the + # kafka schema registry. + topic_subject_map: + # Defines both key & value schema for topic 'my_topic_1' + "my_topic_1-key": "io.acryl.Schema1" + "my_topic_1-value": "io.acryl.Schema2" + # Defines only the value schema for topic 'my_topic_2' (the topic doesn't have a key schema). + "my_topic_2-value": "io.acryl.Schema3" +``` + +#### Custom Schema Registry + +The Kafka Source uses the schema registry to figure out the schema associated with both `key` and `value` for the topic. +By default it uses the [Confluent's Kafka Schema registry](https://docs.confluent.io/platform/current/schema-registry/index.html) +and supports the `AVRO` and `PROTOBUF` schema types. + +If you're using a custom schema registry, or you are using schema type other than `AVRO` or `PROTOBUF`, then you can provide your own +custom implementation of the `KafkaSchemaRegistryBase` class, and implement the `get_schema_metadata(topic, platform_urn)` method that +given a topic name would return object of `SchemaMetadata` containing schema for that topic. Please refer +`datahub.ingestion.source.confluent_schema_registry::ConfluentSchemaRegistry` for sample implementation of this class. + +```python +class KafkaSchemaRegistryBase(ABC): + @abstractmethod + def get_schema_metadata( + self, topic: str, platform_urn: str + ) -> Optional[SchemaMetadata]: + pass +``` + +The custom schema registry class can be configured using the `schema_registry_class` config param of the `kafka` source as shown below. + +```yaml +source: + type: "kafka" + config: + # Set the custom schema registry implementation class + schema_registry_class: "datahub.ingestion.source.confluent_schema_registry.ConfluentSchemaRegistry" + # Coordinates + connection: + bootstrap: "broker:9092" + schema_registry_url: http://localhost:8081 +``` + +#### OAuth Callback + +The OAuth callback function can be set up for both Kafka sources (consumers) and sinks (producers): + +- For sources: `config.connection.consumer_config.oauth_cb` +- For sinks: `config.connection.producer_config.oauth_cb` + +You need to specify a Python function reference in the format <python-module>:<function-name>. + +For example, in the configuration `oauth:create_token`, `create_token` is a function defined in `oauth.py`, and `oauth.py` must be accessible in the PYTHONPATH. + +##### Deploying Custom OAuth Callbacks + +**For Built-in Callbacks (Recommended):** + +DataHub includes pre-built OAuth callbacks for common use cases: + +- **AWS MSK IAM**: `datahub_actions.utils.kafka_msk_iam:oauth_cb` +- **Azure Event Hubs**: `datahub_actions.utils.kafka_eventhubs_auth:oauth_cb` + +**Important:** To use these built-in callbacks, you must install the `acryl-datahub-actions` package: + +```bash +pip install acryl-datahub-actions>=1.3.1.2 +``` + +**For Custom OAuth Callbacks:** + +If you need to implement a custom OAuth callback, you must ensure your Python module is accessible to the DataHub process, e.g. adding it via `PYTHONPATH=/path/to/your/module:$PYTHONPATH` or `pip install my-oauth-package`. + +**Example for Kafka Source:** + +```yaml +source: + type: "kafka" + config: + # Set the custom schema registry implementation class + schema_registry_class: "datahub.ingestion.source.confluent_schema_registry.ConfluentSchemaRegistry" + # Coordinates + connection: + bootstrap: "broker:9092" + schema_registry_url: http://localhost:8081 + consumer_config: + security.protocol: "SASL_PLAINTEXT" + sasl.mechanism: "OAUTHBEARER" + oauth_cb: "oauth:create_token" +# sink configs +``` + +**Example for Kafka Sink (e.g., MSK IAM authentication):** + +```yaml +sink: + type: "datahub-kafka" + config: + connection: + bootstrap: "b-1.msk.us-west-2.amazonaws.com:9098" + schema_registry_url: "http://datahub-gms:8080/schema-registry/api/" + producer_config: + security.protocol: "SASL_SSL" + sasl.mechanism: "OAUTHBEARER" + sasl.oauthbearer.method: "default" + oauth_cb: "datahub_actions.utils.kafka_msk_iam:oauth_cb" +``` + +#### Enriching DataHub metadata with automated meta mapping + +:::note +Meta mapping is currently only available for Avro schemas, and requires that those Avro schemas are pushed to the schema registry. +::: + +Avro schemas are permitted to have additional attributes not defined by the specification as arbitrary metadata. A common pattern is to utilize this for business metadata. The Kafka source has the ability to transform this directly into DataHub Owners, Tags and Terms. + +##### Simple tags + +If you simply have a list of tags embedded into an Avro schema (either at the top-level or for an individual field), you can use the `schema_tags_field` config. + +Example Avro schema: + +```json +{ + "name": "sampleRecord", + "type": "record", + "tags": ["tag1", "tag2"], + "fields": [ + { + "name": "field_1", + "type": "string", + "tags": ["tag3", "tag4"] + } + ] +} +``` + +The name of the field containing a list of tags can be configured with the `schema_tags_field` property: + +```yaml +config: + schema_tags_field: tags +``` + +##### meta mapping + +You can also map specific Avro fields into Owners, Tags and Terms using meta +mapping. + +Example Avro schema: + +```json +{ + "name": "sampleRecord", + "type": "record", + "owning_team": "@Data-Science", + "data_tier": "Bronze", + "fields": [ + { + "name": "field_1", + "type": "string", + "gdpr": { + "pii": true + } + } + ] +} +``` + +This can be mapped to DataHub metadata with `meta_mapping` config: + +```yaml +config: + meta_mapping: + owning_team: + match: "^@(.*)" + operation: "add_owner" + config: + owner_type: group + data_tier: + match: "Bronze|Silver|Gold" + operation: "add_term" + config: + term: "{{ $match }}" + field_meta_mapping: + gdpr.pii: + match: true + operation: "add_tag" + config: + tag: "pii" +``` + +The underlying implementation is similar to [dbt meta mapping](/docs/generated/ingestion/sources/dbt#dbt-meta-automated-mappings), which has more detailed examples that can be used for reference. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +#### `PROTOBUF` Schema Type Limitations + +The current implementation of the support for `PROTOBUF` schema type has the following limitations: + +- Recursive types are not supported. +- If the schemas of different topics define a type in the same package, the source would raise an exception. + +In addition to this, maps are represented as arrays of messages. The following message, + +``` +message MessageWithMap { + map map_1 = 1; +} +``` + +becomes: + +``` +message Map1Entry { + int key = 1; + string value = 2/ +} +message MessageWithMap { + repeated Map1Entry map_1 = 1; +} +``` + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.kafka.kafka.KafkaSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for Kafka, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/ldap.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/ldap.md new file mode 100644 index 00000000..dc03fe7f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/ldap.md @@ -0,0 +1,347 @@ +--- +sidebar_position: 43 +title: LDAP +slug: /generated/ingestion/sources/ldap +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# LDAP + +## Overview + +Ldap is an identity and access management platform. Learn more in the [official Ldap documentation](https://ldap.com/). + +The DataHub integration for Ldap covers identity entities such as users, groups, and memberships. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +While the specific concept mapping is still pending, this shows the generic concept mapping in DataHub. + +| Source Concept | DataHub Concept | Notes | +| -------------------------------------- | ------------------- | ---------------------------------------------------------------- | +| Ownership and collaboration principals | CorpUser, CorpGroup | Emitted by modules that support ownership and identity metadata. | + + +## Module `ldap` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | + +### Overview + +The `ldap` module ingests metadata from Ldap into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +### Prerequisites + +Before running ingestion, ensure network connectivity to the source, valid authentication credentials, and read permissions for metadata APIs required by this module. + + +### Install the Plugin +```shell +pip install 'acryl-datahub[ldap]' +``` + +### Starter Recipe +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + + +For general pointers on writing and running a recipe, see our [main recipe guide](../../../../metadata-ingestion/README.md#recipes). +```yaml +source: + type: "ldap" + config: + # Coordinates + ldap_server: ldap://localhost + + # Credentials + ldap_user: "cn=admin,dc=example,dc=org" + ldap_password: "admin" + + # Options + base_dn: "dc=example,dc=org" + + # TLS Security - Certificate verification is enabled by default + # Set to false only for testing with self-signed certificates + # tls_verify: true + +sink: + # sink configs +``` + +### Config Details + + + +Note that a `.` is used to denote nested fields in the YAML recipe. + + +
+ +| Field | Description | +|:--- |:--- | +|
base_dn ✅
string
| LDAP DN. | +|
ldap_password ✅
string(password)
| LDAP password. | +|
ldap_server ✅
string
| LDAP server URL. | +|
ldap_user ✅
string
| LDAP user. | +|
drop_missing_first_last_name
boolean
| If set to true, any users without first and last names will be dropped.
Default: True
| +|
filter
string
| LDAP extractor filter.
Default: (objectClass=*)
| +|
group_attrs_map
object
|
Default: {}
| +|
manager_filter_enabled
boolean
| Use LDAP extractor filter to search managers.
Default: True
| +|
manager_pagination_enabled
boolean
| [deprecated] Use pagination_enabled
Default: True
| +|
page_size
integer
| Size of each page to fetch when extracting metadata.
Default: 20
| +|
pagination_enabled
boolean
| Use pagination while do search query (enabled by default).
Default: True
| +|
platform_instance
One of string, null
| The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.
Default: None
| +|
tls_verify
boolean
| Verify server TLS certificates for LDAPS connections. Disabling in production exposes connections to Man-in-the-Middle attacks (CWE-295).
Default: True
| +|
use_email_as_username
boolean
| Use email for users' usernames instead of username (disabled by default). If enabled, the user and group urn would be having email as the id part of the urn.
Default: False
| +|
user_attrs_map
object
|
Default: {}
| +|
env
string
| The environment that all assets produced by this connector belong to
Default: PROD
| +|
attrs_list
One of array, null
| Retrieved attributes list
Default: None
| +|
attrs_list.string
string
| | +|
custom_props_list
One of array, null
| A list of custom attributes to extract from the LDAP provider.
Default: None
| +|
custom_props_list.string
string
| | +|
stateful_ingestion
One of StatefulStaleMetadataRemovalConfig, null
|
Default: None
| +|
stateful_ingestion.enabled
boolean
| Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False
Default: False
| +|
stateful_ingestion.fail_safe_threshold
number
| Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.
Default: 75.0
| +|
stateful_ingestion.remove_stale_metadata
boolean
| Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.
Default: True
| + +
+ + + + + +The [JSONSchema](https://json-schema.org/) for this configuration is inlined below. + + +```javascript +{ + "$defs": { + "StatefulStaleMetadataRemovalConfig": { + "additionalProperties": false, + "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.", + "properties": { + "enabled": { + "default": false, + "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False", + "title": "Enabled", + "type": "boolean" + }, + "remove_stale_metadata": { + "default": true, + "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.", + "title": "Remove Stale Metadata", + "type": "boolean" + }, + "fail_safe_threshold": { + "default": 75.0, + "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.", + "maximum": 100.0, + "minimum": 0.0, + "title": "Fail Safe Threshold", + "type": "number" + } + }, + "title": "StatefulStaleMetadataRemovalConfig", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Config used by the LDAP Source.", + "properties": { + "env": { + "default": "PROD", + "description": "The environment that all assets produced by this connector belong to", + "title": "Env", + "type": "string" + }, + "platform_instance": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The instance of the platform that all assets produced by this recipe belong to. This should be unique within the platform. See https://docs.datahub.com/docs/platform-instances/ for more details.", + "title": "Platform Instance" + }, + "stateful_ingestion": { + "anyOf": [ + { + "$ref": "#/$defs/StatefulStaleMetadataRemovalConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "ldap_server": { + "description": "LDAP server URL.", + "title": "Ldap Server", + "type": "string" + }, + "ldap_user": { + "description": "LDAP user.", + "title": "Ldap User", + "type": "string" + }, + "ldap_password": { + "description": "LDAP password.", + "format": "password", + "title": "Ldap Password", + "type": "string", + "writeOnly": true + }, + "base_dn": { + "description": "LDAP DN.", + "title": "Base Dn", + "type": "string" + }, + "filter": { + "default": "(objectClass=*)", + "description": "LDAP extractor filter.", + "title": "Filter", + "type": "string" + }, + "attrs_list": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Retrieved attributes list", + "title": "Attrs List" + }, + "custom_props_list": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A list of custom attributes to extract from the LDAP provider.", + "title": "Custom Props List" + }, + "drop_missing_first_last_name": { + "default": true, + "description": "If set to true, any users without first and last names will be dropped.", + "title": "Drop Missing First Last Name", + "type": "boolean" + }, + "page_size": { + "default": 20, + "description": "Size of each page to fetch when extracting metadata.", + "title": "Page Size", + "type": "integer" + }, + "manager_filter_enabled": { + "default": true, + "description": "Use LDAP extractor filter to search managers.", + "title": "Manager Filter Enabled", + "type": "boolean" + }, + "manager_pagination_enabled": { + "default": true, + "description": "[deprecated] Use pagination_enabled ", + "title": "Manager Pagination Enabled", + "type": "boolean" + }, + "pagination_enabled": { + "default": true, + "description": "Use pagination while do search query (enabled by default).", + "title": "Pagination Enabled", + "type": "boolean" + }, + "use_email_as_username": { + "default": false, + "description": "Use email for users' usernames instead of username (disabled by default). If enabled, the user and group urn would be having email as the id part of the urn.", + "title": "Use Email As Username", + "type": "boolean" + }, + "tls_verify": { + "default": true, + "description": "Verify server TLS certificates for LDAPS connections. Disabling in production exposes connections to Man-in-the-Middle attacks (CWE-295).", + "title": "Tls Verify", + "type": "boolean" + }, + "user_attrs_map": { + "additionalProperties": true, + "default": {}, + "title": "User Attrs Map", + "type": "object" + }, + "group_attrs_map": { + "additionalProperties": true, + "default": {}, + "title": "Group Attrs Map", + "type": "object" + } + }, + "required": [ + "ldap_server", + "ldap_user", + "ldap_password", + "base_dn" + ], + "title": "LDAPSourceConfig", + "type": "object" +} +``` + + + + + +### Capabilities + +Use the **Important Capabilities** table above as the source of truth for supported features and whether additional configuration is required. + +### Limitations + +Module behavior is constrained by source APIs, permissions, and metadata exposed by the platform. Refer to capability notes for unsupported or conditional features. + +### Troubleshooting + +If ingestion fails, validate credentials, permissions, connectivity, and scope filters first. Then review ingestion logs for source-specific errors and adjust configuration accordingly. + + +### Code Coordinates +- Class Name: `datahub.ingestion.source.ldap.LDAPSource` +- Browse on [GitHub](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/ldap.py) + + +:::tip Questions? + +If you've got any questions on configuring ingestion for LDAP, feel free to ping us on [our Slack](https://datahub.com/slack). +::: + + + +:::note 💡 **Contributing to this documentation** +This page is auto-generated from the underlying source code. To make changes, please edit the relevant source files in the [metadata-ingestion](https://github.com/datahub-project/datahub/tree/master/metadata-ingestion) directory. + +**Tip:** For quick typo fixes or documentation updates, you can click the ✏️ **Edit** icon directly in the GitHub UI to open a Pull Request. For larger changes and PR naming conventions, please refer to our [Contributing Guide](/docs/contributing). +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/looker.md b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/looker.md new file mode 100644 index 00000000..4b2818f5 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/generated/ingestion/sources/looker.md @@ -0,0 +1,2119 @@ +--- +sidebar_position: 44 +title: Looker +slug: /generated/ingestion/sources/looker +custom_edit_url: null +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Looker +There are 2 sources that provide integration with Looker + + + + + + + + + + +
Source ModuleDocumentation
+ +`looker` + + + + + +Source that extracts dashboards, explores, and charts from Looker via the Looker API. + +Implementation notes: +- Uses Looker SDK for API access +- Maintains LookerExploreRegistry to cache and resolve explore metadata +- Maintains LookerUserRegistry for ownership resolution +- Implements stateful ingestion for stale entity removal +- Supports usage statistics extraction from Looker's system activity + [Read more...](#module-looker) + + +
+ +`lookml` + + + + + +Source that parses LookML files to extract view and model metadata. + +Implementation notes: +- Uses lkml parser library to parse LookML syntax +- Optional integration with Looker API for enhanced name resolution +- Supports both git-based ingestion (clones repo) and local file ingestion +- Implements stateful ingestion with StaleEntityRemovalHandler +- Uses SQL parsing for extracting upstream table names from derived tables + [Read more...](#module-lookml) + + +
+ + +## Overview + +Looker is a business intelligence and analytics platform. Learn more in the [official Looker documentation](https://cloud.google.com/looker). + +The DataHub integration for Looker covers BI entities such as dashboards, charts, datasets, and related ownership context. Depending on module capabilities, it can also capture features such as lineage, usage, profiling, ownership, tags, and stateful deletion detection. + +## Concept Mapping + +| Looker Concept | DataHub Concept | Notes | +| ------------------------------- | --------------------------- | ------------------------------------------------------- | +| Dashboard / Look | Dashboard / Chart | Ingested by the `looker` module. | +| Explore / View model constructs | Dataset and lineage context | Ingested by the `lookml` module. | +| User, folder, model references | Ownership/container context | Used to enrich governance metadata and discoverability. | + + +## Module `looker` +![Certified](https://img.shields.io/badge/support%20status-certified-brightgreen) + + +### Important Capabilities +| Capability | Status | Notes | +| ---------- | ------ | ----- | +| Asset Containers | ✅ | Enabled by default. Supported for types - LookML Model, Folder. | +| Column-level Lineage | ✅ | Enabled by default, configured using `extract_column_level_lineage`. | +| Dataset Usage | ✅ | Enabled by default, configured using `extract_usage_history`. | +| Descriptions | ✅ | Enabled by default. | +| [Detect Deleted Entities](../../../../metadata-ingestion/docs/dev_guides/stateful.md#stale-entity-removal) | ✅ | Enabled by default via stateful ingestion. | +| Extract Ownership | ✅ | Enabled by default, configured using `extract_owners`. | +| [Platform Instance](../../../platform-instances.md) | ✅ | Use the `platform_instance` field. | +| Table-Level Lineage | ✅ | Supported by default. | +| Test Connection | ✅ | Enabled by default. | + +### Overview + +The `looker` module ingests metadata from Looker into DataHub. It is intended for production ingestion workflows and module-specific capabilities are documented below. + +#### Ingestion through UI + +The following video shows you how to get started with ingesting Looker metadata through the UI. + +:::note + +You will need to run `lookml` ingestion through the CLI after you have ingested Looker metadata through the UI. Otherwise you will not be able to see Looker Views and their lineage to your warehouse tables. + +::: + +
+ +

+ +### GraphQL + +- [searchAcrossEntities](/docs/graphql/queries/#searchacrossentities) +- You can try out the API on the demo instance's public GraphQL interface: [here](https://demo.datahub.com/api/graphiql) + +The same GraphQL API that powers the Search UI can be used +for integrations and programmatic use-cases. + +``` +# Example query - search for datasets matching the example_query_text who have the Dimension tag applied to a schema field and are from the data platform looker +query searchEntities { + search( + input: { + type: DATASET, + query: "example_query_text", + orFilters: [ + { + and: [ + { + field: "fieldTags", + values: ["urn:li:tag:Dimension"] + }, + { + field: "platform", + values: ["urn:li:dataPlatform:looker"] + } + ] + } + ], + start: 0, + count: 10 + } + ) { + start + count + total + searchResults { + entity { + urn + type + ... on Dataset { + name + platform { + name + } + } + } + } + } +} +``` + +### Searching at Scale + +For queries that return more than 10k entities we recommend using the [scrollAcrossEntities](/docs/graphql/queries/#scrollacrossentities) GraphQL API: + +``` +# Example query +{ + scrollAcrossEntities(input: { types: [DATASET], query: "*", count: 10}) { + nextScrollId + count + searchResults { + entity { + type + ... on Dataset { + urn + type + platform { + name + } + name + } + } + } + } +} +``` + +This will return a response containing a `nextScrollId` value which must be used in subsequent queries to retrieve more data, i.e: + +``` +{ + scrollAcrossEntities(input: + { types: [DATASET], query: "*", count: 10, + scrollId: "eyJzb3J0IjpbMy4wLCJ1cm46bGk6ZGF0YXNldDoodXJuOmxpOmRhdGFQbGF0Zm9ybTpiaWdxdWVyeSxiaWdxdWVyeS1wdWJsaWMtZGF0YS5jb3ZpZDE5X2dlb3RhYl9tb2JpbGl0eV9pbXBhY3QucG9ydF90cmFmZmljLFBST0QpIl0sInBpdElkIjpudWxsLCJleHBpcmF0aW9uVGltZSI6MH0="} + ) { + nextScrollId + count + searchResults { + entity { + type + ... on Dataset { + urn + type + platform { + name + } + name + } + } + } + } +} +``` + +In order to complete scrolling through all of the results, continue to request data in batches until the `nextScrollId` returned is null or undefined. + +### DataHub Blog + +- [Using DataHub for Search & Discovery](https://medium.com/datahub-project/using-datahub-for-search-discovery-fa309089be22) + +## Customizing Search + +It is possible to completely customize search ranking, filtering, and queries using a search configuration yaml file. +This no-code solution provides the ability to extend, or replace, the Elasticsearch-based search functionality. The +only limitation is that the information used in the query/ranking/filtering must be present in the entities' document, +however this does include `customProperties`, `tags`, `terms`, `domain`, as well as many additional fields. + +Additionally, multiple customizations can be applied to different query strings. A regex is applied to the search query +to determine which customized search profile to use. This means a different query/ranking/filtering can be applied to +a `select all`/`*` query or one that contains an actual query. + +Search results (excluding select `*`) are a balance between relevancy and the scoring function. In +general, when trying to improve relevancy, focus on changing the query in the `boolQuery` section and rely on the +`functionScore` for surfacing the _importance_ in the case of a relevancy tie. Consider the scenario +where a dataset named `orders` exists in multiple places. The relevancy between the dataset with the **name** `orders` and +the **term** `orders` is the same, however one location may be more important and thus the function score preferred. + +**Note:** The customized query is a pass-through to Elasticsearch and must comply with their API, syntax errors are possible. +It is recommended to test the customized queries prior to production deployment and knowledge of the Elasticsearch query +language is required. + +### Enable Custom Search + +The following environment variables on GMS control whether a search configuration is enabled and the location of the +configuration file. + +Enable Custom Search: + +```shell +ELASTICSEARCH_QUERY_CUSTOM_CONFIG_ENABLED=true +``` + +Custom Search File Location: + +```shell +ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE=search_config.yml +``` + +The location of the configuration file can be on the Java classpath or the local filesystem. A default configuration +file is included with the GMS jar with the name `search_config.yml`. + +### Search Configuration + +The search configuration yaml contains a simple list of configuration profiles selected using the `queryRegex`. If a +single profile is desired, a catch-all regex of `.*` can be used. + +The list of search configurations can be grouped into 4 general sections. + +1. `queryRegex` - Responsible for selecting the search customization based on the [regex matching](https://www.w3schools.com/java/java_regex.asp) the search query string. + _The first match is applied._ +2. Built-in query booleans - There are 3 built-in queries which can be individually enabled/disabled. These include + the `simple query string`[[1]](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-simple-query-string-query.html), + `match phrase prefix`[[2]](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-match-query-phrase-prefix.html), and + `exact match`[[3]](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-term-query.html) queries, + enabled with the following booleans + respectively [`simpleQuery`, `prefixMatchQuery`, `exactMatchQuery`] +3. `boolQuery` - The base Elasticsearch `boolean query`[[4](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html)]. + If enabled in #2 above, those queries will + appear in the `should` section of the `boolean query`[[4](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html)]. +4. `functionScore` - The Elasticsearch `function score`[[5](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-function-score-query.html#score-functions)] section of the overall query. + +#### Examples + +These examples assume a match-all `queryRegex` of `.*` so that it would impact any search query for simplicity. + +##### Example 1: Ranking By Tags/Terms + +Boost entities with tags of `primary` or `gold` and an example glossary term's uuid. + +```yaml +queryConfigurations: + - queryRegex: .* + + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + + functionScore: + functions: + - filter: + terms: + tags.keyword: + - urn:li:tag:primary + - urn:li:tag:gold + weight: 3.0 + + - filter: + terms: + glossaryTerms.keyword: + - urn:li:glossaryTerm:9afa9a59-93b2-47cb-9094-aa342eec24ad + weight: 3.0 + + score_mode: multiply + boost_mode: multiply +``` + +Similar example to boost with `primary` AND `gold` instead of the previous OR condition. + +```yaml +queryConfigurations: + - queryRegex: .* + + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + + functionScore: + functions: + - filter: + bool: + filter: + - term: + tags.keyword: urn:li:tag:primary + - term: + tags.keyword: urn:li:tag:gold + weight: 3.0 + + score_mode: multiply + boost_mode: multiply +``` + +##### Example 2: Preferred Data Platform + +Boost the `urn:li:dataPlatform:hive` platform. + +```yaml +queryConfigurations: + - queryRegex: .* + + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + + functionScore: + functions: + - filter: + terms: + platform.keyword: + - urn:li:dataPlatform:hive + weight: 3.0 + score_mode: multiply + boost_mode: multiply +``` + +##### Example 3: Exclusion & Bury + +This configuration extends the 3 built-in queries with a rule to exclude `deprecated` entities from search results +because they are not generally relevant as well as reduces the score of `materialized`. + +```yaml +queryConfigurations: + - queryRegex: .* + + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + + boolQuery: + must_not: + term: + deprecated: + value: true + + functionScore: + functions: + - filter: + term: + materialized: + value: true + weight: 0.5 + score_mode: multiply + boost_mode: multiply +``` + +##### Example 4: Entity Ranking + +Alter the ranking of entities. For example, chart vs dashboard, you may want the dashboard +to appear above charts. This can be done using the following function score and leverages a prefix match on the entity type +of the URN. Depending on the entity the weight may have to be adjusted based on your data and the entities +involved since often multiple field matches may shift weight towards one entity vs another. + +```yaml +queryConfigurations: + - queryRegex: .* + + simpleQuery: true + prefixMatchQuery: true + exactMatchQuery: true + + functionScore: + functions: + - filter: + prefix: + urn: + value: "urn:li:dashboard:" + weight: 1.5 + score_mode: multiply + boost_mode: multiply +``` + +### Search Autocomplete Configuration + +Similar to the options provided in the previous section for search configuration, there are autocomplete specific options +which can be configured. + +Note: The scoring functions defined in the previous section are inherited for autocomplete by default, unless +overrides are provided in the autocomplete section. + +For the most part the configuration options are identical to the search customization options in the previous +section, however they are located under `autocompleteConfigurations` in the yaml configuration file. + +1. `queryRegex` - Responsible for selecting the search customization based on the [regex matching](https://www.w3schools.com/java/java_regex.asp) the search query string. + _The first match is applied._ +2. The following boolean enables/disables the function score inheritance from the normal search configuration: [`inheritFunctionScore`] + This flag will automatically be set to `false` when the `functionScore` section is provided. If set to `false` with no + `functionScore` provided, the default Elasticsearch `_score` is used. +3. Built-in query booleans - There is 1 built-in query which can be enabled/disabled. These include + the `default autocomplete query` query, + enabled with the following booleans + respectively [`defaultQuery`] +4. `boolQuery` - The base Elasticsearch `boolean query`[[4](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html)]. + If enabled in #2 above, those queries will + appear in the `should` section of the `boolean query`[[4](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-bool-query.html)]. +5. `functionScore` - The Elasticsearch `function score`[[5](https://www.elastic.co/guide/en/elasticsearch/reference/7.17/query-dsl-function-score-query.html#score-functions)] section of the overall query. + +#### Examples + +These examples assume a match-all `queryRegex` of `.*` so that it would impact any search query for simplicity. Also +note that the `queryRegex` is applied individually for `searchConfigurations` and `autocompleteConfigurations` and they +do not have to be identical. + +##### Example 1: Exclude `deprecated` entities from autocomplete + +```yaml +autocompleteConfigurations: + - queryRegex: .* + defaultQuery: true + + boolQuery: + must: + - term: + deprecated: "false" +``` + +#### Example 2: Override scoring for autocomplete + +```yaml +autocompleteConfigurations: + - queryRegex: .* + defaultQuery: true + + functionScore: + functions: + - filter: + term: + materialized: + value: true + weight: 1.1 + - filter: + term: + deprecated: + value: false + weight: 0.5 + score_mode: avg + boost_mode: multiply +``` + +### Field Configurations + +Field configurations allow you to customize which fields are included in search queries and result highlighting. Each configuration is identified by a label and can modify the default behavior using three operations: + +- `add` - Adds fields to the system defaults +- `remove` - Removes fields from the system defaults +- `replace` - Completely replaces the system default field list + +As of today the UI will only exercise the `default` label, however calls to graphql can refer to other labels. See graphql documentation for further details. + +**Note:** The `replace` operation cannot be combined with `add` or `remove`. However, `add` and `remove` can be used together. + +#### Field Configuration Example + +```yaml +fieldConfigurations: + # Default configuration - if searchFlags doesn't specify + default: + searchFields: + remove: + - fieldPaths + + highlightFields: + enabled: true + remove: + - fieldPaths + + # PDL legacy configuration + legacy: + searchFields: + # No modifications - use PDL defaults + + highlightFields: + enabled: true + # No modifications - use PDL defaults + + # Configuration for technical users - adds technical fields + technical: + searchFields: + add: + - fieldPaths + - platform + remove: + - customProperties + highlightFields: + enabled: true + add: + - fieldPaths + - platform + + # Configuration for business users - simplified field set + business: + searchFields: + replace: + - name + - description + - tags + - glossaryTerms + highlightFields: + enabled: true + replace: + - name + - description + + # Configuration with no highlighting + no-highlight: + searchFields: + # Uses system defaults + highlightFields: + enabled: false +``` + +## FAQ and Troubleshooting + +**How are the results ordered?** + +The order of the search results is based on the weight what DataHub gives them based on our search algorithm. The current algorithm in OSS DataHub is based on a text-match score from Elasticsearch. + +**Where to find more information?** + +The sample queries here are non exhaustive. [The link here](https://demo.datahub.com/tag/`urn:li:tag:Searchable)` shows the current list of indexed fields for each entity inside DataHub. Click on the fields inside each entity and see which field has the tag `Searchable`. +However, it does not tell you the specific attribute name to use for specialized searches. One way to do so is to inspect the ElasticSearch indices, for example: +`curl http://localhost:9200/_cat/indices` returns all the ES indices in the ElasticSearch container. + +``` +yellow open chartindex_v2_1643510690325 bQO_RSiCSUiKJYsmJClsew 1 1 2 0 8.5kb 8.5kb +yellow open mlmodelgroupindex_v2_1643510678529 OjIy0wb7RyKqLz3uTENRHQ 1 1 0 0 208b 208b +yellow open dataprocessindex_v2_1643510676831 2w-IHpuiTUCs6e6gumpYHA 1 1 0 0 208b 208b +yellow open corpgroupindex_v2_1643510673894 O7myCFlqQWKNtgsldzBS6g 1 1 3 0 16.8kb 16.8kb +yellow open corpuserindex_v2_1643510672335 0rIe_uIQTjme5Wy61MFbaw 1 1 6 2 32.4kb 32.4kb +yellow open datasetindex_v2_1643510688970 bjBfUEswSoSqPi3BP4iqjw 1 1 15 0 29.2kb 29.2kb +yellow open dataflowindex_v2_1643510681607 N8CMlRFvQ42rnYMVDaQJ2g 1 1 1 0 10.2kb 10.2kb +yellow open dataset_datasetusagestatisticsaspect_v1_1643510694706 kdqvqMYLRWq1oZt1pcAsXQ 1 1 4 0 8.9kb 8.9kb +yellow open .ds-datahub_usage_event-000003 YMVcU8sHTFilUwyI4CWJJg 1 1 186 0 203.9kb 203.9kb +yellow open datajob_datahubingestioncheckpointaspect_v1 nTXJf7C1Q3GoaIJ71gONxw 1 1 0 0 208b 208b +yellow open .ds-datahub_usage_event-000004 XRFwisRPSJuSr6UVmmsCsg 1 1 196 0 165.5kb 165.5kb +yellow open .ds-datahub_usage_event-000005 d0O6l5wIRLOyG6iIfAISGw 1 1 77 0 108.1kb 108.1kb +yellow open dataplatformindex_v2_1643510671426 _4SIIhfAT8yq_WROufunXA 1 1 0 0 208b 208b +yellow open mlmodeldeploymentindex_v2_1643510670629 n81eJIypSp2Qx-fpjZHgRw 1 1 0 0 208b 208b +yellow open .ds-datahub_usage_event-000006 oyrWKndjQ-a8Rt1IMD9aSA 1 1 143 0 127.1kb 127.1kb +yellow open mlfeaturetableindex_v2_1643510677164 iEXPt637S1OcilXpxPNYHw 1 1 5 0 8.9kb 8.9kb +yellow open .ds-datahub_usage_event-000001 S9EnGj64TEW8O3sLUb9I2Q 1 1 257 0 163.9kb 163.9kb +yellow open .ds-datahub_usage_event-000002 2xJyvKG_RYGwJOG9yq8pJw 1 1 44 0 155.4kb 155.4kb +yellow open dataset_datasetprofileaspect_v1_1643510693373 uahwTHGRRAC7w1c2VqVy8g 1 1 31 0 18.9kb 18.9kb +yellow open mlprimarykeyindex_v2_1643510687579 MUcmT8ASSASzEpLL98vrWg 1 1 7 0 9.5kb 9.5kb +yellow open glossarytermindex_v2_1643510686127 cQL8Pg6uQeKfMly9GPhgFQ 1 1 3 0 10kb 10kb +yellow open datajob_datahubingestionrunsummaryaspect_v1 rk22mIsDQ02-52MpWLm1DA 1 1 0 0 208b 208b +yellow open mlmodelindex_v2_1643510675399 gk-WSTVjRZmkDU5ggeFSqg 1 1 1 0 10.3kb 10.3kb +yellow open dashboardindex_v2_1643510691686 PQjSaGhTRqWW6zYjcqXo6Q 1 1 1 0 8.7kb 8.7kb +yellow open datahubpolicyindex_v2_1643510671774 ZyTrYx3-Q1e-7dYq1kn5Gg 1 1 0 0 208b 208b +yellow open datajobindex_v2_1643510682977 K-rbEyjBS6ew5uOQQS4sPw 1 1 2 0 11.3kb 11.3kb +yellow open datahubretentionindex_v2 8XrQTPwRTX278mx1SrNwZA 1 1 0 0 208b 208b +yellow open glossarynodeindex_v2_1643510678826 Y3_bCz0YR2KPwCrrVngDdA 1 1 1 0 7.4kb 7.4kb +yellow open system_metadata_service_v1 36spEDbDTdKgVlSjE8t-Jw 1 1 387 8 63.2kb 63.2kb +yellow open schemafieldindex_v2_1643510684410 tZ1gC3haTReRLmpCxirVxQ 1 1 0 0 208b 208b +yellow open mlfeatureindex_v2_1643510680246 aQO5HF0mT62Znn-oIWBC8A 1 1 20 0 17.4kb 17.4kb +yellow open tagindex_v2_1643510684785 PfnUdCUORY2fnF3I3W7HwA 1 1 3 1 18.6kb 18.6kb +``` + +The index name will vary from instance to instance. Indexed information about Datasets can be found in: +`curl http://localhost:9200/datasetindex_v2_1643510688970/_search?=pretty` + +example information of a dataset: + +``` +{ + "_index" : "datasetindex_v2_1643510688970", + "_type" : "_doc", + "_id" : "urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Akafka%2CSampleKafkaDataset%2CPROD%29", + "_score" : 1.0, + "_source" : { + "urn" : "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)", + "name" : "SampleKafkaDataset", + "browsePaths" : [ + "/prod/kafka/SampleKafkaDataset" + ], + "origin" : "PROD", + "customProperties" : [ + "prop2=pikachu", + "prop1=fakeprop" + ], + "hasDescription" : false, + "hasOwners" : true, + "owners" : [ + "urn:li:corpuser:jdoe", + "urn:li:corpuser:datahub" + ], + "fieldPaths" : [ + "[version=2.0].[type=boolean].field_foo_2", + "[version=2.0].[type=boolean].field_bar", + "[version=2.0].[key=True].[type=int].id" + ], + "fieldGlossaryTerms" : [ ], + "fieldDescriptions" : [ + "Foo field description", + "Bar field description", + "Id specifying which partition the message should go to" + ], + "fieldTags" : [ + "urn:li:tag:NeedsDocumentation" + ], + "platform" : "urn:li:dataPlatform:kafka" + } + }, +``` + + + +### Related Features + +- [Metadata ingestion framework](../../metadata-ingestion/README.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/how/ui-tabs-guide.md b/docs-archive/versioned_docs/version-1.5.0/docs/how/ui-tabs-guide.md new file mode 100644 index 00000000..52bd2c46 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/how/ui-tabs-guide.md @@ -0,0 +1,25 @@ +--- +title: UI Tabs Guide +slug: /how/ui-tabs-guide +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/how/ui-tabs-guide.md +--- +# UI Tabs Guide + +Some of the tabs in the UI might not be enabled by default. This guide is supposed to tell Admins of DataHub how to enable those UI tabs. + +## Datasets + +### Stats and Queries Tab + +To enable these tabs you need to use one of the usage sources which gets the relevant metadata from your sources and ingests them into DataHub. These usage sources are listed under other sources which support them e.g. [Snowflake source](../../docs/generated/ingestion/sources/snowflake.md), [BigQuery source](../../docs/generated/ingestion/sources/bigquery.md) + +### Validation Tab + +This tab is enabled if you use [Data Quality Integration with Great Expectations](../../metadata-ingestion/integration_docs/great-expectations.md). + +## Common to multiple entities + +### Properties Tab + +Properties are a catch-all bag for metadata not captured in other aspects stored for a Dataset. These are populated via the various source connectors when [metadata is ingested](../../metadata-ingestion/README.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/how/updating-datahub.md b/docs-archive/versioned_docs/version-1.5.0/docs/how/updating-datahub.md new file mode 100644 index 00000000..45fa2d86 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/how/updating-datahub.md @@ -0,0 +1,1008 @@ +--- +title: Updating DataHub +slug: /how/updating-datahub +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/how/updating-datahub.md +--- +# Updating DataHub + + + +This file documents any backwards-incompatible changes in DataHub and assists people when migrating to a new version. + +## Next + +### Breaking Changes + +- #16396: Oracle connector: When connecting via `service_name` to a multitenant Oracle database, the database name used in URNs will now reflect the Pluggable Database (PDB) name instead of the Container Database (CDB) name. In Oracle Multitenant architecture, a CDB is the top-level container (e.g. `cdb`) and a PDB is an individual tenant database within it (e.g. `mypdb`); `service_name` typically routes to the PDB, so the PDB name is the correct identifier for your datasets. This affects both dataset URNs (when `add_database_name_to_urn: true`) and database/schema container URNs (always, since containers always include the database name). If your existing metadata was ingested with the old CDB-based URNs, re-ingesting will create new entities under the corrected URNs. To preserve the old URN shape and avoid re-creating entities, set `urn_db_name` explicitly in your recipe to match your previous CDB name. +- **Retention service disabled: only current version retained.** When the retention service is not enabled (not configured or unavailable), the write path now retains only the current version (version 0) and does not create version-history rows. Previously, version history was still written when retention was disabled. **Impact:** Deployments that run without retention enabled will no longer accumulate aspect version history; only the latest aspect value is stored. **Migration:** Enable and configure the retention service (e.g. ingest retention policies from `boot/retention.yaml`) if you need version history for any entity/aspect. +- #16134: Java 17 Runtime Required - DataHub now compiles to Java 17 bytecode (previously Java 11). All Docker images already ship with Java 17 runtime. Self-hosted users must ensure their runtime environment uses Java 17+. The Maven artifact name remains `datahub-client-java8` for backward compatibility, but now requires Java 17+ at runtime. This change also includes: + - Spark integration upgraded from 3.0.3 to 3.3.4 (minimum version required for Java 17 support) + - Hadoop upgraded from 2.7.2 to 3.3.6 (addresses Hadoop CVEs, bundled with Spark 3.3+) + - Note: If you're a self-hosted user still running Java 11, you must upgrade to Java 17 before deploying this release. Spark lineage users must upgrade to Spark 3.3.0+. +- (Frontend) CustomHttpClientFactory now restricts TLS to 1.2 and 1.3 only when using a custom truststore; TLS 1.0 and 1.1 are disabled for security. If the frontend uses a custom truststore to reach GMS or an IdP that only supports TLS 1.0/1.1, those connections will fail until the server is upgraded to support at least TLS 1.2. +- #16176: Vertex AI Source - Two breaking changes: + 1. **Pipeline URN structure changed**: Previously, each pipeline run created a new DataFlow entity. Now, Vertex AI pipelines are modeled as stable DataFlow entities (one per pipeline template), with DataJobs for tasks and DataProcessInstances for individual runs. This enables proper incremental lineage aggregation. Pipeline URNs now use the compiled pipeline spec name (`pipelineInfo.name`, e.g. the `@pipeline(name="...")` argument in Kubeflow Pipelines) as the stable identifier; non-Kubeflow pipelines fall back to `display_name` with any timestamp suffix stripped. After upgrading, existing pipeline entities will appear as separate entities from new ingestion runs. To clean up old entities, enable stateful ingestion with stale entity removal. + 2. **ML Metadata extraction now enabled by default**: Enhanced ML Metadata extraction for CustomJob lineage, hyperparameters, metrics, and model evaluations is now enabled by default. This requires additional GCP permissions: `aiplatform.metadataStores.get`, `aiplatform.metadataStores.list`, `aiplatform.executions.get`, and `aiplatform.executions.list`. If these permissions are missing, the connector gracefully falls back with warnings. To disable these features, set `use_ml_metadata_for_lineage: false`, `extract_execution_metrics: false`, and `include_evaluations: false`. +- #16149 (Ingestion) DataHub source: now uses URN pattern filtering to exclude environment-specific entities by default. This prevents copying credentials and creating invalid entities. + - **Default behavior change**: Previously `exclude_aspects` contained `dataHubIngestionSourceInfo`, `dataHubSecretValue`, `dataHubExecutionRequestInput`, `globalSettingsInfo`. Now `urn_pattern.deny` contains `urn:li:dataHubIngestionSource:.*`, `urn:li:dataHubSecret:.*`, `urn:li:globalSettings:.*`, `urn:li:dataHubExecutionRequest:.*` to exclude entire entity types instead of individual aspects. + - **Note**: If you override `urn_pattern` or `exclude_aspects` in recipe, carefully configure them based on your requirements to avoid syncing sensitive data or creating invalid entities. Recommended to keep new defaults. +- (Ingestion) Kafka Connect: The Debezium SQL Server connector now reports its platform as `mssql` instead of `sqlserver`, matching DataHub's canonical platform name. This fixes column-level lineage when using `use_schema_resolver: true`, as the SchemaResolver now correctly queries for `platform=mssql`. If you have `platform_instance_map` or `connect_to_platform_map` entries keyed on `"sqlserver"` for Debezium SQL Server connectors, update them to use `"mssql"` instead. +- #16385 Default token signing key and salt have been removed from `metadata-service/configuration/src/main/resources/application.yaml`. It is recommended to set `authentication.tokenService.signingKey` or env var `DATAHUB_TOKEN_SERVICE_SIGNING_KEY` and `authentication.tokenService.salt` or env var `DATAHUB_TOKEN_SERVICE_SALT` before starting DataHub. Refer the linked pages to know this is handled for [local development](../developers.md) and [CLI quickstart](../quickstart.md). + - If you are using helm to deploy DataHub you should be unaffected as the helm charts don't use the default values in application.yaml but rather generate a random secret to use. + - IMPACT: Due to the change in signing keys for local development and quickstart, PATs generated before this release will be invalidated and will need to be regenerated in those instances. +- #15744: The `emit_mcps()` method on `DataHubRestEmitter` now returns `List[TraceData]` instead of `int`. Previously it returned the number of chunks/batches sent. Now it returns a list of `TraceData` objects (one per batch) containing trace IDs for debugging and status checking. To get the previous chunk count, use `len(result)` on the returned list. Additionally, `emit_mcp()` now returns `Optional[TraceData]` instead of `None`. + +### Known Issues + +### Potential Downtime + +### Deprecations + +- #16176: Vertex AI Source - The `region` configuration field is deprecated in favor of `regions` (list type). The `region` field continues to work for backward compatibility. +- #16176: Vertex AI Source - Partition handling behavior will change in a future major version. Currently, `normalize_external_dataset_paths` defaults to `false`, meaning partitioned paths like `gs://bucket/data/year=2024/month=01/` create separate dataset entities. In the next major version, this will default to `true`, normalizing paths to `gs://bucket/data/` for stable dataset URNs with lineage aggregation across partitions. To opt-in to the new behavior now, set `normalize_external_dataset_paths: true` in your configuration. + +### Other Notable Changes + +- #15270 (Ingestion) Browse paths: When `platform_instance` is configured, DataFlow and DataJob entities now receive a `browsePathsV2` aspect with the platform instance as the root path. Previously, these entities had no browse path from ingestion and the backend would place them in a generic "Default" folder, causing entities from multiple platform instances to be mixed together. This affects sources like Fivetran, Glue, and Kafka-Connect that emit DataFlow/DataJob entities with `platform_instance`. Sources without `platform_instance` configured are unaffected. +- #16339 (Ingestion) Migrating Python dependency declarations from `setup.py` to `pyproject.toml` (PEP 621). `setup.py` remains the source of truth for editing dependencies for now; `pyproject.toml` is auto-generated from it via `./gradlew :metadata-ingestion:updateLockFile`. Sync verification (`./gradlew :metadata-ingestion:verifyPyprojectSync`) runs automatically during `check` and `buildWheel`. `setup.py` will be deprecated in a future release in favor of `pyproject.toml` as the sole dependency source. +- #16358 (Ingestion) dbt: Added `convert_urns_to_lowercase` config option for dbt ingestion (both dbt-core and dbt-cloud). When enabled, dbt platform URNs are lowercased, preventing duplicate entities caused by schema name casing differences (e.g., `app_sales` vs `APP_SALES`). This is an opt-in flag (default: `false` for all platforms). Recommended for case-insensitive platforms like Snowflake or BigQuery where dbt manifests may contain mixed-case identifiers. +- **Kubernetes: optional scale-down during system-update.** When the system-update job runs in a Kubernetes cluster, an optional Java-based step can scale down deployments (by label selector, e.g. MAE/MCE) and set environment variables on other deployments (by label selector, e.g. GMS) before blocking upgrades (e.g. reindex), then restore them after. Scale-down is conditional: it runs only when a blocking upgrade (such as BuildIndices when reindex is needed) requires it. Rollout and scale-down operations run in parallel. State is stored in a ConfigMap; if retries are exceeded, the step restores state, deletes the ConfigMap, and fails. The Java implementation is disabled by default; enable via Helm (`datahubSystemUpdate.scaleDown.useJavaImplementation`) and the job env vars described in [Environment variables](../deploy/environment-vars.md#kubernetes-scale-down-system-update). Outside Kubernetes or when disabled, the step is a no-op. +- #16176: Vertex AI Source Connector - Additional improvements beyond breaking changes: + - Cross-platform lineage to external data sources (GCS, BigQuery, S3, Azure Blob Storage, Snowflake) referenced in training jobs, with configurable platform instances via `platform_instance_map` to ensure URNs match native connectors + - UI organization improved with hierarchical folders: model versions appear under their model group folders, and pipeline tasks nest under their parent pipelines + - Enhanced lineage relationships: direct model→dataset relationships via TrainingData aspect; experiment run→model outputs and pipeline task run→model/dataset lineage via DataProcessInstance aspects + - For performance with large projects (1000+ training jobs), use stateful ingestion (recommended) which only processes new/changed resources on subsequent runs. Limits like `max_training_jobs_per_type` control which resources are processed and sent to DataHub (most recently updated N items) + - Optional `platform_instance` configuration field for environments running multiple Vertex AI instances + - Existing ingestion configurations continue to work without modification, and URNs remain unchanged unless `platform_instance` is explicitly configured +- #16142: Oracle ingestion: Fixed database container naming when using `service_name` instead of `database` configuration. Previously, Oracle containers had no name (only an ID) when using `service_name` with `data_dictionary_mode: ALL` (the default). Container URNs will change for affected users as the database name is now properly populated in the container GUID. If stateful ingestion is enabled, running ingestion with the latest CLI version will automatically clean up the old containers and create properly named ones. This fix also ensures database names are normalized consistently with schema and table names. +- #16300 (Ingestion) Mode connector: Performance improvements for large Mode workspaces including concurrent API fetching, response caching, and SQL parsing optimizations. Column-level lineage now emits table-level lineage as a fallback when column-level parsing times out. +- #16300 (Ingestion) SQL parsing: Join resolution for queries involving CTEs and subqueries was optimized to avoid expensive deep-copy operations that caused significant slowdowns on complex queries. This affects all connectors that use SQL-based lineage (Snowflake, BigQuery, dbt, Mode, Redshift, etc.). The new approach directly walks SQL scope sources to resolve physical tables, replacing the previous indirect column-tracing fallback. This produces more accurate join metadata — for example, unused CTEs are now correctly excluded from join tables. In rare edge cases with unusual CTE patterns, join metadata may differ from previous results, but the new output better reflects the actual query structure. + +## v1.4.0 + +### Breaking Changes + +- Python 3.9 support has been dropped. All Python modules now require Python 3.10 or later. This affects `acryl-datahub`, `acryl-datahub-airflow-plugin`, `acryl-datahub-dagster-plugin`, `acryl-datahub-gx-plugin`, `prefect-datahub`, and `acryl-datahub-actions`. Upgrade to Python 3.10+ before upgrading these packages. +- #15930: The Airflow plugin's kill switch for Airflow 2.x now uses environment variables instead of Airflow Variables. If you were using `airflow variables set datahub_airflow_plugin_disable_listener true` to disable the plugin, you must now use `export AIRFLOW_VAR_DATAHUB_AIRFLOW_PLUGIN_DISABLE_LISTENER=true` instead. +- #15877: The LDAP ingestion source now enforces TLS certificate verification by default (`tls_verify: true`). This prevents Man-in-the-Middle attacks (CWE-295) but may break existing configurations using self-signed certificates. To restore the previous behavior, explicitly set `tls_verify: false` in your recipe. +- #15748: PowerBI ingestion source `create_corp_user` default changed from `True` to `False`. Previously, PowerBI would create user entities by default, potentially overwriting existing user profiles from LDAP/Okta. Now, PowerBI emits ownership URNs only (soft references) by default. To restore previous behavior, explicitly set `ownership.create_corp_user: true` in your recipe. **Migration note**: If upgrading and users appear with incomplete profiles, re-ingest from your authoritative source (LDAP/Okta/SCIM). Additionally, when `create_corp_user: true`, the connector now emits both `CorpUserKeyClass` and `CorpUserInfoClass` (was only `CorpUserKeyClass`), providing complete user metadata. **Stateful ingestion note**: User entities created by PowerBI are now marked as non-primary (`is_primary_source=False`), so they will NOT be soft-deleted by stateful ingestion when they disappear. This prevents accidental deletion of users who may also exist in LDAP/Okta/SCIM. +- #15415: MLModel and MLModelGroup search field mapping has been updated to resolve duplicate field name conflicts. Existing entities will be automatically reindexed in the background after upgrade. New MLModel and MLModelGroup entities ingested after the upgrade will work immediately. +- #15397: Grafana ingestion source dataset granularity changed from per-datasource to per-panel (per visual). This improves lineage accuracy by ensuring each panel's query results in a unique dataset entity with precise upstream/downstream connections. Dataset URN format changed from `{ds_type}.{ds_uid}` to `{ds_type}.{ds_uid}.{dashboard_uid}.{panel_id}`. This means all existing Grafana dataset entities will have different URNs. If stateful ingestion is enabled, running ingestion with the latest CLI version will automatically clean up old entities and create new ones. Otherwise, we recommend soft deleting all Grafana datasets via the DataHub CLI: `datahub delete --platform grafana --soft` and then re-ingesting with the latest CLI version. +- #15005: `SqlParsingBuilder` is removed, use `SqlParsingAggregator` instead +- #14710: LookML ingestion source migrated to SDKv2 resulting in: + - `browsePaths` aspect replaced with `browsePathsV2` + - Only emits MCPs +- #14693: Looker ingestion source migrated to SDKv2, resulting in: + - `browsePaths` aspect replaced with `browsePathsV2` + - In `Dashboard` entitiy and `dashboardInfo` aspect, `charts` property (deprecated) is replaced with `chartEdges` + - Only emits MCPs +- #16023: In PowerBI, Container URNs change for users with `platform_instance` configured, as this PR now passes `platform_instance` to dataset containers (affects GUID generation). The `env` parameter addition is harmless as it's excluded from GUID calculation. Stateful ingestion will soft-delete old containers and create new ones on the next run. Dataset entities and their lineage are unaffected. +- #16067: Oracle stored procedure URN format has been corrected to match table URN format. For most users (using `service_name` or `database` without `add_database_name_to_urn: true`), stored procedure DataJob URNs will change from `database.schema.stored_procedures` to `schema.stored_procedures`. This fixes a URN mismatch that prevented stored procedure lineage from working. Stateful ingestion will soft-delete old stored procedure entities and create new ones with correct lineage on the next run. Users with `database` config parameter and `add_database_name_to_urn: true` are unaffected. + +### Deprecations + +- (CLI) The `--use-password` flag in `datahub init` command is deprecated. Token generation is now automatically detected when both `--username` and `--password` are provided together. The flag continues to work for backward compatibility but will be removed in a future release. + +### Other Notable Changes + +- (Ingestion) BigQuery source: Improved `dataset_pattern` filtering to apply earlier in the ingestion pipeline, reducing unnecessary API calls to BigQuery for datasets that will be filtered out. +- #15714: Kafka topic partition counts can now automatically be increased during upgrades if configured values exceed existing partition counts. Set `DATAHUB_AUTO_INCREASE_PARTITIONS=true` to enable. +- (CLI) Added `--extra-env` option to `datahub ingest deploy` command to pass environment variables as comma-separated KEY=VALUE pairs (e.g., `--extra-env "VAR1=value1,VAR2=value2"`). These are stored in the ingestion source configuration and made available to the executor at runtime. +- #14968: Added an ingestion source for IBM Db2 databases. +- #15824: The Databricks ingestion source now additionally supports authentication via M2M OAuth or Databricks unified authentication. The Azure authentication method now supports profiling as well as ingesting lineage from system tables. +- #16067: Oracle ingestion source improvements: Added procedure-to-procedure lineage support, improved PL/SQL parsing for control flow keywords (WHILE, LOOP, EXCEPTION, etc.), fixed overloaded procedure handling, and added distinct subtypes for functions vs procedures. Stored procedures and functions now have proper container hierarchy (Database → Schema → Flow) matching tables and views. Both functions and procedures are organized in the same `{schema}.stored_procedures` container (consistent with PostgreSQL, MySQL, and Snowflake), with individual subtypes to distinguish them. + +## 1.3.0 + +### Breaking Changes + +- #14580: (Ingestion) The redshift lineage v1 implementation (`RedshiftLineageExtractor`) has been removed, as lineage v2 (`RedshiftSqlLineageV2`) implementation has been default for a while already. As an effect `use_lineage_v2` config has also been removed along with all lineage v1 references and tests have been updated to v2 implementation. This should not impact most users as change is isolated in redshift ingestion source only. +- #14014: The `acryl-datahub` now requires pydantic v2. Support for pydantic v1 has been dropped and users must upgrade to pydantic v2 when using DataHub python package. + - As a side effect, this upgrade in pydantic version has implicit consequences for `iceberg` ingestion source. If it is run from CLI and `datahub` CLI was installed with all extras (`acryl-datahub[all]`), then `pyiceberg` has been kept at `0.4.0` version in such environment, just to satisfiy the pydantic v1 restriction. However now, `pyiceberg` will be installed in the newest available version. While this is a breaking change in the behaviour, versions `>0.4.0` have been used for some time by Managed Ingestion. + - Additionally, there have been changes to the catalog connection configuration details - especially for AWS-based catalogs and warehouses, the properties `profile_name`, `region_name`, `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` were deprecated and removed in version `0.8.0`. To check whether your configuration will work, consult https://py.iceberg.apache.org/configuration/#catalogs. Because of that, `pyiceberg` dependency has been restricted to be `0.8.0` at least. + - Anyway, **there are no changes needed for iceberg recipes orchestrated via the Managed Ingestion UI** +- The `acryl-datahub-airflow-plugin` now requires specifying the appropriate installation extra based on your Airflow version due to different OpenLineage dependencies: + - **For Airflow 2.x (2.7+)**: Install with `pip install 'acryl-datahub-airflow-plugin[airflow2]'` + - **For Airflow 3.x (3.0+)**: Install with `pip install 'acryl-datahub-airflow-plugin[airflow3]'` + - **For Airflow 3.0.x specifically**: You'll also need to bump pydantic: `pip install 'acryl-datahub-airflow-plugin[airflow3]' 'pydantic>=2.11.8'` + - Installing without the appropriate extra will result in missing OpenLineage dependencies and lineage extraction will not work properly. The plugin automatically detects your Airflow version and uses the appropriate integration once the correct extra is installed. +- Auto-detection for SearchClientShim requires permission for the cluster info endpoint on ElasticSearch/OpenSearch. If you are using a restrictive account to access your cluster, you may need to directly configure the engine type rather than relying on auto-detection. These are the two environment variables needed to be configured for GMS as well as MCE & MAE consumers (available in helm charts as well): + - ELASTICSEARCH_SHIM_ENGINE_TYPE & ELASTICSEARCH_SHIM_AUTO_DETECT (.Values.global.elasticsearch.engineType and .Values.global.elasticsearch.autoDetect respectively for helm) + - Allowed values: ELASTICSEARCH_SHIM_ENGINE_TYPE["ELASTICSEARCH_7", "ELASTICSEARCH_8", "OPENSEARCH_2", "AUTO_DETECT"] , ELASTICSEARCH_SHIM_AUTO_DETECT["true","false"] + +### Known Issues + +### Potential Downtime + +- #15226: Requires reindexing the query entity index on upgrade. Depending on the size of this index, upgrade times may vary. + +### Deprecations + +### Other Notable Changes + +- Airflow 3.x Support: The `acryl-datahub-airflow-plugin` now supports Apache Airflow 3.x while maintaining backward compatibility with Airflow 2.5+. Key changes include: + - **Kill Switch Migration**: The plugin's kill switch behavior has changed between versions: + - **Airflow 2.x**: Continue using Airflow Variables to disable the plugin (set the Airflow Variable `datahub_airflow_plugin_disable_listener` to `true`) + - **Airflow 3.x**: Use environment variable `AIRFLOW_VAR_DATAHUB_AIRFLOW_PLUGIN_DISABLE_LISTENER=true` to disable the plugin. This change is required to comply with Airflow 3.x's strict database access restrictions during listener initialization. + - **Configuration Changes**: Airflow 3.x moved some configuration keys (e.g., `WEBSERVER__BASE_URL` → `API__BASE_URL`). The plugin automatically detects and uses the correct configuration for each version. + - **SubDAG Removal**: SubDAG support has been removed in Airflow 3.x. Users should migrate to TaskGroups for visual grouping of tasks. + - **New SQLParser Integration**: Airflow 3.x uses a unified SQLParser patch architecture for lineage extraction, replacing operator-specific extractors. This provides better consistency and column-level lineage across all SQL operators. + - For detailed migration instructions and compatibility information, see `metadata-ingestion-modules/airflow-plugin/AIRFLOW_3_MIGRATION.md` in the DataHub repository. +- #15118: (Ingestion) The Oracle source now includes stored procedures, functions, packages, and materialized views with automatic lineage generation. Use `procedure_pattern` to filter procedures if needed. See the Oracle source documentation for permissions and configuration details. +- #14717: The Tableau ingestion source now enables `extract_lineage_from_unsupported_custom_sql_queries` by default. This improves the quality of lineage extracted by using DataHub's SQL parser in cases where the Tableau Catalog API fails to return lineage for Custom SQL queries. +- #14824: DataHub now supports CDC (Change Data Capture) mode for generating MetadataChangeLogs with guaranteed ordering based on database transaction commits. CDC mode is optional and disabled by default. When enabled via `CDC_MCL_PROCESSING_ENABLED=true`, MCLs are generated from Debezium-captured database changes rather than directly from GMS. This provides stronger ordering guarantees and decoupled processing. Requires MySQL 5.7+ or PostgreSQL 10+ with replication enabled. See [CDC Configuration Guide](configure-cdc.md) for setup instructions. +- Added multi-client search engine shim for Elasticsearch and OpenSearch support. This enables DataHub to work with ES 7.17 (with API compatibility mode for ES 8.x servers), ES 8.x, and OpenSearch 2.x through a unified interface. The shim includes auto-detection of search engine types and backward compatibility with existing RestHighLevelClient usage. See [elasticsearch-search-client-shim.md](./elasticsearch-search-client-shim.md) for configuration details. +- #14953: Unity Catalog ingestion source now supports extracting usage query history from the `system.query.history` table for improved performance with large query volumes. The new `usage_data_source` configuration (default: `AUTO`) automatically uses system tables when `warehouse_id` is configured, otherwise falls back to the REST API. This change is **not breaking** as the `AUTO` mode gracefully handles configurations without `warehouse_id` by using the existing REST API approach. Users can explicitly force system tables mode by setting `usage_data_source: SYSTEM_TABLES` (requires SELECT permission on `system.query.history`) or continue using the REST API with `usage_data_source: API`. + +## 1.2.0 + +### Breaking Changes + +- All DataHub Python packages now require Python 3.9+. This affects the following packages: + - `acryl-datahub` (DataHub CLI and SDK) + - `acryl-datahub-actions` + - `acryl-datahub-airflow-plugin` + - `acryl-datahub-prefect-plugin` + - `acryl-datahub-gx-plugin` + - `acryl-datahub-dagster-plugin` (already required Python 3.9+) +- #13619: The `acryl-datahub-airflow-plugin` has dropped support for Airflow versions less than 2.7. +- #14054: The v1 plugin in `acryl-datahub-airflow-plugin` has been removed. The v2 plugin has been the default for a while already, so this should not impact most users. Users who were explicitly setting `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN=true` will need to either upgrade or pin to an older version to continue using the v1 plugin. +- #14015: In the sql-queries source, the `default_dialect` configuration parameter has been renamed to `override_dialect`. This also affects the Python SDK methods: + - `DataHubGraph.parse_sql_lineage(default_dialect=...)` → `DataHubGraph.parse_sql_lineage(override_dialect=...)` + - `LineageClient.add_lineage_via_sql(default_dialect=...)` → `LineageClient.add_lineage_via_sql(override_dialect=...)` +- #14059: The `acryl-datahub-gx-plugin` now requires pydantic v2, which means the effective minimum supported version of GX is 0.17.15 (from Sept 2023). +- #13601: The `use_queries_v2` flag is now enabled by default for Snowflake and BigQuery ingestion. This improves the quality of lineage and quantity of queries extracted. + +### Known Issues + +- Internal Schema Registry - The internal schema registry does not supply a compatible schema for older MCP messages. The short term recommendation is to process all MCPs before upgrading to this release. + +### Potential Downtime + +### Deprecations + +- #13858: For folks using `bigquery` and `redshift` connectors please update `schema_pattern` to match against fully qualified schema name `.` and set config `match_fully_qualified_names : True`. Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. The config option `match_fully_qualified_names` will be removed in future and the default behavior will be like `match_fully_qualified_names: True`. + +### Other Notable Changes + +- The `acryl-datahub-actions` package now requires Pydantic V2, while it previously was compatible with both Pydantic V1 and V2. +- #14123: Adds a new environment variable `DATAHUB_REST_EMITTER_BATCH_MAX_PAYLOAD_BYTES` to control batch size limits when using the RestEmitter in ingestions. Default is 15MB but configurable. + +## 1.1.0 + +### Breaking Changes + +- #12795: slack source v2 - now ingests all user and channels. Make sure the ingestion source and GMS are on the same version of DataHub. +- #13004: The `acryl-datahub-airflow-plugin` dropped support for Airflow 2.3 and 2.4. +- #13186: NoCode Migration Removed - This code hasn't been required in many years. If needed, a user should upgrade to DataHub 1.0.x prior to upgrading to a later version. +- #13397: `async_flag` removed from rest emitter, replaced with emit mode ASYNC +- #13120: DataHub Actions Integration: Moved datahub-actions into OSS DataHub. Users building their own DataHub Actions docker images can now build from the OSS project. + +### Known Issues + +- #13397: Ingestion Rest Emitter + - ASYNC_WAIT/ASYNC - Async modes are impacted by kafka lag. + - SYNC_WAIT - Only available with OpenAPI ingestion +- OpenAPI Reports OpenAPI Spec 3.1.0 when it only supports 3.0.1 + +### Potential Downtime + +### Deprecations + +### Other Notable Changes + +- SDK Improvements: Added URN to URL helpers, lineage client, ML model support, and search enhancements. +- Connector Improvements: Significant enhancements to PowerBI, Superset, MLflow, and Redshift connectors. +- New Connectors: Added connectors for Hex notebooks and VertexAI. +- Iceberg Connector Refactoring: The Iceberg connector has been completely refactored to use MCPWs instead of MCEs. +- Entity Subtypes Model Change: The addition of subtypes to most entities. +- #13151 #13255: UI Search Bar Redesign: search bar with new autocomplete functionality has been added. +- #12976: Edit Lineage Feature: Added ability to edit lineage directly in the UI. +- #12983 #13107: Manage Tags Interface: Added a dedicated "Manage Tags" navigation page. +- #13361: Theme Support: Added foundation for basic theme support with primary color configuration. +- #13165 - OpenAPI v3 Patching Improvements +- #13397 - Ingestion Rest Emitter - Added EmitMode parameter for write guarantees. + - SYNC_WAIT: Synchronously updates the primary storage (SQL) but asynchronously updates search storage (Elasticsearch). Provides a balance between consistency and performance. Suitable for updates that need to be immediately reflected in direct entity retrievals but where search index consistency can be slightly delayed. + - SYNC_PRIMARY: Synchronously updates the primary storage (SQL) but asynchronously updates search storage (Elasticsearch). Provides a balance between consistency and performance. Suitable for updates that need to be immediately reflected in direct entity retrievals but where search index consistency can be slightly delayed. + - ASYNC: Queues the metadata change for asynchronous processing and returns immediately. The client continues execution without waiting for the change to be fully processed. Best for high-throughput scenarios where eventual consistency is acceptable. + - ASYNC_WAIT: Queues the metadata change asynchronously but blocks until confirmation that the write has been fully persisted. More efficient than fully synchronous operations due to backend parallelization and batching while still providing strong consistency guarantees. Useful when you need confirmation of successful persistence without sacrificing performance. +- #13426 - Added support for extracting column transformation logic when using `use_queries_v2` with warehouse ingestion. +- #13499 - Added ELASTICSEARCH_MIN_SEARCH_FILTER_LENGTH configuration for ElasticSearch index config. If modified from the default, this configuration can have significant impact on search performance if changed and will trigger reindexing causing large delays in updates. Most users will not want to modify this. + +## 1.0.0 + +### Breaking Changes + +- #12673: Business Glossary ID generation has been modified to handle special characters and URL cleaning. When `enable_auto_id` is false (default), IDs are now generated by cleaning the name (converting spaces to hyphens, removing special characters except periods which are used as path separators) while preserving case. This may result in different IDs being generated for terms with special characters. + +- #12580: The OpenAPI source handled nesting incorrectly. 12580 fixes it to create proper nested field paths, however, this will re-write the incorrect schemas of existing OpenAPI runs. + +- #12408: The `platform` field in the DataPlatformInstance GraphQL type is removed. Clients need to retrieve the platform via the optional `dataPlatformInstance` field. + +- #12671: The `priority` field of the Incident entity is changed from an integer to an enum. This field was previously completely unused in UI and API, so this change should not affect existing deployments. + +- #12716: Fix the `platform_instance` being added twice to the URN. If you want to have the previous behavior back, you need to add your platform_instance twice (i.e. `plat.plat`). + +- #12797: Previously endpoints when used in ASYNC mode would not validate URNs, entity & aspect names immediately. Starting with this release, even in ASYNC mode, these requests will be returned with http code 4xx. This includes URN, Entity Type names, Entity & Aspect names. + +### Known Issues + +- #12601: Jetty 12 introduces a stricter handling of url encoding. We are currently applying a workaround to prevent a regression, while technically breaking the official specifications. +- #12714: API Tracing requires at least one mutation of the aspect being updated using this version of DataHub. +- #12797: See Breaking Change above. Entity Type names are case sensitive, this will result in 4xx exceptions when this rule is violated. +- Python SDK v1.0.0.3 - direct accesses to the `server_config` property on `DataHubRestEmitter` can throw an unknown attribute error if `test_connection` is not called prior to directly accessing it as the default empty map initialization was removed. This is resolved in v1.1.0. + +### Potential Downtime + +### Deprecations + +### Other Notable Changes + +- #12641: Adds a new MCP validator that prevents deletes of any `CorpUser` entity that has a newly introduced `CorpUserInfo#system` flag set to true. +- #12433: Fixes the searchable annotations in the model supporting `Dashboard` to `Dashboard` lineage within the `DashboardInfo` aspect. Mainly, users of Sigma and PowerBI Apps ingestion may be affected by this adjustment. Consequently, a [reindex](/docs/how/restore-indices/) will be automatically triggered during the system upgrade. + +## 0.15.0 + +- OpenAPI Update: PIT Keep Alive parameter added to scroll endpoints. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set. +- OpenAPI Update 2: Previously there was an incorrectly marked parameter named `sort` on the generic list entities endpoint for v3. This parameter is deprecated and only supports a single string value while the documentation indicates it supports a list of strings. This documentation error has been fixed and the correct field, `sortCriteria`, is now documented which supports a list of strings. + +### Known Issues + +- Persistence Exception: No Rows Updated may occur if a transaction does not change any aspect's data. + +### Breaking Changes + +- #12223: For dbt Cloud ingestion, the "View in dbt" link will point at the "Explore" page in the dbt Cloud UI. You can revert to the old behavior of linking to the dbt Cloud IDE by setting `external_url_mode: ide". +- #12191 - Configs `include_view_lineage` and `include_view_column_lineage` are removed from snowflake ingestion source. View and External Table DDL lineage will always be ingested when definitions are available. +- #12181 - Configs `include_view_lineage`, `include_view_column_lineage` and `lineage_parse_view_ddl` are removed from bigquery ingestion source. View and Snapshot lineage will always be ingested when definitions are available. +- #12077: `Kafka` source no longer ingests schemas from schema registry as separate entities by default, set `ingest_schemas_as_entities` to `true` to ingest them +- #11486 - Criterion's `value` parameter has been previously deprecated. Use of `value` instead of `values` is no longer supported and will be completely removed on the next major version. +- #11484 - Metadata service authentication enabled by default +- #11484 - Rest API authorization enabled by default +- #10472 - `SANDBOX` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases. +- #11619 - schema field/column paths can no longer be empty strings +- #11619 - schema field/column paths can no longer be duplicated within the schema +- #11570 - The `DatahubClientConfig`'s server field no longer defaults to `http://localhost:8080`. Be sure to explicitly set this. +- #11570 - If a `datahub_api` is explicitly passed to a stateful ingestion config provider, it will be used. We previously ignored it if the pipeline context also had a graph object. +- #11518 - DataHub Garbage Collection: Various entities that are soft-deleted (after 10d) or are timeseries _entities_ (dataprocess, execution requests) will be removed automatically using logic in the `datahub-gc` ingestion source. +- #12020 - Removed `sql_parser` configuration from the Redash source, as Redash now exclusively uses the sqlglot-based parser for lineage extraction. +- #12020 - Removed `datahub.utilities.sql_parser`, `datahub.utilities.sql_parser_base` and `datahub.utilities.sql_lineage_parser_impl` module along with `SqlLineageSQLParser` and `DefaultSQLParser`. Use `create_lineage_sql_parsed_result` from `datahub.sql_parsing.sqlglot_lineage` module instead. +- #11518 - DataHub Garbage Collection: Various entities that are soft-deleted + (after 10d) or are timeseries _entities_ (dataprocess, execution requests) + will be removed automatically using logic in the `datahub-gc` ingestion + source. +- #12067 - Default behavior of DataJobPatchBuilder in Python sdk has been + changed to NOT fill out `created` and `lastModified` auditstamps by default + for input and output dataset edges. This should not have any user-observable + impact (time-based lineage viz will still continue working based on observed time), but could break assumptions previously being made by clients. +- #12158 - Users provisioned with `user.props` will need to be enabled before login in order to be granted access to DataHub. +- #13273 - When using Opensearch, we will use 'zstd-no-dict' codec instead of 'default'. Elasticsearch still uses 'default' + +### Potential Downtime + +### Deprecations + +- #12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2. +- #11701: The Fivetran `sources_to_database` field is deprecated in favor of setting directly within `sources_to_platform_instance..database`. +- #11560 - The PowerBI ingestion source configuration option include_workspace_name_in_dataset_urn determines whether the workspace name is included in the PowerBI dataset's URN.
PowerBI allows to have identical name of semantic model and their tables across the workspace, It will overwrite the semantic model in-case of multi-workspace ingestion.
+ Entity urn with `include_workspace_name_in_dataset_urn: false` + + ``` + urn:li:dataset:(urn:li:dataPlatform:powerbi,[.].,) + ``` + + Entity urn with `include_workspace_name_in_dataset_urn: true` + + ``` + urn:li:dataset:(urn:li:dataPlatform:powerbi,[.]...,) + ``` + + The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatibility, However, we recommend enabling this flag after performing the necessary cleanup. + If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI: + `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true. + +### Other Notable Changes + +- #12236: Data flow and data job entities may additionally produce container aspect that will require a corresponding upgrade of server. Otherwise server can reject the aspect. +- #12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation. +- #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information. +- #11549 - Manage Operations Privilege is extended from throttle control to all system management and operations APIs. + +## 0.14.1 + +### Breaking Changes + +- #9857 (#10773) `lower` method was removed from `get_db_name` of `SQLAlchemySource` class. This change will affect the urns of all related to `SQLAlchemySource` entities. + + Old `urn`, where `data_base_name` is `Some_Database`: + + ``` + - urn:li:dataJob:(urn:li:dataFlow:(mssql,demodata.Foo.stored_procedures,PROD),Proc.With.SpecialChar) + ``` + + New `urn`, where `data_base_name` is `Some_Database`: + + ``` + - urn:li:dataJob:(urn:li:dataFlow:(mssql,DemoData.Foo.stored_procedures,PROD),Proc.With.SpecialChar) + ``` + + Re-running with stateful ingestion should automatically clear up the entities with old URNS and add entities with new URNs, therefore not duplicating the containers or jobs. + +- #11313 - `datahub get` will no longer return a key aspect for entities that don't exist. +- #11369 - The default datahub-rest sink mode has been changed to `ASYNC_BATCH`. This requires a server with version 0.14.0+. +- #11214 Container properties aspect will produce an additional field that will require a corresponding upgrade of server. Otherwise server can reject the aspects. +- #10190 - `extractor_config.set_system_metadata` of `datahub` source has been moved to be a top level config in the recipe under `flags.set_system_metadata` + +### Potential Downtime + +### Deprecations + +### Other Notable Changes + +- Downgrade to previous version is not automatically supported. +- Data Product Properties Unset side effect introduced + - Previously, Data Products could be set as linked to multiple Datasets if modified directly via the REST API rather than linked through the UI or GraphQL. This side effect aligns the REST API behavior with the GraphQL behavior by introducting a side effect that enforces the 1-to-1 constraint between Data Products and Datasets + - NOTE: There is a pathological pattern of writes for Data Products that can introduce issues with write processing that can occur with this side effect. If you are constantly changing all of the Datasets associated with a Data Product back and forth between multiple Data Products it will result in a high volume of writes due to the need to unset previous associations. + +## 0.14.0.2 + +### Breaking Changes + +- Protobuf CLI will no longer create binary encoded protoc custom properties. Flag added `-protocProp` in case this + behavior is required. +- #10814 Data flow info and data job info aspect will produce an additional field that will require a corresponding upgrade of server. Otherwise server can reject the aspects. +- #10868 - OpenAPI V3 - Creation of aspects will need to be wrapped within a `value` key and the API is now symmetric with respect to input and outputs. + +Example Global Tags Aspect: + +Previous: + +```json +{ + "tags": [ + { + "tag": "string", + "context": "string" + } + ] +} +``` + +New (optional fields `systemMetadata` and `headers`): + +```json +{ + "value": { + "tags": [ + { + "tag": "string", + "context": "string" + } + ] + }, + "systemMetadata": {}, + "headers": {} +} +``` + +- #10858 Profiling configuration for Glue source has been updated. + + Previously, the configuration was: + + ```yaml + profiling: {} + ``` + + Now, it needs to be: + + ```yaml + profiling: + enabled: true + ``` + +### Potential Downtime + +### Deprecations + +- OpenAPI v1: OpenAPI v1 is collectively defined as all endpoints which are not prefixed with `/v2` or `/v3`. The v1 endpoints + will be deprecated in no less than 6 months. Endpoints will be replaced with equivalents in the `/v2` or `/v3` APIs. + No loss of functionality expected unless explicitly mentioned in Breaking Changes. + +### Other Notable Changes + +- #10498 - Tableau ingestion can now be configured to ingest multiple sites at once and add the sites as containers. The feature is currently only available for Tableau Server. +- #10466 - Extends configuration in `~/.datahubenv` to match `DatahubClientConfig` object definition. See full configuration in https://docs.datahub.com/docs/python-sdk/clients/. The CLI should now respect the updated configurations specified in `~/.datahubenv` across its functions and utilities. This means that for systems where ssl certification is disabled, setting `disable_ssl_verification: true` in `~./datahubenv` will apply to all CLI calls. +- #11002 - We will not auto-generate a `~/.datahubenv` file. You must either run `datahub init` to create that file, or set environment variables so that the config is loaded. +- #11023 - Added a new parameter to datahub's `put` cli command: `--run-id`. This parameter is useful to associate a given write to an ingestion process. A use-case can be mimick transformers when a transformer for aspect being written does not exist. +- #11051 - Ingestion reports will now trim the summary text to a maximum of 800k characters to avoid generating `dataHubExecutionRequestResult` that are too large for GMS to handle. + +## 0.13.3 + +### Breaking Changes + +- #10419 - `aws_region` is now a required configuration in the DynamoDB connector. The connector will no longer loop through all AWS regions; instead, it will only use the region passed into the recipe configuration. +- #10389 - Custom validators, mutators, side-effects dropped a previously required constructor +- #10472 - `RVW` added as a FabricType. No rollbacks allowed once metadata with this fabric type is added without manual cleanups in databases. + +### Potential Downtime + +### Deprecations + +### Other Notable Change + +## 0.13.1 + +### Breaking Changes + +- #9934 and #10075 - Stateful ingestion is now enabled by default if a `pipeline_name` is set and either a datahub-rest sink or `datahub_api` is specified. It will still be disabled by default when any other sink type is used or if there is no pipeline name set. +- #10002 - The `DataHubGraph` client no longer makes a request to the backend during initialization. If you want to preserve the old behavior, call `graph.test_connection()` after constructing the client. +- #10026 - The dbt `use_compiled_code` option has been removed, because we now support capturing both source and compiled dbt SQL. This can be configured using `include_compiled_code`, which will be default enabled in 0.13.1. +- #10055 - Assertion entities generated by dbt are now associated with the dbt dataset entity, and not the entity in the data warehouse. +- #10090 - For Redshift ingestion, `use_lineage_v2` is now enabled by default. +- #10147 - For looker ingestion, the browse paths for looker Dashboard, Chart, View, Explore have been updated to align with Looker UI. This does not affect URNs or lineage but primarily affects (improves) browsing experience. +- #10164 - For dbt ingestion, `entities_enabled.model_performance` and `include_compiled_code` are now both enabled by default. Upgrading dbt ingestion will also require upgrading the backend to 0.13.1. +- #10066 - For view access controls, `SEARCH_AUTHORIZATION_ENABLED` replaced by `VIEW_AUTHORIZATION_ENABLED` to more accurately represent the feature. +- #8231 - Google Analytics 3 has been fully sunsetted by Google as of July 2023, so we now support GA4 thanks to this PR and no longer support GA3 (which would have been broken since last year anyways). +- #10278 - Renaming Presto-On-Hive Source to Hive Metastore source to reflect better its purpose + +### Potential Downtime + +### Deprecations + +### Other Notable Changes + +## 0.13.0 + +### Breaking Changes + +- Updating MySQL version for quickstarts to 8.2, may cause quickstart issues for existing instances. +- Neo4j 5.x, may require migration from 4.x +- Build requires JDK17 (Runtime Java 11) +- Build requires Docker Compose > 2.20 +- #9731 - The `acryl-datahub` CLI now requires Python 3.8+ +- #9601 - The Unity Catalog(UC) ingestion source config `include_metastore` is now disabled by default. This change will affect the urns of all entities in the workspace.
+ Entity Hierarchy with `include_metastore: true` (Old) + + ``` + - UC Metastore + - Catalog + - Schema + - Table + ``` + + Entity Hierarchy with `include_metastore: false` (New) + + ``` + - Catalog + - Schema + - Table + ``` + + We recommend using `platform_instance` for differentiating across metastores. + + If stateful ingestion is enabled, running ingestion with latest cli version will perform all required cleanup. Otherwise, we recommend soft deleting all databricks data via the DataHub CLI: + `datahub delete --platform databricks --soft` and then reingesting with latest cli version. + +- #9601 - The Unity Catalog(UC) ingestion source config `include_hive_metastore` is now enabled by default. This requires config `warehouse_id` to be set. You can disable `include_hive_metastore` by setting it to `False` to avoid ingesting legacy hive metastore catalog in Databricks. +- #9904 - The default Redshift `table_lineage_mode` is now MIXED, instead of `STL_SCAN_BASED`. Improved lineage generation is also available by enabling `use_lineaege_v2`. This v2 implementation will become the default in a future release. + +### Potential Downtime + +### Deprecations + +- Spark 2.x (including previous JDK8 build requirements) + +### Other Notable Changes + +## 0.12.1 + +### Breaking Changes + +- #9244: The `redshift-legacy` and `redshift-legacy-usage` sources, which have been deprecated for >6 months, have been removed. The new `redshift` source is a superset of the functionality provided by those legacy sources. +- `database_alias` config is no longer supported in SQL sources namely - Redshift, MySQL, Oracle, Postgres, Trino, Presto-on-hive. The config will automatically be ignored if it's present in your recipe. It has been deprecated since v0.9.6. +- #9257: The Python SDK urn types are now autogenerated. The new classes are largely backwards compatible with the previous, manually written classes, but many older methods are now deprecated in favor of a more uniform interface. The only breaking change is that the signature for the director constructor e.g. `TagUrn("tag", ["tag_name"])` is no longer supported, and the simpler `TagUrn("tag_name")` should be used instead. + The canonical place to import the urn classes from is `datahub.metadata.urns.*`. Other import paths, like `datahub.utilities.urns.corpuser_urn.CorpuserUrn` are retained for backwards compatibility, but are considered deprecated. +- #9286: The `DataHubRestEmitter.emit` method no longer returns anything. It previously returned a tuple of timestamps. +- #8951: A great expectations based profiler has been added for the Unity Catalog source. + To use the old profiler, set `method: analyze` under the `profiling` section in your recipe. + To use the new profiler, set `method: ge`. Profiling is disabled by default, so to enable it, + one of these methods must be specified. + +### Potential Downtime + +### Deprecations + +### Other Notable Changes + +## 0.12.0 + +### Breaking Changes + +- #8687 (datahub-helm #365 #353) - If Helm is used for installation and Neo4j is enabled, update the prerequisites Helm chart to version >=0.1.2 and adjust your value overrides in the `neo4j:` section according to the new structure. +- #9044 - GraphQL APIs for adding ownership now expect either an `ownershipTypeUrn` referencing a customer ownership type or a (deprecated) `type`. Where before adding an ownership without a concrete type was allowed, this is no longer the case. For simplicity you can use the `type` parameter which will get translated to a custom ownership type internally if one exists for the type being added. +- #9010 - In Redshift source's config `incremental_lineage` is set default to off. +- #8810 - Removed support for SQLAlchemy 1.3.x. Only SQLAlchemy 1.4.x is supported now. +- #8942 - Removed `urn:li:corpuser:datahub` owner for the `Measure`, `Dimension` and `Temporal` tags emitted + by Looker and LookML source connectors. +- #8853 - The Airflow plugin no longer supports Airflow 2.0.x or Python 3.7. See the docs for more details. +- #8853 - Introduced the Airflow plugin v2. If you're using Airflow 2.3+, the v2 plugin will be enabled by default, and so you'll need to switch your requirements to include `pip install 'acryl-datahub-airflow-plugin[plugin-v2]'`. To continue using the v1 plugin, set the `DATAHUB_AIRFLOW_PLUGIN_USE_V1_PLUGIN` environment variable to `true`. +- #8943 - The Unity Catalog ingestion source has a new option `include_metastore`, which will cause all urns to be changed when disabled. + This is currently enabled by default to preserve compatibility, but will be disabled by default and then removed in the future. + If stateful ingestion is enabled, simply setting `include_metastore: false` will perform all required cleanup. + Otherwise, we recommend soft deleting all databricks data via the DataHub CLI: + `datahub delete --platform databricks --soft` and then reingesting with `include_metastore: false`. +- #8846 - Changed enum values in resource filters used by policies. `RESOURCE_TYPE` became `TYPE` and `RESOURCE_URN` became `URN`. + Any existing policies using these filters (i.e. defined for particular `urns` or `types` such as `dataset`) need to be upgraded + manually, for example by retrieving their respective `dataHubPolicyInfo` aspect and changing part using filter i.e. + +```yaml + "resources": { + "filter": { + "criteria": [ + { + "field": "RESOURCE_TYPE", + "condition": "EQUALS", + "values": [ + "dataset" + ] + } + ] + } +``` + +into + +```yaml + "resources": { + "filter": { + "criteria": [ + { + "field": "TYPE", + "condition": "EQUALS", + "values": [ + "dataset" + ] + } + ] + } +``` + +for example, using `datahub put` command. Policies can be also removed and re-created via UI. + +- #9077 - The BigQuery ingestion source by default sets `match_fully_qualified_names: true`. + This means that any `dataset_pattern` or `schema_pattern` specified will be matched on the fully + qualified dataset name, i.e. `.`. We attempt to support the old + pattern format by prepending `.*\\.` to dataset patterns lacking a period, so in most cases this + should not cause any issues. However, if you have a complex dataset pattern, we recommend you + manually convert it to the fully qualified format to avoid any potential issues. +- #9110 - The Unity Catalog source will now generate urns based on `env` properly. If you have + been setting `env` in your recipe to something besides `PROD`, we will now generate urns + with that new env variable, invalidating your existing urns. + +### Potential Downtime + +### Deprecations + +### Other Notable Changes + +- Session token configuration has changed, all previously created session tokens will be invalid and users will be prompted to log in. Expiration time has also been shortened which may result in more login prompts with the default settings. + There should be no other interruption due to this change. + +## 0.11.0 + +### Breaking Changes + +### Potential Downtime + +- #8611 Search improvements requires reindexing indices. A `system-update` job will run which will set indices to read-only and create a backup/clone of each index. During the reindexing new components will be prevented from start-up until the reindex completes. The logs of this job will indicate a % complete per index. Depending on index sizes and infrastructure this process can take 5 minutes to hours however as a rough estimate 1 hour for every 2.3 million entities. + +### Deprecations + +- #8525: In LDAP ingestor, the `manager_pagination_enabled` changed to general `pagination_enabled` +- MAE Events are no longer produced. MAE events have been deprecated for over a year. + +### Other Notable Changes + +- In this release we now enable you to create and delete pinned announcements on your DataHub homepage! If you have the “Manage Home Page Posts” platform privilege you’ll see a new section in settings called “Home Page Posts” where you can create and delete text posts and link posts that your users see on the home page. +- The new search and browse experience, which was first made available in the previous release behind a feature flag, is now on by default. Check out our release notes for v0.10.5 to get more information and documentation on this new Browse experience. +- In addition to the ranking changes mentioned above, this release includes changes to the highlighting of search entities to understand why they match your query. You can also sort your results alphabetically or by last updated times, in addition to relevance. In this release, we suggest a correction if your query has a typo in it. +- #8300: Clickhouse source now inherited from TwoTierSQLAlchemy. In old way we have platform_instance -> container -> co + container db (None) -> container schema and now we have platform_instance -> container database. +- #8300: Added `uri_opts` argument; now we can add any options for clickhouse client. +- #8659: BigQuery ingestion no longer creates DataPlatformInstance aspects by default. + This will only affect users that were depending on this aspect for custom functionality, + and can be enabled via the `include_data_platform_instance` config option. +- OpenAPI entity and aspect endpoints expanded to improve developer experience when using this API with additional aspects to be added in the near future. +- The CLI now supports recursive deletes. +- Batching of default aspects on initial ingestion (SQL) +- Improvements to multi-threading. Ingestion recipes, if previously reduced to 1 thread, can be restored to the 15 thread default. +- Gradle 7 upgrade moderately improves build speed +- DataHub Ingestion slim images reduced in size by 2GB+ +- Glue Schema Registry fixed + +## 0.10.5 + +### Breaking Changes + +- #8201: Python SDK: In the DataFlow class, the `cluster` argument is deprecated in favor of `env`. +- #8263: Okta source config option `okta_profile_to_username_attr` default changed from `login` to `email`. + This determines which Okta profile attribute is used for the corresponding DataHub user + and thus may change what DataHub users are generated by the Okta source. And in a follow up `okta_profile_to_username_regex` has been set to `.*` which taken together with previous change brings the defaults in line with OIDC. +- #8331: For all sql-based sources that support profiling, you can no longer specify + `profile_table_level_only` together with `include_field_xyz` config options to ingest + certain column-level metrics. Instead, set `profile_table_level_only` to `false` and + individually enable / disable desired field metrics. +- #8451: The `bigquery-beta` and `snowflake-beta` source aliases have been dropped. Use `bigquery` and `snowflake` as the source type instead. +- #8472: Ingestion runs created with Pipeline.create will show up in the DataHub ingestion tab as CLI-based runs. To revert to the previous behavior of not showing these runs in DataHub, pass `no_default_report=True`. +- #8513: `snowflake` connector will use user's `email` attribute as is in urn. To revert to previous behavior disable `email_as_user_identifier` in recipe. + +### Potential Downtime + +- BrowsePathsV2 upgrade will now be handled by the `system-update` job in non-blocking mode. This process generates data needed for the new search + and browse feature. This process must complete before enabling the new search and browse UI and while upgrading entities will be missing from the UI. + If not using the new search and browse UI, there will be no impact and the update will complete in the background. + +### Deprecations + +- #8198: In the Python SDK, the `PlatformKey` class has been renamed to `ContainerKey`. + +### Other Notable Changes + +0.10.5 introduces the new Unified Search & Browse experience and is disabled by default. You can control whether or not you want to see just the new search filtering experience, the new search and browse experience together, or keep the existing search and browse experiences by toggling the two environment variable feature flags `SHOW_SEARCH_FILTERS_V2` and `SHOW_BROWSE_V2` in your GMS container. + +**Upgrade Considerations:** + +- With the release of Browse V2, we have created a job to run in GMS that will backfill your existing data with new `browsePathsV2` aspects. This job loops over entity types that need a `browsePathsV2` aspect (Dataset, Dashboard, Chart, DataJob, DataFlow, MLModel, MLModelGroup, MLFeatureTable, and MLFeature) and generates one for them. For entities that may have Container parents (Datasets and Dashboards) we will try to fetch their parent containers in order to generate this new aspect. For those deployments with large amounts of data, consider whether running this upgrade job makes sense as it may be a heavy operation and take some time to complete. If you wish to skip this job, simply set the `BACKFILL_BROWSE_PATHS_V2` environment variable flag to `false` in your GMS container. Without this backfill job, though, you will need to rely on the newest CLI of ingestion to create these `browsePathsV2` aspects when running ingestion otherwise your browse sidebar will be out-of-sync. +- Since the new browse experience replaces the old, consider whether having the `SHOW_BROWSE_V2` environment variable feature flag on is the right decision for your organization. If you’re creating custom browse paths with the `browsePaths` aspect, you can continue to do the same with the new experience, however you will have to generate `browsePathsV2` aspects instead which are documented [here](/docs/browsev2/browse-paths-v2/). + +## 0.10.4 + +### Breaking Changes + +### Potential Downtime + +### Deprecations + +- #8045: With the introduction of custom ownership types, the `Owner` aspect has been updated where the `type` field is deprecated in favor of a new field `typeUrn`. This latter field is an urn reference to the new OwnershipType entity. GraphQL endpoints have been updated to use the new field. For pre-existing ownership aspect records, DataHub now has logic to map the old field to the new field. + +### Other notable Changes + +- #8191: Updates GMS's health check endpoint to account for its dependency on external components. Notably, at this time, elasticsearch. This means that DataHub operators can now use GMS health status more reliably. + +## 0.10.3 + +### Breaking Changes + +- #7900: The `catalog_pattern` and `schema_pattern` options of the Unity Catalog source now match against the fully qualified name of the catalog/schema instead of just the name. Unless you're using regex `^` in your patterns, this should not affect you. +- #7942: Renaming the `containerPath` aspect to `browsePathsV2`. This means any data with the aspect name `containerPath` will be invalid. We had not exposed this in the UI or used it anywhere, but it was a model we recently merged to open up other work. This should not affect many people if anyone at all unless you were manually creating `containerPath` data through ingestion on your instance. +- #8068: In the `datahub delete` CLI, if an `--entity-type` filter is not specified, we automatically delete across all entity types. The previous behavior was to use a default entity type of dataset. +- #8068: In the `datahub delete` CLI, the `--start-time` and `--end-time` parameters are not required for timeseries aspect hard deletes. To recover the previous behavior of deleting all data, use `--start-time min --end-time max`. + +### Potential Downtime + +### Deprecations + +- The signature of `Source.get_workunits()` is changed from `Iterable[WorkUnit]` to the more restrictive `Iterable[MetadataWorkUnit]`. +- Legacy usage creation via the `UsageAggregation` aspect, `/usageStats?action=batchIngest` GMS endpoint, and `UsageStatsWorkUnit` metadata-ingestion class are all deprecated. + +### Other notable Changes + +## 0.10.2 + +### Breaking Changes + +- #7016 Add `add_database_name_to_urn` flag to Oracle source which ensure that Dataset urns have the DB name as a prefix to prevent collision (.e.g. {database}.{schema}.{table}). ONLY breaking if you set this flag to true, otherwise behavior remains the same. +- The Airflow plugin no longer includes the DataHub Kafka emitter by default. Use `pip install acryl-datahub-airflow-plugin[datahub-kafka]` for Kafka support. +- The Airflow lineage backend no longer includes the DataHub Kafka emitter by default. Use `pip install acryl-datahub[airflow,datahub-kafka]` for Kafka support. +- Java SDK PatchBuilders have been modified in a backwards incompatible way to align more with the Python SDK and support more use cases. Any application utilizing the Java SDK for patch building may be affected on upgrading this dependency. + +### Deprecations + +- The docker image and script for updating from Elasticsearch 6 to 7 is no longer being maintained and will be removed from the `/contrib` section of + the repository. Please refer to older releases if needed. + +## 0.10.0 + +### Breaking Changes + +- #7103 This should only impact users who have configured explicit non-default names for DataHub's Kafka topics. The environment variables used to configure Kafka topics for DataHub used in the `kafka-setup` docker image have been updated to be in-line with other DataHub components, for more info see our docs on [Configuring Kafka in DataHub + ](https://docs.datahub.com/docs/how/kafka-config). They have been suffixed with `_TOPIC` where as now the correct suffix is `_TOPIC_NAME`. This change should not affect any user who is using default Kafka names. +- #6906 The Redshift source has been reworked and now also includes usage capabilities. The old Redshift source was renamed to `redshift-legacy`. The `redshift-usage` source has also been renamed to `redshift-usage-legacy` will be removed in the future. + +### Potential Downtime + +- #6894 Search improvements requires reindexing indices. A `system-update` job will run which will set indices to read-only and create a backup/clone of each index. During the reindexing new components will be prevented from start-up until the reindex completes. The logs of this job will indicate a % complete per index. Depending on index sizes and infrastructure this process can take 5 minutes to hours however as a rough estimate 1 hour for every 2.3 million entities. + +#### Helm Notes + +Helm without `--atomic`: The default timeout for an upgrade command is 5 minutes. If the reindex takes longer (depending on data size) it will continue to run in the background even though helm will report a failure. Allow this job to finish and then re-run the helm upgrade command. + +Helm with `--atomic`: In general, it is recommended to not use the `--atomic` setting for this particular upgrade since the system update job will be terminated before completion. If `--atomic` is preferred, then increase the timeout using the `--timeout` flag to account for the reindexing time (see note above for estimating this value). + +### Deprecations + +## 0.9.6 + +### Breaking Changes + +- #6742 The metadata file sink's output format no longer contains nested JSON strings for MCP aspects, but instead unpacks the stringified JSON into a real JSON object. The previous sink behavior can be recovered using the `legacy_nested_json_string` option. The file source is backwards compatible and supports both formats. +- #6901 The `env` and `database_alias` fields have been marked deprecated across all sources. We recommend using `platform_instance` where possible instead. + +### Potential Downtime + +### Deprecations + +- #6851 - Sources bigquery-legacy and bigquery-usage-legacy have been removed + +### Other notable Changes + +- If anyone faces issues with login please clear your cookies. Some security updates are part of this release. That may cause login issues until cookies are cleared. + +## 0.9.4 / 0.9.5 + +### Breaking Changes + +- #6243 apache-ranger authorizer is no longer the core part of DataHub GMS, and it is shifted as plugin. Please refer updated documentation [Configuring Authorization with Apache Ranger](./configuring-authorization-with-apache-ranger.md#configuring-your-datahub-deployment) for configuring `apache-ranger-plugin` in DataHub GMS. +- #6243 apache-ranger authorizer as plugin is not supported in DataHub Kubernetes deployment. +- #6243 Authentication and Authorization plugins configuration are removed from [application.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-service/configuration/src/main/resources/application.yaml). Refer documentation [Migration Of Plugins From application.yaml](../plugins.md#migration-of-plugins-from-applicationyml) for migrating any existing custom plugins. +- `datahub check graph-consistency` command has been removed. It was a beta API that we had considered but decided there are better solutions for this. So removing this. +- `graphql_url` option of `powerbi-report-server` source deprecated as the options is not used. +- #6789 BigQuery ingestion: If `enable_legacy_sharded_table_support` is set to False, sharded table names will be suffixed with \_yyyymmdd to make sure they don't clash with non-sharded tables. This means if stateful ingestion is enabled then old sharded tables will be recreated with a new id and attached tags/glossary terms/etc will need to be added again. _This behavior is not enabled by default yet, but will be enabled by default in a future release._ + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +- #6611 - Snowflake `schema_pattern` now accepts pattern for fully qualified schema name in format `.` by setting config `match_fully_qualified_names : True`. Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. The config option `match_fully_qualified_names` will be deprecated in future and the default behavior will assume `match_fully_qualified_names: True`." +- #6636 - Sources `snowflake-legacy` and `snowflake-usage-legacy` have been removed. + +## 0.9.3 + +### Breaking Changes + +- The beta `datahub check graph-consistency` command has been removed. + +### Potential Downtime + +### Deprecations + +- PowerBI source: `workspace_id_pattern` is introduced in place of `workspace_id`. `workspace_id` is now deprecated and set for removal in a future version. + +### Other notable Changes + +## 0.9.2 + +- LookML source will only emit views that are reachable from explores while scanning your git repo. Previous behavior can be achieved by setting `emit_reachable_views_only` to False. +- LookML source will always lowercase urns for lineage edges from views to upstream tables. There is no fallback provided to previous behavior because it was inconsistent in application of lower-casing earlier. +- dbt config `node_type_pattern` which was previously deprecated has been removed. Use `entities_enabled` instead to control whether to emit metadata for sources, models, seeds, tests, etc. +- The dbt source will always lowercase urns for lineage edges to the underlying data platform. +- The DataHub Airflow lineage backend and plugin no longer support Airflow 1.x. You can still run DataHub ingestion in Airflow 1.x using the [PythonVirtualenvOperator](https://airflow.apache.org/docs/apache-airflow/1.10.15/_api/airflow/operators/python_operator/index.html?highlight=pythonvirtualenvoperator#airflow.operators.python_operator.PythonVirtualenvOperator). + +### Breaking Changes + +- #6570 `snowflake` connector now populates created and last modified timestamps for snowflake datasets and containers. This version of snowflake connector will not work with **datahub-gms** version older than `v0.9.3` + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## 0.9.1 + +### Breaking Changes + +- We have promoted `bigquery-beta` to `bigquery`. If you are using `bigquery-beta` then change your recipes to use the type `bigquery`. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## 0.9.0 + +### Breaking Changes + +- Java version 11 or greater is required. +- For any of the GraphQL search queries, the input no longer supports value but instead now accepts a list of values. These values represent an OR relationship where the field value must match any of the values. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.45` + +### Breaking Changes + +- The `getNativeUserInviteToken` and `createNativeUserInviteToken` GraphQL endpoints have been renamed to + `getInviteToken` and `createInviteToken` respectively. Additionally, both now accept an optional `roleUrn` parameter. + Both endpoints also now require the `MANAGE_POLICIES` privilege to execute, rather than `MANAGE_USER_CREDENTIALS` + privilege. +- One of the default policies shipped with DataHub (`urn:li:dataHubPolicy:7`, or `All Users - All Platform Privileges`) + has been edited to no longer include `MANAGE_POLICIES`. Its name has consequently been changed to + `All Users - All Platform Privileges (EXCEPT MANAGE POLICIES)`. This change was made to prevent all users from + effectively acting as superusers by default. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.44` + +### Breaking Changes + +- Browse Paths have been upgraded to a new format to align more closely with the intention of the feature. + Learn more about the changes, including steps on upgrading, here: +- The dbt ingestion source's `disable_dbt_node_creation` and `load_schema` options have been removed. They were no longer necessary due to the recently added sibling entities functionality. +- The `snowflake` source now uses newer faster implementation (earlier `snowflake-beta`). Config properties `provision_role` and `check_role_grants` are not supported. Older `snowflake` and `snowflake-usage` are available as `snowflake-legacy` and `snowflake-usage-legacy` sources respectively. + +### Potential Downtime + +- [Helm] If you're using Helm, please ensure that your version of the `datahub-actions` container is bumped to `v0.0.7` or `head`. + This version contains changes to support running ingestion in debug mode. Previous versions are not compatible with this release. + Upgrading to helm chart version `0.2.103` will ensure that you have the compatible versions by default. + +### Deprecations + +### Other notable Changes + +## `v0.8.42` + +### Breaking Changes + +- Python 3.6 is no longer supported for metadata ingestion +- #5451 `GMS_HOST` and `GMS_PORT` environment variables deprecated in `v0.8.39` have been removed. Use `DATAHUB_GMS_HOST` and `DATAHUB_GMS_PORT` instead. +- #5478 DataHub CLI `delete` command when used with `--hard` option will delete soft-deleted entities which match the other filters given. +- #5471 Looker now populates `userEmail` in dashboard user usage stats. This version of looker connnector will not work with older version of **datahub-gms** if you have `extract_usage_history` looker config enabled. +- #5529 - `ANALYTICS_ENABLED` environment variable in **datahub-gms** is now deprecated. Use `DATAHUB_ANALYTICS_ENABLED` instead. +- #5485 `--include-removed` option was removed from delete CLI + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.41` + +### Breaking Changes + +- The `should_overwrite` flag in `csv-enricher` has been replaced with `write_semantics` to match the format used for other sources. See the [documentation](/docs/generated/ingestion/sources/csv-enricher/) for more details +- Closing an authorization hole in creating tags adding a Platform Privilege called `Create Tags` for creating tags. This is assigned to `datahub` root user, along + with default All Users policy. Notice: You may need to add this privilege (or `Manage Tags`) to existing users that need the ability to create tags on the platform. +- #5329 Below profiling config parameters are now supported in `BigQuery`: + + - profiling.profile_if_updated_since_days (default=1) + - profiling.profile_table_size_limit (default=1GB) + - profiling.profile_table_row_limit (default=50000) + + Set above parameters to `null` if you want older behaviour. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.40` + +### Breaking Changes + +- #5240 `lineage_client_project_id` in `bigquery` source is removed. Use `storage_project_id` instead. + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +## `v0.8.39` + +### Breaking Changes + +- Refactored the `health` field of the `Dataset` GraphQL Type to be of type **list of HealthStatus** (was type **HealthStatus**). See [this PR](https://github.com/datahub-project/datahub/pull/5222/files) for more details. + +### Potential Downtime + +### Deprecations + +- #4875 Lookml view file contents will no longer be populated in custom_properties, instead view definitions will be always available in the View Definitions tab. +- #5208 `GMS_HOST` and `GMS_PORT` environment variables being set in various containers are deprecated in favour of `DATAHUB_GMS_HOST` and `DATAHUB_GMS_PORT`. +- `KAFKA_TOPIC_NAME` environment variable in **datahub-mae-consumer** and **datahub-gms** is now deprecated. Use `METADATA_AUDIT_EVENT_NAME` instead. +- `KAFKA_MCE_TOPIC_NAME` environment variable in **datahub-mce-consumer** and **datahub-gms** is now deprecated. Use `METADATA_CHANGE_EVENT_NAME` instead. +- `KAFKA_FMCE_TOPIC_NAME` environment variable in **datahub-mce-consumer** and **datahub-gms** is now deprecated. Use `FAILED_METADATA_CHANGE_EVENT_NAME` instead. + +### Other notable Changes + +- #5132 Profile tables in `snowflake` source only if they have been updated since configured (default: `1`) number of day(s). Update the config `profiling.profile_if_updated_since_days` as per your profiling schedule or set it to `None` if you want older behaviour. + +## `v0.8.38` + +### Breaking Changes + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +- Create & Revoke Access Tokens via the UI +- Create and Manage new users via the UI +- Improvements to Business Glossary UI +- FIX - Do not require reindexing to migrate to using the UI business glossary + +## `v0.8.36` + +### Breaking Changes + +- In this release we introduce a brand new Business Glossary experience. With this new experience comes some new ways of indexing data in order to make viewing and traversing the different levels of your Glossary possible. Therefore, you will have to [restore your indices](/docs/how/restore-indices/) in order for the new Glossary experience to work for users that already have existing Glossaries. If this is your first time using DataHub Glossaries, you're all set! + +### Potential Downtime + +### Deprecations + +### Other notable Changes + +- #4961 Dropped profiling is not reported by default as that caused a lot of spurious logging in some cases. Set `profiling.report_dropped_profiles` to `True` if you want older behaviour. + +## `v0.8.35` + +### Breaking Changes + +### Potential Downtime + +### Deprecations + +- #4875 Lookml view file contents will no longer be populated in custom_properties, instead view definitions will be always available in the View Definitions tab. + +### Other notable Changes + +## `v0.8.34` + +### Breaking Changes + +- #4644 Remove `database` option from `snowflake` source which was deprecated since `v0.8.5` +- #4595 Rename confusing config `report_upstream_lineage` to `upstream_lineage_in_report` in `snowflake` connector which was added in `0.8.32` + +### Potential Downtime + +### Deprecations + +- #4644 `host_port` option of `snowflake` and `snowflake-usage` sources deprecated as the name was confusing. Use `account_id` option instead. + +### Other notable Changes + +- #4760 `check_role_grants` option was added in `snowflake` to disable checking roles in `snowflake` as some people were reporting long run times when checking roles. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/iceberg-catalog.md b/docs-archive/versioned_docs/version-1.5.0/docs/iceberg-catalog.md new file mode 100644 index 00000000..acb12617 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/iceberg-catalog.md @@ -0,0 +1,564 @@ +--- +title: DataHub Iceberg Catalog +sidebar_label: Iceberg Catalog +slug: /iceberg-catalog +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/iceberg-catalog.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHub Iceberg Catalog + + + +> Note that this feature is currently in open **Beta**. With any questions or issues, please reach out to your DataHub +> representative. + +> Open Source DataHub: 1.0.0 + +## Introduction + +DataHub Iceberg Catalog provides integration with Apache Iceberg, allowing you to manage Iceberg tables through DataHub. This tutorial walks through setting up and using the DataHub Iceberg Catalog to create, read, and manage Iceberg tables. The catalog provides a secure way to manage Iceberg tables through DataHub's permissions model while also enabling discovery use-cases for humans instantly. + +## Use Cases + +- Create and manage Iceberg tables through DataHub +- Maintain consistent metadata across DataHub and Iceberg +- Facilitate data discovery by exposing Iceberg table metadata in DataHub +- Enable secure access to Iceberg tables through DataHub's permissions model + +## Conceptual Mapping + +| Iceberg Concept | DataHub Concept | Notes | +| ---------------- | -------------------- | ------------------------------------------------- | +| Overall platform | dataPlatform | `iceberg` platform | +| Warehouse | dataPlatformInstance | Stores info such as storage credentials | +| Namespace | container | Hierarchical containers of subtype "Namespace" | +| Tables, Views | dataset | Dataset URN is UUID that persists across renames | +| Table/View Names | platformResource | Points to dataset, changes with rename operations | + +## Prerequisites + +Before starting, ensure you have: + +1. DataHub installed and running locally, and `datahub` cli is installed and setup to point to it. +2. AWS credentials and appropriate permissions configured. +3. The following environment variables set: + + ```bash + DH_ICEBERG_CLIENT_ID="your_client_id" + DH_ICEBERG_CLIENT_SECRET="your_client_secret" + DH_ICEBERG_AWS_ROLE="arn:aws:iam::123456789012:role/your-role-name" # Example format + DH_ICEBERG_DATA_ROOT="s3://your-bucket/path" + + ``` + + The `DH_ICEBERG_CLIENT_ID` is the `AWS_ACCESS_KEY_ID` and `DH_ICEBERG_CLIENT_SECRET` is the `AWS_SECRET_ACCESS_KEY` + +4. If using pyiceberg, configure pyiceberg to use your local datahub using one of its supported ways. For example, create `~/.pyiceberg.yaml` with + +```commandline +catalog: + local_datahub: + uri: http://localhost:8080/iceberg + warehouse: arctic_warehouse +``` + +Note: +The python code snippets in this tutorial are based on the code available in the `metadata-ingestion/examples/iceberg` folder of the DataHub repository. These snippets require `pyiceberg[duckdb] >=0.8.1` to be installed. +For the spark examples, the tested version of spark is 3.5.3_2.12 + +### Required AWS Permissions + +The AWS role must have read and write permissions for the S3 location specified in `DH_ICEBERG_DATA_ROOT`. + +Note: These permissions must be granted for the specific S3 bucket and path prefix where your Iceberg tables will be stored (as specified in `DH_ICEBERG_DATA_ROOT`). Additionally, the role must have a trust policy that allows it to be assumed using the AWS credentials provided in `DH_ICEBERG_CLIENT_ID` and `DH_ICEBERG_CLIENT_SECRET`. + +## Setup Instructions + +### 1. Provision a Warehouse + +Create an Iceberg warehouse in DataHub + + + + +``` +datahub iceberg create -w arctic_warehouse -d $DH_ICEBERG_DATA_ROOT -i $DH_ICEBERG_CLIENT_ID --client_secret $DH_ICEBERG_CLIENT_SECRET --region "us-east-1" --role $DH_ICEBERG_AWS_ROLE +``` + + + + +```python +# File: provision_warehouse.py +import os + +from constants import warehouse + +# Assert that env variables are present + +assert os.environ.get("DH_ICEBERG_CLIENT_ID"), ( + "DH_ICEBERG_CLIENT_ID variable is not present" +) +assert os.environ.get("DH_ICEBERG_CLIENT_SECRET"), ( + "DH_ICEBERG_CLIENT_SECRET variable is not present" +) +assert os.environ.get("DH_ICEBERG_AWS_ROLE"), ( + "DH_ICEBERG_AWS_ROLE variable is not present" +) +assert os.environ.get("DH_ICEBERG_DATA_ROOT"), ( + "DH_ICEBERG_DATA_ROOT variable is not present" +) + +assert os.environ.get("DH_ICEBERG_DATA_ROOT", "").startswith("s3://") + +os.system( + f"datahub iceberg create --warehouse {warehouse} --data_root $DH_ICEBERG_DATA_ROOT/{warehouse} --client_id $DH_ICEBERG_CLIENT_ID --client_secret $DH_ICEBERG_CLIENT_SECRET --region 'us-east-1' --role $DH_ICEBERG_AWS_ROLE" +) +``` + + + + +After provisioning the warehouse, ensure your DataHub user has the following privileges to the resource type Data Platform Instance, which were introduced with Iceberg support: + +- `DATA_MANAGE_VIEWS_PRIVILEGE` +- `DATA_MANAGE_TABLES_PRIVILEGE` +- `DATA_MANAGE_NAMESPACES_PRIVILEGE` +- `DATA_LIST_ENTITIES_PRIVILEGE` + +You can grant these privileges through the DataHub UI under the Policies section. + +### 2. Create a Table + +You can create Iceberg tables using PyIceberg with a defined schema. Here's an example creating a ski resort metrics table: + + + + +Connect to the DataHub Iceberg Catalog using Spark SQL by defining `$GMS_HOST`, `$GMS_PORT`, `$WAREHOUSE` to connect to and `$USER_PAT` - the DataHub Personal Access Token used to connect to the catalog. +When using DataHub Cloud, the Iceberg Catalog URL is `https://.acryl.io/gms/iceberg/` +If you're running DataHub locally, set `GMS_HOST` to `localhost` and `GMS_PORT` to `8080`. + +For this example, set `WAREHOUSE` to `arctic_warehouse` + +```cli + +spark-sql --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,org.apache.iceberg:iceberg-aws-bundle:1.6.1 \ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ + --conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog \ + --conf spark.sql.catalog.local=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.local.type=rest \ + --conf spark.sql.catalog.local.uri=http://$GMS_HOST:$GMS_PORT/iceberg/ \ + --conf spark.sql.catalog.local.warehouse=$WAREHOUSE \ + --conf spark.sql.catalog.local.token=$USER_PAT \ + --conf spark.sql.catalog.local.rest-metrics-reporting-enabled=false \ + --conf spark.sql.catalog.local.header.X-Iceberg-Access-Delegation=vended-credentials \ + --conf spark.sql.defaultCatalog=local + +``` + +Use the following SQL via spark-sql + +```sql +CREATE NAMESPACE alpine_db; +CREATE TABLE alpine_db.ski_resorts ( + resort_id BIGINT NOT NULL COMMENT 'Unique identifier for each ski resort', + resort_name STRING NOT NULL COMMENT 'Official name of the ski resort', + daily_snowfall BIGINT COMMENT 'Amount of new snow in inches during the last 24 hours', + conditions STRING COMMENT 'Current snow conditions description', + last_updated TIMESTAMP COMMENT 'Timestamp of when the snow report was last updated' +); +``` + + + + +```python +from constants import namespace, table_name, warehouse + +from pyiceberg.schema import Schema +from pyiceberg.types import LongType, NestedField, StringType, TimestampType +from pyiceberg.catalog import load_catalog +from datahub.ingestion.graph.client import get_default_graph + +# Get DataHub graph client for authentication +graph = get_default_graph() + +# Define schema with documentation +schema = Schema( + NestedField( + field_id=1, + name="resort_id", + field_type=LongType(), + required=True, + doc="Unique identifier for each ski resort" + ), + NestedField( + field_id=2, + name="resort_name", + field_type=StringType(), + required=True, + doc="Official name of the ski resort" + ), + NestedField( + field_id=3, + name="daily_snowfall", + field_type=LongType(), + required=False, + doc="Amount of new snow in inches during the last 24 hours" + ), + NestedField( + field_id=4, + name="conditions", + field_type=StringType(), + required=False, + doc="Current snow conditions description" + ), + NestedField( + field_id=5, + name="last_updated", + field_type=TimestampType(), + required=False, + doc="Timestamp of when the snow report was last updated" + ) +) + +# Load catalog and create table +catalog = load_catalog("local_datahub", warehouse=warehouse, token=graph.config.token) +catalog.create_namespace(namespace) +catalog.create_table(f"{namespace}.{table_name}", schema) +``` + + + + +### 3. Write Data + + + + +```sql +INSERT INTO alpine_db.ski_resorts (resort_id, resort_name, daily_snowfall, conditions, last_updated) +VALUES + (1, 'Snowpeak Resort', 12, 'Powder', CURRENT_TIMESTAMP()), + (2, 'Alpine Valley', 8, 'Packed', CURRENT_TIMESTAMP()), + (3, 'Glacier Heights', 15, 'Fresh Powder', CURRENT_TIMESTAMP()); +``` + + + + +You can write data to your Iceberg table using PyArrow. Note the importance of matching the schema exactly: + +```python +from constants import namespace, table_name, warehouse +from pyiceberg.catalog import load_catalog +from datahub.ingestion.graph.client import get_default_graph + +import pyarrow as pa +from datetime import datetime + +graph = get_default_graph() +catalog = load_catalog("local_datahub", warehouse=warehouse, token=graph.config.token) + +# Create PyArrow schema to match Iceberg schema +pa_schema = pa.schema([ + ("resort_id", pa.int64(), False), # False means not nullable + ("resort_name", pa.string(), False), + ("daily_snowfall", pa.int64(), True), + ("conditions", pa.string(), True), + ("last_updated", pa.timestamp("us"), True), +]) + +# Create sample data +sample_data = pa.Table.from_pydict( + { + "resort_id": [1, 2, 3], + "resort_name": ["Snowpeak Resort", "Alpine Valley", "Glacier Heights"], + "daily_snowfall": [12, 8, 15], + "conditions": ["Powder", "Packed", "Fresh Powder"], + "last_updated": [ + datetime.now(), + datetime.now(), + datetime.now() + ] + }, + schema=pa_schema +) + +# Write to table +table = catalog.load_table(f"{namespace}.{table_name}") +table.overwrite(sample_data) + +# Refresh table to see changes +table.refresh() +``` + + + + +### 4. Read Data + + + + +```sql +SELECT * from alpine_db.resort_metrics; +``` + + + + +Reading data from an Iceberg table using DuckDB integration: + +```python +from pyiceberg.catalog import load_catalog +from constants import namespace, table_name, warehouse + +from datahub.ingestion.graph.client import get_default_graph + +# Get DataHub graph client for authentication +graph = get_default_graph() + +catalog = load_catalog("local_datahub", warehouse=warehouse, token=graph.config.token) +table = catalog.load_table(f"{namespace}.{table_name}") +con = table.scan().to_duckdb(table_name=table_name) + +# Query the data +print("\nResort Metrics Data:") +print("-" * 50) +for row in con.execute(f"SELECT * FROM {table_name}").fetchall(): + print(row) +``` + + + + + +## Reference Information + +### Trust Policy Example + +When setting up your AWS role, you'll need to configure a trust policy. Here's an example: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::123456789012:user/iceberg-user" // The IAM user or role associated with your credentials + }, + "Action": "sts:AssumeRole" + } + ] +} +``` + +### Vended Credentials and Session Expiry + +When creating a warehouse, the session expiry duration for the vended credentials can be configured via the +`datahub iceberg` cli by passing the `--duration_seconds` parameter. If not specified, this currently defaults to 3600 +seconds. This needs to be a minimum of 900 seconds (15 minutes). + +With AWS, at the time of creating the role for use with the catalog, a `MaxSessionDuration` can also be set. AWS currently +supports values between 1 hour to 12 hours. This configuration will set the upper limit for what can be configured for the +warehouse via `--duration_seconds` flag. + +See `datahub iceberg create --help` for the flags and default if not specified. This can also be updated using +`datahub iceberg update` + +### Namespace and Configuration Requirements + +1. **Namespace Creation**: A namespace must be created before any tables can be created within it. + +2. **Spark SQL Configuration**: To simplify SQL statements by avoiding the need to prefix catalog name and namespace, you can set defaults in your Spark configuration: + ``` + --conf spark.sql.defaultCatalog= \ + --conf spark.sql.catalog.local.default-namespace= + ``` + +### Public access of Iceberg Tables + +It is possible to enable public read-only access of specific Iceberg tables if needed. +Enabling public access requires the following steps. + +1. Ensure that the DATA ROOT folder in s3 has public read access policy set to enable the files with that prefix to be read without AWS credentials. +2. Update the GMS Configuration to enable public access, ensure the GMS service is run with the following environment variables set. + + - Set the env var `ENABLE_PUBLIC_READ` to `true` to enable the capability. If unset, this is by default `false`. + - Set the env var `PUBLICLY_READABLE_TAG` to a specific Tag name that indicates public access when applied to an Iceberg DataSet. If unset, this defaults to the tag `PUBLICLY_READABLE` + + Alternatively, these can be set in metadata-service/configuration/src/main/resources/application.yaml under `icebergCatalog` key. The defaults are populated under that key. + +3. Once GMS is started with enabling the public read capability, apply the Tag defined for public access on each Dataset that should be accessible without authentication. + +To access these tables that have public access, start the spark-sql with the following settings + +```commandline +spark-sql --packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,org.apache.iceberg:iceberg-aws-bundle:1.6.1\ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ + --conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog \ + --conf spark.sql.catalog.local=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.local.type=rest \ + --conf spark.sql.catalog.local.uri=http://${GMS_HOST}:${GMS_PORT}/public-iceberg/ \ + --conf spark.sql.catalog.local.warehouse=arctic_warehouse \ + --conf spark.sql.catalog.local.header.X-Iceberg-Access-Delegation=false \ + --conf spark.sql.catalog.local.rest-metrics-reporting-enabled=false \ + --conf spark.sql.catalog.local.client.region=us-east-1 \ + --conf spark.sql.catalog.local.client.credentials-provider=software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider \ + --conf spark.sql.defaultCatalog=local \ + --conf spark.sql.catalog.local.default-namespace=alpine_db +``` + +Note the specific differences from the authenticated access: + +- The REST Catalog URI is gms_host:port/public-iceberg/ +- The DATA ROOT AWS s3 region is specified via `spark.sql.catalog.local.client.region` +- The `X-Iceberg-Access-Delegation` header is set to `false` instead of `vended-credentials` +- The `credentials-provider` is set to `credentials-provider=software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider` and no Personal Access Token is provided. + +In such unauthenticated sessions, attempts to access tables that do not have access will fail with a NoSuchTableException error instead of an authorization failure. + +### DataHub Iceberg CLI + +The DataHub CLI provides several commands for managing Iceberg warehouses: + +1. List Warehouses: + + ``` + datahub iceberg list + ``` + +2. Update Warehouse Configuration: + + ``` + datahub iceberg update \ + -w $WAREHOUSE_NAME \ + -d $DH_ICEBERG_DATA_ROOT \ + -i $DH_ICEBERG_CLIENT_ID \ + --client_secret $DH_ICEBERG_CLIENT_ID \ + --region $DH_ICEBERG_REGION \ + --role DH_ICEBERG_AWS_ROLE + ``` + +3. Delete Warehouse: + ``` + datahub iceberg delete -w $WAREHOUSE_NAME + ``` + Note: This command deletes Containers associated with Namespaces and Datasets associated with Tables and Views in this warehouse. However, it does not delete any backing data in the warehouse data_root location - it only deletes entries from the catalog. + +For additional options and help, run: + +``` +datahub iceberg [command] --help +``` + +### Migrating from Other Iceberg Catalogs + +When migrating from another Iceberg catalog, you can register existing Iceberg tables using the `system.register_table` command. This allows you to start managing existing Iceberg tables through DataHub without moving the underlying data. + +Example of registering an existing table: + +``` +call system.register_table('myTable', 's3://my-s3-bucket/my-data-root/myNamespace/myTable/metadata/00000-f9dbba67-df4f-4742-9ba5-123aa2bb4076.metadata.json'); + +-- Read from newly registered table +select * from myTable; +``` + +### Security and Permissions + +DataHub provides granular access control for Iceberg operations through policies. The following privileges were introduced with Iceberg support: + +| Operation | Required Privilege | Resource Types | +| -------------------------------------- | --------------------------------- | ------------------------------- | +| CREATE or DROP namespaces | Manage Namespaces | Data Platform Instance | +| CREATE, ALTER or DROP tables | Manage Tables | Data Platform Instance | +| CREATE, ALTER or DROP views | Manage Views | Data Platform Instance | +| SELECT from tables or views | Read Only data-access | Dataset, Data Platform Instance | +| INSERT, UPDATE, DELETE or ALTER tables | Read-write data-access | Dataset, Data Platform Instance | +| List tables or views | List tables, views and namespaces | Data Platform Instance | + +To configure access: + +1. Create policies with the appropriate privileges for Iceberg users +

+ +

+ +2. Scope the policies by: + - Selecting specific warehouse Data Platform Instances for namespace, table/view management, and listing privileges + - Selecting specific DataSets and/or Data Platform Instances for table and view data access privileges + - Alternatively, resources can be filtered by + - Selecting specific namespace Containers for namespace, table/view management, listing and data access privileges: when a container is selected, all datasets in that container and its descendant containers (in the container hierarchy) are included in the scope. + - Selecting Tags / Domains that may be applied to DataSets or Data Platform Instances + +

+ +

+ +3. Assign the policies to relevant users or groups +

+ +

+ +### Iceberg tables in DataHub + +Once you create tables in iceberg, each of those tables show up in DataHub as a DataSet + +

+ +

+ +Some of the standard metadata fields are populated in the Dataset Properties. Additionally, any Iceberg `TBLPROPERTIES` set via DDL on tables or views are also shown in DatasetProperties with a prefix `TBLPROPERTIES:` +The iceberg 'TBLPROPERTIES' can only be set via DDL and is a one way sync. Changes to iceberg `TBLPROPERTIES` properties via datahub APIs do not get written to the Iceberg table or view properties. + +## Troubleshooting + +### Q: How do I verify my warehouse was created successfully? + +A: Check the DataHub UI under Data Platform Instances, or try creating and reading a table using the provided scripts. + +### Q: What permissions are needed for S3 access? + +A: The role specified in `DH_ICEBERG_AWS_ROLE` should have permissions to read and write to the S3 bucket specified in `DH_ICEBERG_DATA_ROOT`. + +### Q: How does my compute engine authenticate with DataHub? + +A: A DataHub Personal Access Token can be used via bearer token to authenticate with DataHub. + +### Q: What should I do if table creation fails? + +A: Check that: + +1. All required environment variables are set correctly +2. Your AWS role has the necessary permissions +3. The namespace exists (create it if needed) +4. The table name doesn't already exist + +## Known Limitations + +- AWS S3 only support +- Multi-table transactional `/commit` is not yet supported +- Table operation purge is not yet implemented +- Metric collection and credential refresh mechanisms are still in development + +## Future Enhancements + +- Support for Iceberg multi-table transactional `/commit` +- Additional table APIs (scan, drop table purge) +- Azure and GCP Support +- Enhanced metrics +- Credential refresh +- Proxy to another REST Catalog to use its capabilities while DataHub has real-time metadata updates. + +## Related Documentation + +- [Apache Iceberg Documentation](https://iceberg.apache.org/) +- [PyIceberg Documentation](https://py.iceberg.apache.org/) +- [DataHub Documentation](/docs/) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/incidents/incidents.md b/docs-archive/versioned_docs/version-1.5.0/docs/incidents/incidents.md new file mode 100644 index 00000000..090fe432 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/incidents/incidents.md @@ -0,0 +1,434 @@ +--- +description: This page provides an overview of working with the DataHub Incidents API. +title: Incidents +slug: /incidents/incidents +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/incidents/incidents.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Incidents + + + +## Introduction + +**Incidents** are a concept used to flag particular Data Assets as being in an unhealthy state. Each incident has an independent lifecycle and details including a state (active, resolved), a title, a description, & more. + +A couple scenarios in which incidents can be useful are + +1. **Communicating Assets with Ongoing Issues**: You can mark a known-bad data asset as under an ongoing incident so consumers and stakeholders can be informed about the health status of a data asset via the DataHub UI. Moreover, they can follow the incident as it progresses toward resolution. +2. **Pipeline Circuit Breaking (advanced):** You can use Incidents as a basis for orchestrating and blocking data pipelines that have inputs with active issues to avoid propagating bad data downstream. + +In the next section, we'll walk through how to + +1. Create a new incident +2. Fetch all incidents for a data asset +3. Resolve an incident + +for **Datasets**, **Dashboards**, **Charts**, **Data Pipelines** (Data Flows), and **Data Tasks** (Data Jobs) using the DataHub UI or [GraphQL API](docs/api/graphql/overview.md). + +Let's get started! + +## Creating an Incident + +To create an incident, simply navigate to the profile page for the asset of interest, click +the 3-dot menu icon on the right side of the header, and click **Raise Incident**. + +Choose an existing type, or define your own, and then author a title and description of the issue. Finally, +click `Add` to create the new issue. This will mark the asset with a health status badge indicating that it +is possibly unfit for use due to an ongoing issue. + +## Resolving an Incident + +To resolve an incident, simply naviagte to the profile page for the asset of interest, click +the **Incidents** tab, and then click the **Resolve** button for the incident of interest. +This will resolve the incident from the list of active incidents for the asset, removing it from the +asset's health status. + +## Finding Assets with Active Incidents + +To view all assets with active incidents, simply apply the `Has Active Incidents` filter on the search results page of DataHub. +To view all assets first, click **Explore all** on the DataHub homepage. + +## Creating an Incident via API + +Oftentimes it is desirable to raise and resolve incidents for particular data assets in automated fashion using the DataHub API, e.g. as part of an +orchestration pipeline. + +To create (i.e. raise) a new incident for a data asset, simply create a GraphQL request using the `raiseIncident` mutation. + +``` +type Mutation { + """ + Raise a new incident for a data asset + """ + raiseIncident(input: RaiseIncidentInput!): String! # Returns new Incident URN. +} + +input RaiseIncidentInput { + """ + The type of incident, e.g. OPERATIONAL + """ + type: IncidentType! + + """ + A custom type of incident. Present only if type is 'CUSTOM' + """ + customType: String + + """ + An optional title associated with the incident + """ + title: String + + """ + An optional description associated with the incident + """ + description: String + + """ + The resource (dataset, dashboard, chart, dataFlow, etc) that the incident is associated with. + """ + resourceUrn: String! + + """ + The source of the incident, i.e. how it was generated + """ + source: IncidentSourceInput +} +``` + +### Examples + +First, we'll create a demo GraphQL query, then show how to represent it via CURL & Python. + +Imagine we want to raise a new incident on a Dataset with URN `urn:li:dataset:(abc)` because it's failed automated quality checks. To do so, we could make the following GraphQL query: + +_Request_ + +``` +mutation raiseIncident { + raiseIncident(input: { + type: OPERATIONAL + title: "Dataset Failed Quality Checks" + description: "Dataset failed 2/6 Quality Checks for suite run id xy123mksj812pk23." + resourceUrn: "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)" + }) +} +``` + +After we make this query, we will get back a unique URN for the incident. + +_Response_ + +``` +{ + "data": { + "raiseIncident": "urn:li:incident:bfecab62-dc10-49a6-a305-78ce0cc6e5b1" + } +} +``` + +Now we'll see how to issue this query using a CURL or Python. + +#### CURL + +To issue the above GraphQL as a CURL: + +``` +curl --location --request POST 'https://your-account.acryl.io/api/graphql' \ +--header 'Authorization: Bearer your-access-token' \ +--header 'Content-Type: application/json' \ +--data-raw '{"query":"mutation raiseIncident {\n raiseIncident(input: {\n type: OPERATIONAL\n title: \"Dataset Failed Quality Checks\"\n description: \"Dataset failed 2/6 Quality Checks for suite run id xy123mksj812pk23.\"\n resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)\"\n })\n}","variables":{}}' +``` + +#### Python + +To issue the above GraphQL query in Python (requests): + +``` +import requests + +datahub_session = requests.Session() + +headers = { + "Content-Type": "application/json", + "Authorization": "Bearer your-personal-access-token", +} + +json = { + "query": """mutation raiseIncident {\n + raiseIncident(input: {\n + type: OPERATIONAL\n + resourceUrn: \"urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)\"\n + })}""", + "variables": {}, +} + +response = datahub_session.post(f"https://your-account.acryl.io/api/graphql", headers=headers, json=json) +response.raise_for_status() +res_data = response.json() # Get result as JSON +``` + +## Retrieving Active Incidents + +To fetch the the ongoing incidents for a data asset, we can use the `incidents` GraphQL field on the entity of interest. + +To retrieve all incidents for a Dataset with a particular [URN](docs/what/urn.md), you can reference the 'incidents' field of the Dataset type: + +``` +type Dataset { + .... + """ + Incidents associated with the Dataset + """ + incidents( + """ + Optional incident state to filter by, defaults to any state. + """ + state: IncidentState, + """ + Optional start offset, defaults to 0. + """ + start: Int, + """ + Optional start offset, defaults to 20. + """ + count: Int): EntityIncidentsResult # Returns a list of incidents. +} +``` + +### Examples + +Now that we've raised an incident on it, imagine we want to fetch the first 10 "active" incidents for the Dataset with URN `urn:li:dataset:(abc`). To do so, we could issue the following request: + +_Request_ + +``` +query dataset { + dataset(urn: "urn:li:dataset:(abc)") { + incidents(state: ACTIVE, start: 0, count: 10) { + total + incidents { + urn + title + description + status { + state + } + } + } + } +} +``` + +After we make this query, we will get back a unique URN for the incident. + +_Response_ + +``` +{ + "data": { + "dataset": { + "incidents": { + "total": 1, + "incidents": [ + { + "urn": "urn:li:incident:bfecab62-dc10-49a6-a305-78ce0cc6e5b1", + "title": "Dataset Failed Quality Check", + "description": "Dataset failed 2/6 Quality Checks for suite run id xy123mksj812pk23.", + "status": { + "state": "ACTIVE" + } + } + ] + } + } + } +} +``` + +Now we'll see how to issue this query using a CURL or Python. + +#### CURL + +To issue the above GraphQL as a CURL: + +``` +curl --location --request POST 'https://your-account.acryl.io/api/graphql' \ +--header 'Authorization: Bearer your-access-token' \ +--header 'Content-Type: application/json' \ +--data-raw '{"query":"query dataset {\n dataset(urn: "urn:li:dataset:(abc)") {\n incidents(state: ACTIVE, start: 0, count: 10) {\n total\n incidents {\n urn\n title\n description\n status {\n state\n }\n }\n }\n }\n}","variables":{}}'Python +``` + +To issue the above GraphQL query in Python (requests): + +``` +import requests + +datahub_session = requests.Session() + +headers = { + "Content-Type": "application/json", + "Authorization": "Bearer your-personal-access-token", +} + +json = { + "query": """query dataset {\n + dataset(urn: "urn:li:dataset:(abc)") {\n + incidents(state: ACTIVE, start: 0, count: 10) {\n + total\n + incidents {\n + urn\n + title\n + description\n + status {\n + state\n + }\n + }\n + }\n + }\n + }""", + "variables": {}, +} + +response = datahub_session.post(f"https://your-account.acryl.io/api/graphql", headers=headers, json=json) +response.raise_for_status() +res_data = response.json() # Get result as JSON +``` + +## Resolving an Incident via API + +To resolve an incident for a data asset, simply create a GraphQL request using the `updateIncidentStatus` mutation. To mark an incident as resolved, simply update its state to `RESOLVED`. + +``` +type Mutation { + """ + Update an existing incident for a resource (asset) + """ + updateIncidentStatus( + """ + The urn for an existing incident + """ + urn: String! + + """ + Input required to update the state of an existing incident + """ + input: UpdateIncidentStatusInput!): String +} + +""" +Input required to update status of an existing incident +""" +input UpdateIncidentStatusInput { + """ + The new state of the incident + """ + state: IncidentState! + + """ + An optional message associated with the new state + """ + message: String +} +``` + +### Examples + +Imagine that we've fixed our Dataset with urn `urn:li:dataset:(abc)` so that it's passing validation. Now we want to mark the Dataset as healthy, so stakeholders and downstream consumers know it's ready to use. + +To do so, we need the URN of the Incident that we raised previously. + +_Request_ + +``` +mutation updateIncidentStatus { + updateIncidentStatus(urn: "urn:li:incident:bfecab62-dc10-49a6-a305-78ce0cc6e5b1", + input: { + state: RESOLVED + message: "Dataset is now passing validations. Verified by John Joyce on Data Platform eng." + }) +} +``` + +_Response_ + +``` +{ + "data": { + "updateIncidentStatus": "true" + } +} +``` + +True is returned if the incident's was successfully marked as resolved. + +#### CURL + +To issue the above GraphQL as a CURL: + +``` +curl --location --request POST 'https://your-account.acryl.io/api/graphql' \ +--header 'Authorization: Bearer your-access-token' \ +--header 'Content-Type: application/json' \ +--data-raw '{"query":"mutation updateIncidentStatus {\n updateIncidentStatus(urn: "urn:li:incident:bfecab62-dc10-49a6-a305-78ce0cc6e5b1", \n input: {\n state: RESOLVED\n message: "Dataset is now passing validations. Verified by John Joyce on Data Platform eng."\n })\n}","variables":{}}'Python +``` + +To issue the above GraphQL query in Python (requests): + +``` +import requests + +datahub_session = requests.Session() + +headers = { + "Content-Type": "application/json", + "Authorization": "Bearer your-personal-access-token", +} + +json = { + "query": """mutation updateIncidentStatus {\n + updateIncidentStatus(urn: \"urn:li:incident:bfecab62-dc10-49a6-a305-78ce0cc6e5b1\",\n + input: {\n + state: RESOLVED\n + message: \"Dataset is now passing validations. Verified by John Joyce on Data Platform eng.\"\n + })\n + }""", + "variables": {}, +} + +response = datahub_session.post(f"https://your-account.acryl.io/api/graphql", headers=headers, json=json) +response.raise_for_status() +res_data = response.json() # Get result as JSON +``` + +## Tips + +:::info +**Authorization** + +Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows: + +``` +Authorization: Bearer +``` + +**Exploring GraphQL API** + +Also, remember that you can play with an interactive version of the GraphQL API at `https://your-account-id.acryl.io/api/graphiql` +::: + +## Enabling Slack Notifications (DataHub Cloud Only) + +In DataHub Cloud, you can configure your to send Slack notifications to a specific channel when incidents are raised or their status is changed. + +These notifications are also able to tag the immediate asset's owners, along with the owners of downstream assets consuming it. + +

+ +

+ +To do so, simply follow the [Slack Integration Guide](docs/managed-datahub/slack/saas-slack-setup.md) and contact your DataHub Cloud customer success team to enable the feature! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/lineage/airflow.md b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/airflow.md new file mode 100644 index 00000000..8d363da0 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/airflow.md @@ -0,0 +1,547 @@ +--- +title: Airflow Integration +slug: /lineage/airflow +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/lineage/airflow.md' +--- +# Airflow Integration + +:::note + +If you're looking to schedule DataHub ingestion using Airflow, see the guide on [scheduling ingestion with Airflow](../../metadata-ingestion/schedule_docs/airflow.md). + +::: + +The DataHub Airflow plugin supports: + +- Automatic column-level lineage extraction from various operators e.g. SQL operators (including `MySqlOperator`, `PostgresOperator`, `SnowflakeOperator`, `BigQueryInsertJobOperator`, and more), `S3FileTransformOperator`, and more. +- Airflow DAG and tasks, including properties, ownership, and tags. +- Task run information, including task successes and failures. +- Manual lineage annotations using `inlets` and `outlets` on Airflow operators. + +The plugin requires Airflow 2.7+ and Python 3.10+. If you're using Airflow older than 2.7, it's possible to use the plugin with older versions of `acryl-datahub-airflow-plugin`. See the [compatibility section](#compatibility) for more details. + + + + +## DataHub Plugin Setup + +### Installation + +The plugin requires Airflow 2.7+ and Python 3.10+. If you don't meet these requirements, see the [compatibility section](#compatibility) for other options. + +```shell +pip install 'acryl-datahub-airflow-plugin>=1.1.0.4' +``` + +### Configuration + +Set up a DataHub connection in Airflow, either via command line or the Airflow UI. + +#### Command Line + +```shell +airflow connections add --conn-type 'datahub-rest' 'datahub_rest_default' --conn-host 'http://datahub-gms:8080' --conn-password '' +``` + +If you are using DataHub Cloud then please use `https://YOUR_PREFIX.acryl.io/gms` as the `--conn-host` parameter. + +#### Airflow UI + +On the Airflow UI, go to Admin -> Connections and click the "+" symbol to create a new connection. Select "DataHub REST Server" from the dropdown for "Connection Type" and enter the appropriate values. + +

+ +

+ +#### Optional Configurations + +No additional configuration is required to use the plugin. However, there are some optional configuration parameters that can be set in the `airflow.cfg` file. + +```ini title="airflow.cfg" +[datahub] +# Optional - additional config here. +enabled = True # default +``` + +| Name | Default value | Description | +| ---------------------------------- | -------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| enabled | true | If the plugin should be enabled. | +| conn_id | datahub_rest_default | The name of the datahub rest connection. | +| cluster | prod | name of the airflow cluster, this is equivalent to the `env` of the instance | +| platform_instance | None | The instance of the platform that all assets produced by this plugin belong to. It is optional. | +| capture_ownership_info | true | Extract DAG ownership. | +| capture_ownership_as_group | false | When extracting DAG ownership, treat DAG owner as a group rather than a user | +| capture_tags_info | true | Extract DAG tags. | +| capture_executions | true | Extract task runs and success/failure statuses. This will show up in DataHub "Runs" tab. | +| materialize_iolets | true | Create or un-soft-delete all entities referenced in lineage. | +| enable_extractors | true | Enable automatic lineage extraction. | +| disable_openlineage_plugin | true | Disable the OpenLineage plugin to avoid duplicative processing. | +| enable_multi_statement_sql_parsing | false | Parse multiple SQL statements within a single task. Resolves temp tables and merges lineage across statements in one execution. | +| log_level | _no change_ | [debug] Set the log level for the plugin. | +| debug_emitter | false | [debug] If true, the plugin will log the emitted events. | +| dag_filter_str | { "allow": [".*"] } | AllowDenyPattern value in form of JSON string to filter the DAGs from running. | +| enable_datajob_lineage | true | If true, the plugin will emit input/output lineage for DataJobs. | +| capture_airflow_assets | true | Capture native Airflow Assets/Datasets as DataHub lineage. See [Native Airflow Assets/Datasets](#native-airflow-assetsdatasets). | + +## Automatic lineage extraction + +To automatically extract lineage information, the plugin builds on top of Airflow's built-in [OpenLineage support](https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/supported_classes.html). +As such, we support a superset of the default operators that Airflow/OpenLineage supports. + +The SQL-related extractors have been updated to use [DataHub's SQL lineage parser](./sql_parsing.md), which is more robust than the built-in one and uses DataHub's metadata information to generate column-level lineage. + +Supported operators: + +- `SQLExecuteQueryOperator`, including any subclasses. Note that in newer versions of Airflow (generally Airflow 2.5+), most SQL operators inherit from this class. +- `AthenaOperator` and `AWSAthenaOperator` +- `BigQueryOperator` and `BigQueryExecuteQueryOperator` +- `BigQueryInsertJobOperator` (incubating) +- `MySqlOperator` +- `PostgresOperator` +- `RedshiftSQLOperator` +- `SnowflakeOperator` and `SnowflakeOperatorAsync` +- `SqliteOperator` +- `TeradataOperator` (_Note: Teradata uses two-tier `database.table` naming without a schema level_) +- `TrinoOperator` + + + +### Multi-Statement SQL Parsing + +When a task executes multiple SQL statements (e.g., `CREATE TEMP TABLE ...; INSERT ... FROM temp_table;`), enable this to parse all statements together and resolve temporary table dependencies. By default (False), only the first statement is parsed. + +```ini title="airflow.cfg" +[datahub] +enable_multi_statement_sql_parsing = True # Default: False +``` + +**Note:** Use a list of SQL strings (recommended) or semicolon-separated statements in a single string: + +## Manual Lineage Annotation + +### Using `inlets` and `outlets` + +You can manually annotate lineage by setting `inlets` and `outlets` on your Airflow operators. This is useful if you're using an operator that doesn't support automatic lineage extraction, or if you want to override the automatic lineage extraction. + +We have a few code samples that demonstrate how to use `inlets` and `outlets`: + +- [`lineage_backend_demo.py`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_demo.py) +- [`lineage_backend_taskflow_demo.py`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_backend_taskflow_demo.py) - uses the [TaskFlow API](https://airflow.apache.org/docs/apache-airflow/stable/concepts/taskflow.html) + +For more information, take a look at the [Airflow lineage docs](https://airflow.apache.org/docs/apache-airflow/stable/lineage.html). + +### Native Airflow Assets/Datasets + +Starting with Airflow 2.4+, you can use native Airflow [Datasets](https://airflow.apache.org/docs/apache-airflow/stable/authoring-and-scheduling/datasets.html) (renamed to [Assets](https://airflow.apache.org/docs/apache-airflow/3.0.0/authoring-and-scheduling/assets.html) in Airflow 3.x) for data-aware scheduling. The DataHub plugin automatically captures these as lineage when used in `inlets` and `outlets`. + +```python +from airflow.sdk.definitions.asset import Asset # Airflow 3.x +# or: from airflow.datasets import Dataset as Asset # Airflow 2.4+ + +s3_input = Asset("s3://my-bucket/input/data.parquet") +bigquery_output = Asset("bigquery://my-project/dataset/result_table") + +task = BashOperator( + task_id="process_data", + bash_command="echo 'Processing'", + inlets=[s3_input], + outlets=[bigquery_output], +) +``` + +The plugin maps URI schemes to DataHub platforms: + +| URI Scheme | DataHub Platform | +| --------------------- | ---------------- | +| `s3://`, `s3a://` | s3 | +| `gs://`, `gcs://` | gcs | +| `postgresql://` | postgres | +| `mysql://` | mysql | +| `bigquery://` | bigquery | +| `snowflake://` | snowflake | +| `file://` | file | +| `hdfs://` | hdfs | +| `abfs://`, `abfss://` | adls | + +Plain name assets (e.g., from the `@asset` decorator) default to the `airflow` platform. + +#### Configuration + +```ini title="airflow.cfg" +[datahub] +# Set to false to disable capturing Airflow Assets as lineage (default: true) +capture_airflow_assets = true +``` + +#### Limitations + +Native Airflow Assets have the following limitations compared to using DataHub's `Dataset` or `Urn` entities directly: + +1. **No `platform_instance` support**: The URN generated from an Airflow Asset URI cannot include a platform instance. The plugin only extracts the platform, dataset name, and environment from the URI. + +2. **Environment uses global plugin config**: All native Airflow Assets use the `cluster` setting from the plugin configuration as their environment. You cannot specify a different environment per asset. + +If you need `platform_instance` or per-asset environment control, use the DataHub entity classes instead: + +```python +from datahub_airflow_plugin.entities import Dataset + +# Full control over URN components +s3_input = Dataset( + platform="s3", + name="my-bucket/input/data.parquet", + env="PROD", + platform_instance="us-west-2" # Specify platform instance +) + +task = BashOperator( + task_id="process_data", + bash_command="echo 'Processing'", + inlets=[s3_input], +) +``` + +### Custom Operators + +If you have created a [custom Airflow operator](https://airflow.apache.org/docs/apache-airflow/stable/howto/custom-operator.html) that inherits from the BaseOperator class, +when overriding the `execute` function, set inlets and outlets via `context['ti'].task.inlets` and `context['ti'].task.outlets`. +The DataHub Airflow plugin will then pick up those inlets and outlets after the task runs. + +You can only set table-level lineage using inlets and outlets. For column-level lineage, you need to write a custom extractor for your custom operator. + +```python +class DbtOperator(BaseOperator): + ... + + def execute(self, context): + # do something + inlets, outlets = self._get_lineage() + # inlets/outlets are lists of either datahub_airflow_plugin.entities.Dataset or datahub_airflow_plugin.entities.Urn + context['ti'].task.inlets = self.inlets + context['ti'].task.outlets = self.outlets + + def _get_lineage(self): + # Do some processing to get inlets/outlets + + return inlets, outlets +``` + +If you override the `pre_execute` and `post_execute` function, ensure they include the `@prepare_lineage` and `@apply_lineage` decorators respectively. Reference the [Airflow docs](https://airflow.apache.org/docs/apache-airflow/stable/administration-and-deployment/lineage.html#lineage) for more details. + +See example implementation of a custom operator using SQL parser to capture table level lineage [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/airflow3/custom_operator_sql_parsing.py) + +### Custom SQL Operators with Automatic Lineage + +If you're building a custom SQL operator, you have two approaches depending on your needs: + +#### Option 1: Inherit from SQLExecuteQueryOperator (Recommended) + +**This is the easiest approach** - inherit from Airflow's `SQLExecuteQueryOperator` and you automatically get: + +- ✅ OpenLineage support built-in +- ✅ DataHub's enhanced SQL parser (via our SQLParser patch) +- ✅ Column-level lineage extraction +- ✅ No extra code needed + +```python +from typing import Any +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator + +class MyCustomSQLOperator(SQLExecuteQueryOperator): + """ + Custom SQL operator that inherits OpenLineage support. + + DataHub automatically enhances the SQL parsing with column-level lineage! + """ + + def __init__(self, my_custom_param: str, **kwargs): + # Add your custom parameters + self.my_custom_param = my_custom_param + super().__init__(**kwargs) + + def execute(self, context: Any) -> Any: + # Add any custom logic before SQL execution + self.log.info(f"Custom param: {self.my_custom_param}") + + # Parent class handles SQL execution + OpenLineage lineage + return super().execute(context) +``` + +**How it works:** + +1. `SQLExecuteQueryOperator` already has `get_openlineage_facets_on_complete()` implemented +2. It uses the hook's OpenLineage methods, which internally call `SQLParser` +3. DataHub patches `SQLParser` globally, so all SQL parsing gets enhanced automatically +4. You get column-level lineage without writing any lineage-specific code! + +**When to use this:** + +- Building operators for standard SQL databases (Postgres, MySQL, Snowflake, BigQuery, etc.) +- Want the simplest integration +- Don't need custom lineage extraction logic + +#### Option 2: Implement OpenLineage Interface from Scratch + +Only needed if you're building a completely custom operator that doesn't fit the `SQLExecuteQueryOperator` pattern: + +```python +from typing import Any, Optional +from airflow.models.baseoperator import BaseOperator + +class MyCompletelyCustomOperator(BaseOperator): + """ + For special cases where SQLExecuteQueryOperator doesn't fit. + """ + + def execute(self, context: Any) -> Any: + # Your custom SQL execution logic + pass + + def get_openlineage_facets_on_complete( + self, task_instance: Any + ) -> Optional["OperatorLineage"]: + """ + Implement OpenLineage interface manually. + DataHub's SQLParser patch still enhances this automatically! + """ + from airflow.providers.openlineage.sqlparser import SQLParser + + hook = self.get_db_hook() + parser = SQLParser( + dialect=hook.get_openlineage_database_dialect(hook.get_connection(self.conn_id)), + default_schema=hook.get_openlineage_default_schema(), + ) + + # This uses DataHub's patched SQLParser - column lineage included! + return parser.generate_openlineage_metadata_from_sql( + sql=self.sql, + hook=hook, + database_info=hook.get_openlineage_database_info(hook.get_connection(self.conn_id)), + ) +``` + +**When to use this:** + +- Building a very specialized operator from scratch +- Need complete control over lineage extraction +- The standard SQLExecuteQueryOperator pattern doesn't apply + +**Key Point:** DataHub patches `SQLParser.generate_openlineage_metadata_from_sql()` globally at import time, so **any operator** using OpenLineage's SQLParser automatically gets DataHub's enhanced parsing with column-level lineage! + +### Alternative: Custom Operators with Manual Lineage (Airflow 2.x and 3.x) + +If you prefer not to use OpenLineage, or are on older Airflow versions, you can manually extract and set lineage using DataHub's SQL parser: + +```python +from typing import Any, List, Tuple, Union +from airflow.models.baseoperator import BaseOperator +from datahub_airflow_plugin._config import get_enable_multi_statement +from datahub_airflow_plugin._sql_parsing_common import parse_sql_with_datahub +from datahub_airflow_plugin.entities import Urn + +class CustomSQLOperator(BaseOperator): + def __init__(self, sql: Union[str, List[str]], database: str, **kwargs: Any): + super().__init__(**kwargs) + self.sql = sql + self.database = database + + def execute(self, context: Any) -> Any: + # Execute SQL + # ... + + # Extract and set lineage + inlets, outlets = self._get_lineage() + context["ti"].task.inlets = inlets + context["ti"].task.outlets = outlets + + def _get_lineage(self) -> Tuple[List, List]: + # Get multi-statement config flag + enable_multi_statement = get_enable_multi_statement() + + # Parse SQL with multi-statement support + # Handles both string and list of SQL statements + sql_parsing_result = parse_sql_with_datahub( + sql=self.sql, + platform="postgres", # your platform + default_database=self.database, + env="PROD", + default_schema=None, + graph=None, + enable_multi_statement=enable_multi_statement, + ) + + inlets = [Urn(table) for table in sql_parsing_result.in_tables] + outlets = [Urn(table) for table in sql_parsing_result.out_tables] + return inlets, outlets +``` + +See [full example](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/tests/integration/dags/airflow3/custom_operator_sql_parsing.py). + +### Custom Extractors (Advanced - Legacy OpenLineage) + +For advanced use cases with the legacy OpenLineage package (`openlineage-airflow`), you can create a custom extractor. This is useful if you're using a built-in Airflow operator for which we don't support automatic lineage extraction. + +See this [example PR](https://github.com/datahub-project/datahub/pull/10452) which adds a custom extractor for the `BigQueryInsertJobOperator` operator. + +## Cleanup obsolete pipelines and tasks from DataHub + +There might be a case where the DAGs are removed from the Airflow but the corresponding pipelines and tasks are still there in the DataHub, let's call such pipelines ans tasks, `obsolete pipelines and tasks` + +Following are the steps to cleanup them from the datahub: + +- create a DAG named `Datahub_Cleanup`, i.e. + +```python +from datetime import datetime + +from airflow import DAG +from airflow.operators.bash import BashOperator + +from datahub_airflow_plugin.entities import Dataset, Urn + +with DAG( + "Datahub_Cleanup", + start_date=datetime(2024, 1, 1), + schedule_interval=None, + catchup=False, +) as dag: + task = BashOperator( + task_id="cleanup_obsolete_data", + dag=dag, + bash_command="echo 'cleaning up the obsolete data from datahub'", + ) + +``` + +- ingest this DAG, and it will remove all the obsolete pipelines and tasks from the DataHub based on the `cluster` value set in the `airflow.cfg` + +## Get all dataJobs associated with a dataFlow + +If you are looking to find all tasks (aka DataJobs) that belong to a specific pipeline (aka DataFlow), you can use the following GraphQL query: + +```graphql +query { + dataFlow(urn: "urn:li:dataFlow:(airflow,db_etl,prod)") { + childJobs: relationships( + input: { types: ["IsPartOf"], direction: INCOMING, start: 0, count: 100 } + ) { + total + relationships { + entity { + ... on DataJob { + urn + } + } + } + } + } +} +``` + +## Emit Lineage Directly + +If you can't use the plugin or annotate inlets/outlets, you can also emit lineage using the `DatahubEmitterOperator`. + +Reference [`lineage_emission_dag.py`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/lineage_emission_dag.py) for a full example. + +In order to use this example, you must first configure the DataHub hook. Like in ingestion, we support a DataHub REST hook and a Kafka-based hook. See the plugin configuration for examples. + +## Debugging + +### Missing lineage + +If you're not seeing lineage in DataHub, check the following: + +- Validate that the plugin is loaded in Airflow. Go to Admin -> Plugins and check that the DataHub plugin is listed. +- It should also print a log line like `INFO [datahub_airflow_plugin.datahub_listener] DataHub plugin using DataHubRestEmitter: configured to talk to ` during Airflow startup, and the `airflow plugins` command should list `datahub_plugin` with a listener enabled. +- If using the plugin's automatic lineage, ensure that the `enable_extractors` config is set to true and that automatic lineage is supported for your operator. +- If using manual lineage annotation, ensure that you're using the `datahub_airflow_plugin.entities.Dataset` or `datahub_airflow_plugin.entities.Urn` classes for your inlets and outlets. + +### Incorrect URLs + +If your URLs aren't being generated correctly (usually they'll start with `http://localhost:8080` instead of the correct hostname), you may need to set the webserver `base_url` config. + +```ini title="airflow.cfg" +[webserver] +base_url = http://airflow.mycorp.example.com +``` + +### TypeError ... missing 3 required positional arguments + +If you see errors like the following: + +```shell +ERROR - on_task_instance_success() missing 3 required positional arguments: 'previous_state', 'task_instance', and 'session' +Traceback (most recent call last): + File "/home/airflow/.local/lib/python3.8/site-packages/datahub_airflow_plugin/datahub_listener.py", line 124, in wrapper + f(*args, **kwargs) +TypeError: on_task_instance_success() missing 3 required positional arguments: 'previous_state', 'task_instance', and 'session' +``` + +The solution is to upgrade `acryl-datahub-airflow-plugin>=0.12.0.4` or upgrade `pluggy>=1.2.0`. See this [PR](https://github.com/datahub-project/datahub/pull/9365) for details. + +### Scheduler stalling + +For extremely large Airflow deployments with thousands of tasks, you may see issues where the plugin interferes with the performance of the Airflow scheduler. In those cases, you can set the `DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD_TIMEOUT=0` environment variable. This makes the DataHub plugin run fully in background threads, but can cause us to miss some metadata if the scheduler shuts down soon after processing a task. + +### Disabling the DataHub Plugin + +There are two ways to disable the DataHub Plugin: + +#### 1. Disable via Configuration + +Set the `datahub.enabled` configuration property to `False` in the `airflow.cfg` file and restart the Airflow environment to reload the configuration and disable the plugin. + +```ini title="airflow.cfg" +[datahub] +enabled = False +``` + +#### 2. Disable via Environment Variable (Kill-Switch) + +If a restart is not possible and you need a faster way to disable the plugin, you can use the kill-switch. Set the `AIRFLOW_VAR_DATAHUB_AIRFLOW_PLUGIN_DISABLE_LISTENER` environment variable to `true`. This ensures that the listener won't process anything. + +```shell +export AIRFLOW_VAR_DATAHUB_AIRFLOW_PLUGIN_DISABLE_LISTENER=true +``` + +This will immediately disable the plugin without requiring a restart. + +:::note Why Environment Variable Instead of Airflow Variable? +The plugin uses environment variables instead of Airflow's `Variable.get()` because listener hooks are called during SQLAlchemy's `after_flush` event (before the main transaction commits). Calling `Variable.get()` in this context creates a nested database session that can interfere with the outer transaction and cause data loss, such as missing TaskInstanceHistory records for retried tasks. +::: + +## Compatibility + +We try to support Airflow releases for ~2 years after their release. This is a best-effort guarantee - it's not always possible due to dependency / security issues cropping up in older versions. + +We no longer officially support Airflow <2.7. However, you can use older versions of `acryl-datahub-airflow-plugin` with older versions of Airflow. +We previously had two implementations of the plugin - v1 and v2. The v2 plugin is now the default, and the v1 plugin has since been removed. The v1 plugin had many limitations, chiefly that it does not support automatic lineage extraction. Docs for the v1 plugin can be accessed in our [docs archive](https://docs-website-r5eolot5n-acryldata.vercel.app/docs/lineage/airflow#datahub-plugin-v1). + +All recent versions require Python 3.10+. + +- Airflow 1.10.x, use acryl-datahub-airflow-plugin <= 0.9.1.0 (v1 plugin). +- Airflow 2.0.x, use acryl-datahub-airflow-plugin <= 0.11.0.1 (v1 plugin). +- Airflow 2.2.x, use acryl-datahub-airflow-plugin <= 0.14.1.5 (v2 plugin). +- Airflow 2.3 - 2.4.3, use acryl-datahub-airflow-plugin <= 1.0.0 (v2 plugin). +- Airflow 2.5 and 2.6, use acryl-datahub-airflow-plugin <= 1.1.0.4 (v2 plugin). + +DataHub also previously supported an Airflow [lineage backend](https://airflow.apache.org/docs/apache-airflow/2.2.0/lineage.html#lineage-backend) implementation. The lineage backend functionality was pretty limited - it did not support automatic lineage extraction, did not capture task failures, and did not work in AWS MWAA - and so it has been removed from the codebase. The [documentation for the lineage backend](https://docs-website-1wmaehubl-acryldata.vercel.app/docs/lineage/airflow/#using-datahubs-airflow-lineage-backend-deprecated) has been archived. + +## Additional references + +Related DataHub videos: + +- [Airflow Lineage](https://www.youtube.com/watch?v=3wiaqhb8UR0) +- [Airflow Run History in DataHub](https://www.youtube.com/watch?v=YpUOqDU5ZYg) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/lineage/dagster.md b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/dagster.md new file mode 100644 index 00000000..4a5c28b0 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/dagster.md @@ -0,0 +1,242 @@ +--- +title: Dagster Integration +slug: /lineage/dagster +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/lineage/dagster.md' +--- +# Dagster Integration + +This connector supports extracting: + +- Dagster Pipeline and Task Metadata +- Pipeline Run Status +- Table Lineage + +from Dagster. + +## Supported Versions + +This integration was verified using Dagster 1.7.0+. That does not necessary mean it will not be compatible will older versions. + +## Using DataHub's Dagster Sensor + +Dagster Sensors allow us to perform actions when important events occur in Dagster. DataHub's Dagster Sensor allows you to emit metadata after every Dagster pipeline run. This sensor can emit Pipelines, Tasks, and run results. For more details about Dagster Sensors, please refer to [the documentation](https://docs.dagster.io/concepts/partitions-schedules-sensors/sensors). + +### Prerequisites + +1. Create a Dagster project. See [Create New Project](https://docs.dagster.io/getting-started/create-new-project). +2. Create a [Definitions](https://docs.dagster.io/_apidocs/definitions#dagster.Definitions) class or [Repositories](https://docs.dagster.io/concepts/repositories-workspaces/repositories#repositories). +3. The creation of a new Dagster project via the UI uses the Definition class to define Dagster pipelines. + +### Setup + +1. Install the DataHub Dagster plugin package: + + ```shell + pip install acryl_datahub_dagster_plugin + ``` + +2. Import the DataHub Dagster Sensor, which is provided in the plugin package, to your Dagster Definition or Repository before starting the Dagster UI: + + **Example Definitions Class:** + + ```python + from dagster import Definitions + + from datahub.ingestion.graph.client import DatahubClientConfig + from datahub_dagster_plugin.sensors.datahub_sensors import ( + DatahubDagsterSourceConfig, + make_datahub_sensor, + ) + + config = DatahubDagsterSourceConfig( + datahub_client_config=DatahubClientConfig( + server="https://your_datahub_url/gms", token="your_datahub_token" + ), + dagster_url="https://my-dagster-cloud.dagster.cloud", + ) + + datahub_sensor = make_datahub_sensor(config=config) + + defs = Definitions( + sensors=[datahub_sensor], + ) + + ``` + +3. The DataHub Dagster plugin-provided sensor internally uses the following configurations. You can override the default config values using environment variables. + + **Configuration options:** + + | Configuration Option | Default Value | Description | + | ----------------------------------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | + | datahub_client_config | | The DataHub client config | + | dagster_url | | The URL to your Dagster Webserver. | + | capture_asset_materialization | True | Whether to capture asset keys as DataHub Datasets on AssetMaterialization event | + | capture_input_output | False | Whether to capture and try to parse input and output from HANDLED_OUTPUT, LOADED_INPUT events. (currently only [PathMetadataValue](https://github.com/dagster-io/dagster/blob/7e08c05dcecef9fd07f887c7846bd1c9a90e7d84/python_modules/dagster/dagster/_core/definitions/metadata/__init__.py#L655) metadata supported (EXPERIMENTAL) | + | platform_instance | | The instance of the platform that all assets produced by this recipe belong to. It is optional | + | asset_lineage_extractor | | You can implement your own logic to capture asset lineage information. See [example](/docs/lineage/dagster/#using-custom-logic-for-extracting-lineage) for details | + | enable_asset_query_metadata_parsing | True | Whether to enable parsing query from asset metadata. See [below](/docs/lineage/dagster/#extracting-lineage-from-sql-queries) for details | + | connect_ops_to_ops | False | Whether to connect ops to ops based on the order of execution | + | capture_dataset_from_asset_key | True | Whether to capture dataset from asset key | + | asset_keys_to_dataset_urn_converter | | Custom asset key to urn converter function. See details [below](/docs/lineage/dagster/#capturing-table-lineage) | + | materialize_dependencies | False | Whether to materialize asset dependency in DataHub. It emits a datasetKey for each dependencies | + | emit_queries | False | Whether to emit queries aspects | + | emit_assets | True | Whether to emit assets aspects | + | debug_mode | False | Whether to enable debug mode | + +4. Once the Dagster UI is running, turn on the provided Sensor execution. To turn on the Sensor, click on the **Overview** tab and then on the **Sensors** tab. Simply toggle the DataHub sensor on. + +Woohoo! Now, the DataHub Sensor is ready to emit metadata after every pipeline run. + +### How to Validate Installation + +1. Navigate to the Dagster UI. +2. Go to **Overview** > **Sensors** and look for `datahub_sensor`. +3. Start a Dagster Job. In the daemon logs, you should see DataHub-related log messages: + + ``` + datahub_sensor - Emitting metadata... + ``` + + This means that DataHub's sensor is correctly configured to emit metadata to DataHub. + +## Capturing Table Lineage + +There are a few ways to extract lineage, or relationships between tables, from Dagster. We recommend one or more of the following approaches to extract lineage automatically. + +### Extracting Lineage from SQL Queries + +#### But First: Extracting Asset Identifiers + +When naming Dagster Assets, we recommend the following structure: + +`key_prefix=["env", "platform", "db_name", "schema_name"]` + +This ensures that we correctly resolve the Asset name to a Dataset URN in DataHub. + +For example: + +```python +@asset( + key_prefix=["prod", "snowflake", "db_name", "schema_name"], # the fqdn asset name to be able to identify platform and make sure the asset is unique + deps=[iris_dataset], +) +``` + +If you properly name your Dagster Asset, you can establish a connection between the Asset and the dataset it is referring to, which is likely already stored in DataHub. This allows for accurate tracking and lineage information in the next steps. + +If you follow a different naming convention, you can create your own `asset_keys_to_dataset_urn_converter` logic and set a custom callback function. This can be used to generate a DataHub Dataset URN in any way you please, from metadata or otherwise. + +Here is an example that can create a DataHub URN from the Asset key naming convention specified above: + +```python +def asset_keys_to_dataset_urn_converter( + self, asset_key: Sequence[str] +) -> Optional[DatasetUrn]: + """ + Convert asset key to dataset urn + + By default, we assume the following asset key structure: + key_prefix=["prod", "snowflake", "db_name", "schema_name"] + """ + if len(asset_key) >= 3: + return DatasetUrn( + platform=asset_key[1], + env=asset_key[0], + name=".".join(asset_key[2:]), + ) + else: + return None +``` + +DataHub's Dagster integration can automatically detect dataset inputs and outputs for Software Defined Assets by analyzing the SQL queries it executes. To enable this feature, simply add the executed query to the Asset Metadata using the `Query` tag. + +Here's an example of a Software Defined Asset with an annotated Query: + +```python +@asset(key_prefix=["prod", "snowflake", "db_name", "schema_name"]) +def my_asset_table_a(snowflake: SnowflakeResource) -> MaterializeResult: + query = """ + create or replace table db_name.schema_name.my_asset_table_a as ( + SELECT * + FROM db_name.schema_name.my_asset_table_b + ); + """ + with snowflake.get_connection() as connection: + with connection.cursor() as cursor: + cursor.execute(query) + return MaterializeResult( + metadata={ + "Query": MetadataValue.text(query), + } + ) +``` + +In this example, the plugin will automatically identify and set the upstream lineage as `db_name.schema_name.my_asset_table_b`. + +Note: Proper asset naming is crucial, as the query parser determines the query language from the generated URN. In the example above, it will be `snowflake`. + +For a complete example job, refer to the [iris.py file](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/dagster-plugin/examples/iris.py) in the DataHub repository. + +### Extracting Lineage using SnowflakePandasIOManager + +The plugin offers an extended version of base SnowflakePandasIOManager provided by Dagster called `DataHubSnowflakePandasIOManager`. This version automatically captures Snowflake assets created by the IO manager and adds DataHub URN and links to the assets in Dagster. + +To use it, simply replace `SnowflakePandasIOManager` with `DataHubSnowflakePandasIOManager`. The enhanced version accepts two additional parameters: + +1. `datahub_base_url`: The base URL of the DataHub UI, used to generate direct links to Snowflake Datasets in DataHub. If not set, no URL will be generated. +2. `datahub_env`: The DataHub environment to use when generating URNs. Defaults to `PROD` if not specified. + +Example usage: + +```python +from datahub_dagster_plugin.modules.snowflake_pandas.datahub_snowflake_pandas_io_manager import ( + DataHubSnowflakePandasIOManager, +) +# ... +resources={ + "snowflake_io_manager": DataHubSnowflakePandasIOManager( + database="MY_DB", + account="my_snowflake_account", + warehouse="MY_WAREHOUSE", + user="my_user", + password="my_password", + role="my_role", + datahub_base_url="http://localhost:9002", + ), +} +``` + +### Using Dagster Ins and Out + +We can provide inputs and outputs to both Assets and Ops explicitly using a dictionary of `Ins` and `Out` corresponding to the decorated function arguments. While providing inputs and outputs, we can provide additional metadata as well. + +To create dataset upstream and downstream dependency for the Assets and Ops, you can use an ins and out dictionary with metadata provided. For reference, look at the sample jobs created using assets [`assets_job.py`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/dagster-plugin/examples/assets_job.py), or ops [`ops_job.py`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/dagster-plugin/examples/ops_job.py). + +### Using Custom Logic for Extracting Lineage + +You can define your own logic to capture asset lineage information. + +The output Tuple contains two dictionaries, one for input assets and the other for output assets. The key of the dictionary is the op key and the value is the set of asset URNs that are upstream or downstream of the op. + +```python +from datahub_dagster_plugin.client.dagster_generator import DagsterGenerator, DatasetLineage + +def asset_lineage_extractor( + context: RunStatusSensorContext, + dagster_generator: DagsterGenerator, + graph: DataHubGraph, +) -> Dict[str, DatasetLineage]: + dataset_lineage: Dict[str, DatasetLineage] = {} + + # Extracting input and output assets from the context + return dataset_lineage +``` + +[See an example job here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/dagster-plugin/examples/advanced_ops_jobs.py). + +## Debugging + +### Connection Error for DataHub Rest URL + +If you get `ConnectionError: HTTPConnectionPool(host='localhost', port=8080)`, then in that case your DataHub GMS service is not up. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/lineage/openlineage.md b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/openlineage.md new file mode 100644 index 00000000..b049e4ef --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/openlineage.md @@ -0,0 +1,200 @@ +--- +title: OpenLineage +slug: /lineage/openlineage +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/lineage/openlineage.md +--- +# OpenLineage + +DataHub, now supports [OpenLineage](https://openlineage.io/) integration. With this support, DataHub can ingest and display lineage information from various data processing frameworks, providing users with a comprehensive understanding of their data pipelines. + +## Features + +- **REST Endpoint Support**: DataHub now includes a REST endpoint that can understand OpenLineage events. This allows users to send lineage information directly to DataHub, enabling easy integration with various data processing frameworks. + +- **[Spark Event Listener Plugin](/docs/metadata-integration/java/acryl-spark-lineage)**: DataHub provides a Spark Event Listener plugin that seamlessly integrates OpenLineage's Spark plugin. This plugin enhances DataHub's OpenLineage support by offering additional features such as PathSpec support, column-level lineage, patch support and more. + +## OpenLineage Support with DataHub + +### 1. REST Endpoint Support + +DataHub's REST endpoint allows users to send OpenLineage events directly to DataHub. This enables easy integration with various data processing frameworks, providing users with a centralized location for viewing and managing data lineage information. + +With Spark and Airflow we recommend using the Spark Lineage or DataHub's Airflow plugin for tighter integration with DataHub. + +#### How to Use + +To send OpenLineage messages to DataHub using the REST endpoint, simply make a POST request to the following endpoint: + +``` +POST GMS_SERVER_HOST:GMS_PORT/openapi/openlineage/api/v1/lineage +``` + +Include the OpenLineage message in the request body in JSON format. + +Example: + +```json +{ + "eventType": "START", + "eventTime": "2020-12-28T19:52:00.001+10:00", + "run": { + "runId": "d46e465b-d358-4d32-83d4-df660ff614dd" + }, + "job": { + "namespace": "workshop", + "name": "process_taxes" + }, + "inputs": [ + { + "namespace": "postgres://workshop-db:None", + "name": "workshop.public.taxes", + "facets": { + "dataSource": { + "_producer": "https://github.com/OpenLineage/OpenLineage/tree/0.10.0/integration/airflow", + "_schemaURL": "https://raw.githubusercontent.com/OpenLineage/OpenLineage/main/spec/OpenLineage.json#/definitions/DataSourceDatasetFacet", + "name": "postgres://workshop-db:None", + "uri": "workshop-db" + } + } + } + ], + "producer": "https://github.com/OpenLineage/OpenLineage/blob/v1-0-0/client" +} +``` + +##### How to set up Airflow + +Follow the Airflow guide to setup the Airflow DAGs to send lineage information to DataHub. The guide can be found [here](https://airflow.apache.org/docs/apache-airflow-providers-openlineage/stable/guides/user.html). +The transport should look like this: + +```json +{ + "type": "http", + "url": "https://GMS_SERVER_HOST:GMS_PORT/openapi/openlineage/", + "endpoint": "api/v1/lineage", + "auth": { + "type": "api_key", + "api_key": "your-datahub-api-key" + } +} +``` + +#### How to modify configurations + +To modify the configurations for the OpenLineage REST endpoint, you can change it using environment variables. The following configurations are available: + +##### DataHub OpenLineage Configuration + +This document describes all available configuration options for the DataHub OpenLineage integration, including environment variables, application properties, and their usage. + +##### Configuration Overview + +The DataHub OpenLineage integration can be configured using environment variables, application properties files (`application.yml` or `application.properties`), or JVM system properties. All configuration options are prefixed with `datahub.openlineage`. + +##### Environment Variables + +| Environment Variable | Property | Type | Default | Description | +| ------------------------------------------------------ | ------------------------------------------------------ | ------- | ------- | ----------------------------------------------------------------------------------------------------------------- | +| `DATAHUB_OPENLINEAGE_ENV` | `datahub.openlineage.env` | String | `PROD` | Environment for DataFlow cluster and Dataset fabricType (see valid values below) | +| `DATAHUB_OPENLINEAGE_ORCHESTRATOR` | `datahub.openlineage.orchestrator` | String | `null` | Orchestrator name for DataFlow entities. When set, takes precedence over processing_engine facet and producer URL | +| `DATAHUB_OPENLINEAGE_PLATFORM_INSTANCE` | `datahub.openlineage.platform-instance` | String | `null` | Override DataFlow cluster (defaults to env if not specified) | +| `DATAHUB_OPENLINEAGE_COMMON_DATASET_ENV` | `datahub.openlineage.common-dataset-env` | String | `null` | Override Dataset environment independently from DataFlow cluster | +| `DATAHUB_OPENLINEAGE_COMMON_DATASET_PLATFORM_INSTANCE` | `datahub.openlineage.common-dataset-platform-instance` | String | `null` | Common platform instance for dataset entities | +| `DATAHUB_OPENLINEAGE_MATERIALIZE_DATASET` | `datahub.openlineage.materialize-dataset` | Boolean | `true` | Whether to materialize dataset entities | +| `DATAHUB_OPENLINEAGE_INCLUDE_SCHEMA_METADATA` | `datahub.openlineage.include-schema-metadata` | Boolean | `true` | Whether to include schema metadata in lineage | +| `DATAHUB_OPENLINEAGE_CAPTURE_COLUMN_LEVEL_LINEAGE` | `datahub.openlineage.capture-column-level-lineage` | Boolean | `true` | Whether to capture column-level lineage information | +| `DATAHUB_OPENLINEAGE_USE_PATCH` | `datahub.openlineage.use-patch` | Boolean | `false` | Whether to use patch operations for lineage/incremental lineage | +| `DATAHUB_OPENLINEAGE_FILE_PARTITION_REGEXP_PATTERN` | `datahub.openlineage.file-partition-regexp-pattern` | String | `null` | Regular expression pattern for file partition detection | + +> **Valid `env` values**: `PROD`, `DEV`, `TEST`, `QA`, `UAT`, `EI`, `PRE`, `STG`, `NON_PROD`, `CORP`, `RVW`, `PRD`, `TST`, `SIT`, `SBX`, `SANDBOX` +> +> **How `env` works**: +> +> - **By default**, `env` sets both the DataFlow cluster and Dataset fabricType for simplicity +> - **For advanced scenarios**, use `platform-instance` to override the DataFlow cluster or `common-dataset-env` to override the Dataset environment independently +> +> **Note**: The `env` property naming matches DataHub SDK conventions where `env` is the user-facing parameter that internally maps to the URN `cluster` field. + +##### Usage Examples + +**Setting Environment and Orchestrator** + +_Simple Configuration (Recommended):_ + +For most use cases, set `env` to configure both DataFlow and Datasets: + +```bash +# Development environment - sets DataFlow cluster to "dev" and Dataset fabricType to DEV +DATAHUB_OPENLINEAGE_ENV=DEV +DATAHUB_OPENLINEAGE_ORCHESTRATOR=my-orchestrator + +# Production environment - sets DataFlow cluster to "prod" and Dataset fabricType to PROD +DATAHUB_OPENLINEAGE_ENV=PROD +DATAHUB_OPENLINEAGE_ORCHESTRATOR=dagster + +# Staging environment +DATAHUB_OPENLINEAGE_ENV=STG +DATAHUB_OPENLINEAGE_ORCHESTRATOR=custom-pipeline +``` + +_Advanced Configuration (Multi-Region/Complex Deployments):_ + +Override DataFlow cluster or Dataset environment independently: + +```bash +# DataFlow in specific regional cluster, but datasets marked as generic PROD +DATAHUB_OPENLINEAGE_ENV=PROD +DATAHUB_OPENLINEAGE_PLATFORM_INSTANCE=prod-us-west-2 # DataFlow cluster override + +# Test pipeline against DEV data (cross-environment testing) +DATAHUB_OPENLINEAGE_ENV=PROD # DataFlow cluster: prod +DATAHUB_OPENLINEAGE_COMMON_DATASET_ENV=DEV # Dataset fabricType: DEV + +# Blue-green deployment +DATAHUB_OPENLINEAGE_ENV=PROD +DATAHUB_OPENLINEAGE_PLATFORM_INSTANCE=prod-blue # or prod-green +``` + +**Using Application Properties** + +Alternatively, configure via `application.yml`: + +```yaml +datahub: + openlineage: + env: PROD + orchestrator: my-custom-orchestrator + platform-instance: us-west-2 + capture-column-level-lineage: true +``` + +**Priority Order for Orchestrator Determination** + +The orchestrator name is determined in the following priority order: + +1. `DATAHUB_OPENLINEAGE_ORCHESTRATOR` environment variable (highest priority) +2. `processing_engine` facet in the OpenLineage event +3. Parsing the `producer` URL field with known patterns (Airflow, etc.) + +#### Known Limitations + +With Spark and Airflow we recommend using the Spark Lineage or DataHub's Airflow plugin for tighter integration with DataHub. + +- **[PathSpec](/docs/metadata-integration/java/acryl-spark-lineage/#configuring-hdfs-based-dataset-urns) Support**: While the REST endpoint supports OpenLineage messages, full [PathSpec](/docs/metadata-integration/java/acryl-spark-lineage/#configuring-hdfs-based-dataset-urns)) support is not yet available in the OpenLineage endpoint but it is available in the DataHub Cloud Spark Plugin. + +etc... + +### 2. Spark Event Listener Plugin + +DataHub's Spark Event Listener plugin enhances OpenLineage support by providing additional features such as PathSpec support, column-level lineage, and more. + +#### How to Use + +Follow the guides of the Spark Lineage plugin page for more information on how to set up the Spark Lineage plugin. The guide can be found [here](/docs/metadata-integration/java/acryl-spark-lineage) + +## References + +- [OpenLineage](https://openlineage.io/) +- [DataHub OpenAPI Guide](../api/openapi/openapi-usage-guide.md) +- [DataHub Spark Lineage Plugin](/docs/metadata-integration/java/acryl-spark-lineage) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/lineage/prefect.md b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/prefect.md new file mode 100644 index 00000000..7fd7bff1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/prefect.md @@ -0,0 +1,142 @@ +--- +title: Prefect Integration with DataHub +slug: /lineage/prefect +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/lineage/prefect.md' +--- +# Prefect Integration with DataHub + +## Overview + +DataHub supports integration with Prefect, allowing you to ingest: + +- Prefect flow and task metadata +- Flow run and Task run information +- Lineage information (when available) + +This integration enables you to track and monitor your Prefect workflows within DataHub, providing a comprehensive view of your data pipeline activities. + +## Prefect DataHub Block + +### What is a Prefect DataHub Block? + +Blocks in Prefect are primitives that enable the storage of configuration and provide an interface for interacting with external systems. The `prefect-datahub` block uses the [DataHub REST](../../metadata-ingestion/sink_docs/datahub.md#datahub-rest) emitter to send metadata events while running Prefect flows. + +### Prerequisites + +1. Use either Prefect Cloud (recommended) or a self-hosted Prefect server. +2. For Prefect Cloud setup, refer to the [Cloud Quickstart](https://docs.prefect.io/latest/getting-started/quickstart/) guide. +3. For self-hosted Prefect server setup, refer to the [Host Prefect Server](https://docs.prefect.io/latest/guides/host/) guide. +4. Ensure the Prefect API URL is set correctly. Verify using: + + ```shell + prefect profile inspect + ``` + +5. API URL format: + - Prefect Cloud: `https://api.prefect.cloud/api/accounts//workspaces/` + - Self-hosted: `http://:/api` + +## Setup Instructions + +### 1. Installation + +Install `prefect-datahub` using pip: + +```shell +pip install 'prefect-datahub' +``` + +Note: Requires Python 3.10+ + +### 2. Saving Configurations to a Block + +Save your configuration to the [Prefect block document store](https://docs.prefect.io/latest/concepts/blocks/#saving-blocks): + +```python +from prefect_datahub.datahub_emitter import DatahubEmitter + +DatahubEmitter( + datahub_rest_url="http://localhost:8080", + env="PROD", + platform_instance="local_prefect" +).save("MY-DATAHUB-BLOCK") +``` + +Configuration options: + +| Config | Type | Default | Description | +| ----------------- | ----- | ----------------------- | ---------------------------------------------------------------------------------------------------------- | +| datahub_rest_url | `str` | `http://localhost:8080` | DataHub GMS REST URL | +| env | `str` | `PROD` | Environment for assets (see [FabricType](/docs/graphql/enums/#fabrictype)) | +| platform_instance | `str` | `None` | Platform instance for assets (see [Platform Instances](/docs/platform-instances/)) | + +### 3. Using the Block in Prefect Workflows + +Load and use the saved block in your Prefect workflows: + +```python +from prefect import flow, task +from prefect_datahub.dataset import Dataset +from prefect_datahub.datahub_emitter import DatahubEmitter + +datahub_emitter = DatahubEmitter.load("MY-DATAHUB-BLOCK") + +@task(name="Transform", description="Transform the data") +def transform(data): + data = data.split(" ") + datahub_emitter.add_task( + inputs=[Dataset("snowflake", "mydb.schema.tableA")], + outputs=[Dataset("snowflake", "mydb.schema.tableC")], + ) + return data + +@flow(name="ETL flow", description="Extract transform load flow") +def etl(): + data = transform("This is data") + datahub_emitter.emit_flow() +``` + +**Note**: To emit tasks, you must call `emit_flow()`. Otherwise, no metadata will be emitted. + +## Concept Mapping + +| Prefect Concept | DataHub Concept | +| -------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | +| [Flow](https://docs.prefect.io/latest/concepts/flows/) | [DataFlow](/docs/generated/metamodel/entities/dataflow/) | +| [Flow Run](https://docs.prefect.io/latest/concepts/flows/#flow-runs) | [DataProcessInstance](/docs/generated/metamodel/entities/dataprocessinstance) | +| [Task](https://docs.prefect.io/latest/concepts/tasks/) | [DataJob](/docs/generated/metamodel/entities/datajob/) | +| [Task Run](https://docs.prefect.io/latest/concepts/tasks/#tasks) | [DataProcessInstance](/docs/generated/metamodel/entities/dataprocessinstance) | +| [Task Tag](https://docs.prefect.io/latest/concepts/tasks/#tags) | [Tag](/docs/generated/metamodel/entities/tag/) | + +## Validation and Troubleshooting + +### Validating the Setup + +1. Check the Prefect UI's Blocks menu for the DataHub emitter. +2. Run a Prefect workflow and look for DataHub-related log messages: + + ```text + Emitting flow to datahub... + Emitting tasks to datahub... + ``` + +### Debugging Common Issues + +#### Incorrect Prefect API URL + +If the Prefect API URL is incorrect, set it manually: + +```shell +prefect config set PREFECT_API_URL='http://127.0.0.1:4200/api' +``` + +#### DataHub Connection Error + +If you encounter a `ConnectionError: HTTPConnectionPool(host='localhost', port=8080)`, ensure that your DataHub GMS service is running. + +## Additional Resources + +- [Prefect Documentation](https://docs.prefect.io/) +- [DataHub Documentation](/docs/) + +For more information or support, please refer to the official Prefect and DataHub documentation or reach out to their respective communities. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/lineage/sql_parsing.md b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/sql_parsing.md new file mode 100644 index 00000000..07f0492f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/lineage/sql_parsing.md @@ -0,0 +1,69 @@ +--- +title: SQL Parsing +sidebar_label: SQL Parsing +slug: /lineage/sql_parsing +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/lineage/sql_parsing.md +--- + +# The DataHub SQL Parser + +Many data platforms are built on top of SQL, which means deeply understanding SQL queries is critical for understanding column-level lineage, usage, and more. + +DataHub's SQL parser is built on top of [sqlglot](https://github.com/tobymao/sqlglot) and adds a number of additional features to improve the accuracy of SQL parsing. + +In our benchmarks, the DataHub SQL parser generates lineage with 97-99% accuracy and outperforms other SQL parsers by a wide margin. + +We've published a blog post on some of the technical details of the parser: [Extracting Column Lineage from SQL Queries](https://medium.com/datahub-project/extracting-column-level-lineage-from-sql-779b8ce17567). + +## Built-in SQL Parsing Support + +If you're using a tool that DataHub already [integrates with](/integrations), check the documentation for that specific integration. +Most of our integrations, including Snowflake, BigQuery, Redshift, dbt, Looker, PowerBI, Airflow, etc, use the SQL parser to generate column-level lineage and usage statistics. + +If you’re using a different database system for which we don’t support column-level lineage out of the box, but you do have a database query log available, the [SQL queries](../generated/ingestion/sources/sql-queries.md) connector can generate column-level lineage and table/column usage statistics from the query log. + +## SDK Support + +Our SDK provides a [`DataHubGraph.parse_sql_lineage()`](../../python-sdk/clients/graph-client.mdx#datahub.ingestion.graph.client.DataHubGraph.parse_sql_lineage) method for programmatically parsing SQL queries. + +The resulting object contains a `sql_parsing_result.debug_info.confidence_score` field, which is a 0-1 value indicating the confidence of the parser. + +There are also a number of utilities in the `datahub.sql_parsing` module. The `SqlParsingAggregator` is particularly useful, as it can also resolve lineage across temp tables and table renames/swaps. +Note that these utilities are not officially part of the DataHub SDK and hence do not have the same level of stability and support as the rest of the SDK. + +## Capabilities + +### Supported + +- Table-level lineage for `SELECT`, `CREATE`, `INSERT`, `UPDATE`, `DELETE`, and `MERGE` statements +- Column-level lineage for `SELECT` (including `SELECT INTO`), `CREATE VIEW`, `CREATE TABLE AS SELECT` (CTAS), `INSERT`, and `UPDATE` statements +- Subqueries +- CTEs +- `UNION ALL` constructs - will merge lineage across the clauses of the `UNION` +- `SELECT *` and similar expressions will automatically be expanded with the table schemas registered in DataHub. This includes support for platform instances. +- Automatic handling for systems where table and column names are case insensitive. Generally requires that `convert_urns_to_lowercase` is enabled when the corresponding table schemas were ingested into DataHub. + - Specifically, we'll do fuzzy matching against the table names and schemas to resolve the correct URNs. We do not support having multiple tables/columns that only differ in casing. +- For BigQuery, sharded table suffixes will automatically be normalized. For example, `proj.dataset.table_20230616` will be normalized to `proj.dataset.table_yyyymmdd`. This matches the behavior of our BigQuery ingestion connector, and hence will result in lineage linking up correctly. + +### Not supported + +- Scalar `UDFs` - We will generate lineage pointing at the columns that are inputs to the UDF, but will not be able to understand the UDF itself. +- Table-valued functions, including tabular `UDFs` +- `json_extract` and similar functions +- `UNNEST` - We will do a best-effort job, but cannot reliably generate column-level lineage in the presence of `UNNEST` constructs. +- Structs - We will do a best-effort attempt to resolve struct subfields, but it is not guaranteed. This will only impact column-level lineage. + - This extends to things like dynamic table unpacking e.g. `SELECT IF (main.id is not null, main, extras).* FROM my_schema.main_users main FULL JOIN my_schema.external_users extras USING (id)` in BigQuery. +- Snowflake's multi-table inserts +- Multi-statement SQL / SQL scripting + +### Limitations + +- We only support the 20+ SQL dialects supported by the underlying [sqlglot](https://github.com/tobymao/sqlglot) library. +- There's a few SQL syntaxes that we don't support yet, but intend to support in the future. + - `INSERT INTO (col1_new, col2_new) SELECT col1_old, col2_old FROM ...`. We only support `INSERT INTO` statements that either (1) don't specify a column list, or (2) specify a column list that matches the columns in the `SELECT` clause. + - `MERGE INTO` statements - We don't generate column-level lineage for these. +- In cases where the table schema information in DataHub is outdated or otherwise incorrect, we may not be able to generate accurate column-level lineage. +- We sometimes trip over BigQuery queries that use the `_partitiontime` and `_partitiondate` pseudo-columns with a table name prefix e.g. `my_table._partitiontime` fails. However, unqualified references like `_partitiontime` and `_partitiondate` will be fine. +- We do not consider columns referenced in filtering or organizational clauses such as `WHERE`, `GROUP BY`, `ORDER BY`, `JOIN`, `HAVING`, or `PARTITION BY` to be part of lineage. For example, `SELECT col1, col2 FROM upstream_table WHERE col3 = 3` will not generate any lineage related to `col3`. +- We generally only analyze static table references. For example, this Snowflake query will not generate any lineage: `SELECT * FROM identifier('my_db.my_schema.my_table')`, since the `identifier` function is resolved at SQL runtime. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/links.md b/docs-archive/versioned_docs/version-1.5.0/docs/links.md new file mode 100644 index 00000000..dbd3563e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/links.md @@ -0,0 +1,72 @@ +--- +title: Articles & Talks +slug: /links +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/links.md' +--- +# Articles & Talks + +## Overviews + +- [Tech Deep Dive: DataHub Metadata Service Authentication](https://medium.com/datahub-project/tech-deep-dive-introducing-datahub-metadata-service-authentication-661e3aabbad0) and [video](https://www.youtube.com/watch?v=DPY0G3Ix7Y8) +- [Data in Context: Lineage Explorer in DataHub](https://medium.com/datahub-project/data-in-context-lineage-explorer-in-datahub-a53a9a476dc4) +- [DataHub Basics — Users, Groups, & Authentication 101](https://www.youtube.com/watch?v=8Osw6p9vDYY) +- [DataHub Basics: Lineage 101](https://www.youtube.com/watch?v=rONGpsndzRw) + +## Best Practices + +- [Tags and Terms: Two Powerful DataHub Features, Used in Two Different Scenarios](https://medium.com/datahub-project/tags-and-terms-two-powerful-datahub-features-used-in-two-different-scenarios-b5b4791e892e) + +## Case Studies + +- [Enabling Data Discovery in a Data Mesh: The Saxo Journey](https://medium.com/datahub-project/enabling-data-discovery-in-a-data-mesh-the-saxo-journey-451b06969c8f) +- [DataHub @ Grofers Case Study](https://www.youtube.com/watch?v=m9kUYAuezFI) +- [DataHub @ LinkedIn: Extending the OSS UI](https://www.youtube.com/watch?v=Rdt4kJqDoww) +- [DataHub @ hipages Case Study: Oct 29 2021](https://www.youtube.com/watch?v=OFNzjUdMcJQ) +- [DataHub @ Adevinta Case Study: Sept 24 2021 Community Town Hall](https://www.youtube.com/watch?v=u9DRa_5uPIM) +- [DataHub at Bizzy (Case Study): Aug 27 2021 Community Meeting](https://www.youtube.com/watch?v=SuhLRr3QKt8) + +## Related Articles + +- [DataHub: A Generalized Metadata Search & Discovery Tool](https://engineering.linkedin.com/blog/2019/data-hub) +- [Open sourcing DataHub: LinkedIn’s metadata search and discovery platform](https://engineering.linkedin.com/blog/2020/open-sourcing-datahub--linkedins-metadata-search-and-discovery-p) +- [Data Catalogue — Knowing your data](https://medium.com/albert-franzi/data-catalogue-knowing-your-data-15f7d0724900) +- [LinkedIn DataHub Application Architecture Quick Understanding](https://medium.com/@liangjunjiang/linkedin-datahub-application-architecture-quick-understanding-a5b7868ee205) +- [LinkIn DataHub Metadata Ingestion Scripts Unofficical Guide](https://medium.com/@liangjunjiang/linkin-datahub-etl-unofficical-guide-7c3949483f8b) +- [DataHub - RPubs](https://rpubs.com/Priya_Shaji/dataHub) +- [A Dive Into Metadata Hubs](https://www.holistics.io/blog/a-dive-into-metadata-hubs/) +- [How LinkedIn, Uber, Lyft, Airbnb and Netflix are Solving Data Management and Discovery for Machine Learning Solutions](https://www.kdnuggets.com/2019/08/linkedin-uber-lyft-airbnb-netflix-solving-data-management-discovery-machine-learning-solutions.html) +- [Data Discovery in 2020](https://medium.com/@torokyle/data-discovery-in-2020-3c907383caa0) +- [Work-Bench Snapshot: The Evolution of Data Discovery & Catalog](https://medium.com/work-bench/work-bench-snapshot-the-evolution-of-data-discovery-catalog-2f6c0425616b) +- [In-house Data Discovery platforms](https://datastrategy.substack.com/p/in-house-data-discovery-platforms) +- [A Data Engineer’s Perspective On Data Democratization](https://towardsdatascience.com/a-data-engineers-perspective-on-data-democratization-a8aed10f4253) +- [25 Hot New Data Tools and What They DON’T Do](https://blog.amplifypartners.com/25-hot-new-data-tools-and-what-they-dont-do/) +- [4 Data Trends to Watch in 2020](https://medium.com/memory-leak/4-data-trends-to-watch-in-2020-491707902c09) +- [Application Performance Monitor and Distributed Tracing with Apache SkyWalking in DataHub](https://medium.com/@liangjunjiang/application-performance-monitor-and-distributed-tracing-with-apache-skywalking-in-datahub-16bc65e6c670) +- [Emerging Architectures for Modern Data Infrastructure](https://a16z.com/2020/10/15/the-emerging-architectures-for-modern-data-infrastructure/) +- [Almost Everything You Need To Know on Data Discovery Platforms](https://eugeneyan.com/writing/data-discovery-platforms/) +- [Creating Notebook-based Dynamic Dashboards](https://towardsdatascience.com/creating-notebook-based-dynamic-dashboards-91f936adc6f3) + +## Talks & Presentations + +- [DataHub: Powering LinkedIn's Metadata](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/DataHub_-_Powering_LinkedIn_Metadata.pdf) @ [Budapest Data Forum 2020](https://budapestdata.hu/2020/en/) +- [Taming the Data Beast Using DataHub](https://www.youtube.com/watch?v=bo4OhiPro7Y) @ [Data Engineering Melbourne Meetup November 2020](https://www.meetup.com/Data-Engineering-Melbourne/events/kgnvlrybcpbjc/) +- [Metadata Management And Integration At LinkedIn With DataHub](https://www.dataengineeringpodcast.com/datahub-metadata-management-episode-147/) @ [Data Engineering Podcast](https://www.dataengineeringpodcast.com) +- [The evolution of metadata: LinkedIn’s story](https://speakerdeck.com/shirshanka/the-evolution-of-metadata-linkedins-journey-strata-nyc-2019) @ [Strata Data Conference 2019](https://conferences.oreilly.com/strata/strata-ny-2019.html) +- [Journey of metadata at LinkedIn](https://www.youtube.com/watch?v=OB-O0Y6OYDE) @ [Crunch Data Conference 2019](https://crunchconf.com/2019) +- [DataHub Journey with Expedia Group](https://www.youtube.com/watch?v=ajcRdB22s5o) +- [Saxo Bank's Data Workbench](https://www.slideshare.net/SheetalPratik/linkedinsaxobankdataworkbench) +- [Data Discoverability at SpotHero](https://www.slideshare.net/MaggieHays/data-discoverability-at-spothero) + +## Non-English + +- [LinkedIn 元数据之旅的最新进展—Data Hub](https://blog.csdn.net/DataPipeline/article/details/100155781) +- [数据治理篇: 元数据之 datahub-概述](https://www.jianshu.com/p/04630b0c63f7) +- [DataHub——实时数据治理平台](https://segmentfault.com/a/1190000022563622) +- [数据治理工具-元数据管理](https://blog.csdn.net/weixin_42526352/article/details/105371012) +- [元数据管理框架的独舞](https://mp.weixin.qq.com/s/J6xtX3js70brdN3c_7ZkNg) +- [【DataHub】DataHub QuickStart](https://www.jianshu.com/p/eb34e7088c77) +- [数据治理工具调研之 DataHub](https://www.cnblogs.com/CodingJacob/p/di2jiang-gong-ju-diao-yan-zhidatahub.html) +- [LinkedIn gibt die Datenplattform DataHub als Open Source frei](https://www.heise.de/developer/meldung/LinkedIn-gibt-die-Datenplattform-DataHub-als-Open-Source-frei-4663773.html) +- [Linkedin bringt Open-Source-DataHub](https://www.itmagazine.ch/artikel/71532/Linkedin_bringt_Open-Source-DataHub.html) +- [DataHub: универсальный инструмент поиска и обнаружения метаданных](https://habr.com/ru/post/520930/) +- [DataHub с открытым исходным кодом: платформа поиска и обнаружения метаданных от LinkedIn](https://habr.com/ru/post/521536/) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/change-proposals.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/change-proposals.md new file mode 100644 index 00000000..faa59546 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/change-proposals.md @@ -0,0 +1,368 @@ +--- +title: Change Proposals +slug: /managed-datahub/change-proposals +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/change-proposals.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Change Proposals + + + +## Overview + +Keeping your data organized can be hard work when you have a limited number of data owners. With DataHub Cloud, you can crowdsource metadata completion using change proposals. Change Proposals enable data users to suggest Tags, Terms, Domains, Owners, Descriptions, and even Structured Properties to be added to data assets. Once a change proposal is raised, data owners and stewards can review change proposals, approving or denying these suggestion. + +## Permissions + +### Creating Proposals + +To create proposals on assets, users need to have the following resource privileges: + +- `Propose Tags` +- `Propose Glossary Terms` +- `Propose Owners` +- `Propose Domain` +- `Propose Properties` +- `Propose Description` +- `Propose Dataset Column Tags` +- `Propose Dataset Column Glossary Terms` +- `Propose Dataset Column Structured Properties` +- `Propose Dataset Column Descriptions` + +To create proposals to change the Business Glossary, users need to have the following platform privileges: + +- `Propose Create Glossary Term` +- `Propose Create Glossary Node (Term Group)` + +Which are granted by default using the **Reader**, **Editor**, and **Admin** roles by default. + +### Reviewing Proposals + +To review proposals for an asset, users need to have the following resource privileges: + +- `Manage Tag Proposals` +- `Manage Glossary Term Proposals` +- `Manage Property Proposals` +- `Manage Domain Proposals` +- `Manage Owner Proposals` +- `Manage Description Proposals` + +To review proposals to change the Business Glossary, users need to have the following platform privileges: + +- `Manage Glossaries` + +Which are granted by default **Editor** and **Admin** roles. + +## Using Approval Workflows + +### Proposing Tags and Glossary Terms + +1. When adding a Tag or Glossary Term to a column or entity, you will see a propose button. + +

+ +

+ +2. After proposing the Glossary Term, you will see it appear in a proposed state. + +

+ +

+ +3. This proposal will be sent to the inbox of reviewers. + +

+ +

+ +4. From there, they can choose to either accept or reject the proposal. A full log of all accepted or rejected proposals is kept for each user. + +### Proposing Owners + +1. When adding an Owner to an entity, you will see a propose button. + +

+ +

+ +2. After proposing the Owner(s), you will see the owner(s) appear in a proposed state. + +

+ +

+ +3. This proposal will be sent to the inbox of reviewers. + +

+ +

+ +4. From there, they can choose to either accept or reject the proposal. A full log of all accepted or rejected proposals is kept for each user. + +### Proposing Domain + +1. When adding a Domain to an entity, you will see a propose button. + +

+ +

+ +2. After proposing the Domain, you will see the Domain appear in a proposed state. + +

+ +

+ +3. This proposal will be sent to the inbox of reviewers. + +

+ +

+ +4. From there, they can choose to either accept or reject the proposal. A full log of all accepted or rejected proposals is kept for each user. + +### Proposing Structured Properties + +1. When adding a Structured property to a column or an entity, you will see a propose button. + +

+ +

+ +2. After proposing the Structured Properties, you will see them appear in a proposed state. + +

+ +

+ +3. This proposal will be sent to the inbox of reviewers. + +

+ +

+ +4. From there, they can choose to either accept or reject the proposal. A full log of all accepted or rejected proposals is kept for each user. + +### Proposing Documentation or Description Updates + +1. When updating the documentation of any entity, or description of a dataset column, you can click the propose button + +2. This proposal will be sent to the inbox of reviewers. + +

+ +

+ +3. From there, they can choose to either accept or reject the proposal. + +### Proposing additions to your Business Glossary + +1. Navigate to your glossary by going to the Govern menu in the top right and selecting Glossary. + +2. Click the plus button to create a new Glossary Term. From that menu, select Propose. + +

+ +

+ +3. This proposal will be sent to the inbox of reviewers. + +

+ +

+ +4. From there, they can choose to either accept or reject the proposal. A full log of all accepted or rejected proposals is kept for each user. + +### Reviewing Proposals + +Proposals will be visible inside your **Task Center**, which is accessible via the navigation sidebar. From the task center, you can choose to accept or deny proposals sourced for assets you are responsible for. + +## Change Proposal Notifications + +You can enable notifications in the following scenarios: + +- A proposal you raised is approved or denied +- You are assigned to a new change proposal +- A proposal you are assigned to is approved or denied + +Via **Slack** and **Email**. + +To enable notifications, navigate to **Settings > My Notifications**. + +## Creating Proposals via API + +DataHub exposes a GraphQL API for each type of change proposal. At a high level, callers of this API will be required to provide the following details: + +1. A unique identifier for the target Metadata Entity (URN) +2. An optional sub-resource identifier which designates a sub-resource to attach the Tag, Glossary Term, owner, domain or Structured property to. For example reference to a particular "field" within a Dataset. +3. A unique identifier for the Tag/Glossary Term/Owner/Domain/Structured property they wish to propose (URN) + +In the following sections, we will describe how to construct each of these items and use the DataHub GraphQL API to submit Tag or Glossary Term proposals. + +#### Constructing an Entity Identifier + +Inside DataHub, each Metadata Entity is uniquely identified by a Universal Resource Name, or an URN. This identifier can be copied from the entity page, extracted from the API, or read from a downloaded search result. You can also use the helper methods in the datahub python library given a set of components. + +#### Constructing a Sub-Resource Identifier + +Specific Metadata Entity types have additional sub-resources to which Tags may be applied. +Today, this only applies for Dataset Metadata Entities, which have a "fields" sub-resource. In this case, the `subResource` value would be the field path for the schema field. + +#### Finding an Identifier for Tag/Glossary Term/Owner/Domain/Structure property + +All of these are uniquely identified by an URN. + +Tag URNs have the following format: +`urn:li:tag:` + +Glossary Term URNs have the following format: +`urn:li:glossaryTerm:` + +Domain URNs have the following format: +`urn:li:domain:` + +Owner URNs have the following format: +`urn:li:corpuser:` or `urn:li:corpGroup:` + +Structured Property URNs have the following format: +`urn:li:structuredProperty:` + +These identifiers can be copied from the url of the corresponding entity pages. + +#### Issuing a GraphQL Query + +Once we've constructed an Entity URN, any relevant sub-resource identifiers, we're ready to propose! To do so, we'll use the DataHub GraphQL API. + +In particular, we'll be using the proposeTags, proposeTerms, proposeDomain, proposeOwners, proposeStructuredProperties, proposeCreateGlossaryTerm, proposeCreateGlossaryNode, proposeDataContract, and proposeUpdateDescription Mutations, which have the following interface: + +``` +type Mutation { + proposeTags(input: ProposeTagsInput!): String! # Returns Proposal URN. +} + +input ProposeTagsInput { + description: String # Optional note explaining the proposal + resourceUrn: String! # Required. e.g. "urn:li:dataset:(...)" + subResource: String # Optional. e.g. "fieldName" + subResourceType: String # Optional. "DATASET_FIELD" for dataset fields + tagUrns: [String!]! # Required. e.g. ["urn:li:tag:Marketing"] +} +``` + +``` +type Mutation { + proposeTerms(input: ProposeTermsInput!): String! # Returns Proposal URN. +} + +input ProposeTermsInput { + description: String # Optional note explaining the proposal + resourceUrn: String! # Required. e.g. "urn:li:dataset:(...)" + subResource: String # Optional. e.g. "fieldName" + subResourceType: String # Optional. "DATASET_FIELD" for dataset fields + termUrns: [String!]! # Required. e.g. ["urn:li:glossaryTerm:Marketing"] +} +``` + +``` +type Mutation { + proposeDomain(input: ProposeDomainInput!): String! # Returns Proposal URN. +} + +input ProposeDomainInput { + description: String # Optional note explaining the proposal + resourceUrn: String! # Required. e.g. "urn:li:dataset:(...)" + domainUrn: String! # Required. e.g. ["urn:li:domain:Marketing"] +} +``` + +``` +type Mutation { + proposeOwners(input: ProposeOwnersInput!): String! # Returns Proposal URN. +} + +input ProposeOwnersInput { + description: String # Optional note explaining the proposal + resourceUrn: String! # Required. e.g. "urn:li:dataset:(...)" + owners: [OwnerInput!]! # Required +} + +input OwnerInput { + ownerUrn: String! # Required. e.g. "urn:li:owner:(...)" + ownerEntityType: OwnerEntityType! # Required. e.g. "CORP_USER" + type: OwnershipType # Optional + ownershipTypeUrn: String # Optional. The urn of the ownership type entity. +} +``` + +``` +type Mutation { + proposeStructuredProperties(input: ProposeStructuredPropertiesInput!): String! # Returns Proposal URN. +} + +input ProposeStructuredPropertiesInput { + description: String # Optional note explaining the proposal + resourceUrn: String! # Required. e.g. "urn:li:dataset:(...)" + subResource: String # Optional. e.g. "fieldName" + subResourceType: String # Optional. "DATASET_FIELD" for dataset fields + structuredProperties: [StructuredPropertyInputParams!]! +} + +input StructuredPropertyInputParams { + structuredPropertyUrn: String! # Required. e.g. "urn:li:structuredProperty:(...)" + values: [PropertyValueInput!]! # Required. e.g. "{ stringValue: ''}" +} +``` + +``` +type Mutation { + proposeCreateGlossaryTerm(input: CreateGlossaryEntityInput!): Boolean +} + +input CreateGlossaryEntityInput { + id: String # Optional. Otherwise uuid is generated + name: String! # Required. e.g. "Marketing" + description: String # Optional + parentNode: String # Optional. e.g. "urn:li:glossaryNode:(...)" + proposalNote: String # Optional. Context for the proposal +} +``` + +``` +type Mutation { + proposeCreateGlossaryNode(input: CreateGlossaryEntityInput!): Boolean +} + +input CreateGlossaryEntityInput { + id: String # Optional. Otherwise uuid is generated + name: String! # Required. e.g. "Marketing" + description: String # Optional + parentNode: String # Optional. e.g. "urn:li:glossaryNode:(...)" + proposalNote: String # Optional. Context for the proposal +} +``` + +``` +mutation proposeUpdateDescription($input: DescriptionUpdateInput!) { + proposeUpdateDescription(input: $input) +} + +""" +Currently supports DatasetField descriptions only +""" +input DescriptionUpdateInput { + description: String! # the new description + resourceUrn: String! + subResourceType: SubResourceType + subResource: String + proposalNote: String # Context for the proposal +} + +``` + +## FAQ + +**1. My colleagues have created some proposals, but I'm not seeing this in my Task Center. Why not?** + +Most likely, this means your privileges are not configured properly. If you are a DataHub Admin, navigate to **Settings > Permissions** to edit your roles and policies to ensure you have the privileges listed in the **Permissions** section above. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/chrome-extension.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/chrome-extension.md new file mode 100644 index 00000000..a9b9f814 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/chrome-extension.md @@ -0,0 +1,99 @@ +--- +description: Learn how to use the DataHub Cloud Chrome extension. +title: DataHub Cloud Chrome Extension +sidebar_label: Cloud Chrome Extension +slug: /managed-datahub/chrome-extension +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/chrome-extension.md +--- + +# DataHub Cloud Chrome Extension + +## Supported Tools + +The DataHub Cloud Chrome extension currently supports the following platforms and entity types: + +- **Looker** - Dashboards and Explores +- **Tableau** - Workbooks, Views, and Datasources +- **PowerBI** - Dashboards and Reports +- **BigQuery** - Datasets, Tables, Sharded Tables, and Partitioned Tables +- **Databricks** - Catalogs, Databases, and Tables +- **Mode Analytics** - Reports, Charts, Queries, and Datasets +- **Superset/Preset** - Dashboards and Charts +- **Hex** - Projects and Components +- **Metabase** - Dashboards, Questions, and Models + +## Installing the Extension + +In order to use the DataHub Cloud Chrome extension, you need to download it onto your browser from the Chrome web store [here](https://chrome.google.com/webstore/detail/datahub-chrome-extension/aoenebhmfokhglijmoacfjcnebdpchfj). + +

+ +

+ +Simply click "Add to Chrome" then "Add extension" on the ensuing popup. + +## Configuring the Extension + +Once you have your extension installed, you'll need to configure it to work with your DataHub Cloud deployment. + +1. Click the extension button on the right of your browser's address bar to view all of your installed extensions. Click on the newly installed DataHub extension. + +

+ +

+ +2. Fill in your DataHub domain and click "Continue" in the extension popup that appears. + +

+ +

+ +If your organization uses standard SaaS domains for Looker, you should be ready to go! + +### Additional Configurations + +Some organizations have custom SaaS domains for Looker and some DataHub Cloud deployments utilize **Platform Instances** and set custom **Environments** when creating DataHub assets. If any of these situations applies to you, please follow the next few steps to finish configuring your extension. + +1. Click on the extension button and select your DataHub extension to open the popup again. Now click the settings icon in order to open the configurations page. + +

+ +

+ +2. Fill out any and save custom configurations you have in the **TOOL CONFIGURATIONS** section. Here you can configure a custom domain, a Platform Instance associated with that domain, and the Environment set on your DataHub assets. If you don't have a custom domain but do have a custom Platform Instance or Environment, feel free to leave the field domain empty. + +

+ +

+ +## Using the Extension + +Once you have everything configured on your extension, it's time to use it! + +1. First ensure that you are logged in to your DataHub Cloud instance. + +2. Navigate to Looker or Tableau and log in to view your data assets. + +3. Navigate to a page where DataHub can provide insights on your data assets (Dashboards and Explores). + +4. Click the DataHub Cloud extension button on the bottom right of your page to open a drawer where you can now see additional information about this asset right from your DataHub instance. + +

+ +

+ +## Advanced: Self-Hosted DataHub + +If you are using the DataHub Cloud Chrome extension for your self-hosted DataHub instance, everything above is applicable. However, there is one additional step you must take in order to set up your instance to be compatible with the extension. + +### Configure Auth Cookies + +In order for the Chrome extension to work with your instance, it needs to be able to make authenticated requests. Therefore, authentication cookies need to be set up so that they can be shared with the extension on your browser. You must update the values of two environment variables in your `datahub-frontend` container: + +``` +AUTH_COOKIE_SAME_SITE="NONE" +AUTH_COOKIE_SECURE=true +``` + +Once your re-deploy your `datahub-frontend` container with these values, you should be good to go! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-ms-entra.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-ms-entra.md new file mode 100644 index 00000000..8e75de52 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-ms-entra.md @@ -0,0 +1,100 @@ +--- +title: 'SCIM Integration: MS Entra and DataHub' +hide_title: true +sidebar_label: 'SCIM Integration: MS Entra and DataHub' +slug: /managed-datahub/configuring-identity-provisioning-with-ms-entra +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/configuring-identity-provisioning-with-ms-entra.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +## SCIM Integration: MS Entra and DataHub + + + +## Overview + +On completion of this setup the MS Entra will automatically manage the groups/users/roles from MS Entra to DataHub. + +Consider following configuration in MS Entra + +- A group `governance-team` group +- And it has two memeber `john` and `sid` +- And the group has role `Reader` + +If you configure the `governance-team` for auto provisioning, MS Entra will creates the `governance-team` group and it's member automatically on DataHub and set the `Reader` roles on users. + +If you remove `john` from group `governance-team` then MS Entra will automatically removes the `john` from DataHub's `governance-team` group. + +If you permanently deletes a user or group from MS Entra then MS Entra will automatically deletes the user or group from the DataHub. + +> MS Entra doesn't send the user's password on user creation and hence DataHub Admin need to reset their password to be able to login into the DataHub. + +> Only Admin, Editor and Reader roles are supported in DataHub. These roles are preconfigured/created on DataHub + +## Configuring User/Group/Roles provisioning from MS Entra to DataHub + +1. **Generate Personal Access Token**: + Generate a personal access token from [DataHub](../../docs/authentication/personal-access-tokens.md#creating-personal-access-tokens). + +2. **Integrate DataHub With MS Entra**: Follow steps [Integrate your SCIM endpoint with the Microsoft Entra provisioning service](https://learn.microsoft.com/en-gb/entra/identity/app-provisioning/use-scim-to-provision-users-and-groups#integrate-your-scim-endpoint-with-the-microsoft-entra-provisioning-service) to integrate DataHub SCIM endpoint into MS Entra. + + a. Set the `Tenant URL` to `https:///gms/openapi/scim/v2`. Replace `` with your DataHub instance hostname. + + b. Set the `Secret Token` to Personal Access Token created in Step 1. + +3. **Update Attribute Mapping For Role**: + + a. Go to `Provisioning` section inside the App and click on `Provision Microsoft Entra ID Users` as shown in below image + +

+ +

+ + b. Click on `Add Mapping` + +

+ +

+ + c. Fill detail as shown in below image + + Fill listed fields + + - Set `Mapping type` to `Expression` + - Set `Expression` to `SingleAppRoleAssignment([appRoleAssignments])` + - Set `Target attribute` to `roles[primary eq "True"].value` + - Set `Match objects using this attribute` to `No` + - Set `Apply this mapping` to `Always` + +

+ +

+ + d. **Create Role**: Go back to the app created in Step #1 and go to the Provisioning section and click on application registration. to create the role + +

+ +

+ + Create three roles having `Display Name` and `Value` as mentioned below + + - Admin + - Editor + - Reader + + Only these three roles are supported in DataHub. + + e. While creating the App Role set `Allowed member types` to `Users/Groups` + +4. **Add Users/Groups/Roles in the App**: Go to application created in step #1 and click on `Add user/group` as shown in below image + +

+ +

+ + On the screen choose + + - Group/User + - And role for the Group/User. The role should be one of the role created in Step 3 diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-okta.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-okta.md new file mode 100644 index 00000000..296b45c2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/configuring-identity-provisioning-with-okta.md @@ -0,0 +1,117 @@ +--- +title: 'SCIM Integration: Okta and DataHub' +hide_title: true +sidebar_label: 'SCIM Integration: Okta and DataHub' +slug: /managed-datahub/configuring-identity-provisioning-with-okta +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/configuring-identity-provisioning-with-okta.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +## SCIM Integration: Okta and DataHub + + + +## Overview + +This document covers the steps required to enable SCIM provisioning from Okta to DataHub. + +This document assumes you are using OIDC for SSO with DataHub. +Since Okta doesn't currently support SCIM with OIDC, you would need to create an additional SWA-app-integration to enable SCIM provisioning. + +After completing this guide, Okta will automatically sync user and group changes from your SWA app integration to DataHub. This streamlines user and group provisioning in DataHub. + +Important notes about roles and permissions: + +- User and group roles from Okta are not transferred to DataHub +- When a group is first synced to DataHub, you can assign roles to that group directly in DataHub +- All users in the group will inherit the group's assigned roles +- You can also apply DataHub policies to groups, which will affect users based on their group membership + +### Why SCIM provisioning? + +Let us look at an example of the flows enabled through SCIM provisioning. + +Consider the following configuration in Okta + +- A group `governance-team` +- And it has two members `john` and `sid` + +Through SCIM provisioning, the following are enabled: + +- Okta can create a group `governance-team` in DataHub when you "Push Groups" in Okta. +- If the `governance-team` group is assigned to the DataHub app in Okta, Okta will create the users `john` and `sid` in DataHub. +- If you remove `john` from group `governance-team` then `john` would automatically get deactivated in DataHub. +- If you remove `sid` from the DataHub app in Okta, then `sid` would automatically get deactivated in DataHub. + +By default, groups and users synced from Okta to DataHub through this integration have no roles assigned. You can assign +roles to these groups directly within DataHub. + +Generally, any user assignment/unassignment to the app in Okta - directly or through groups - are automatically reflected in the DataHub application. + +## Configuring SCIM provisioning + +### 1. Create an SWA app integration + +a). Create a new [SWA app integration](https://help.okta.com/en-us/content/topics/apps/apps_app_integration_wizard_swa.htm), called say, `DataHub-SCIM-SWA`. + +Note: this app-integration will only be used for SCIM provisioning. You would continue to use the existing OIDC-app-integration for SSO. + +b). In the `General` tab of the `DataHub-SCIM-SWA` application, check the `Enable SCIM provisioning` option + +

+ +

+ +You may also want to configure the other selections as shown in the above image, so that this application isn't visible to your users. + +### 2. Configure SCIM + +a). Generate a personal access token from [DataHub](../../docs/authentication/personal-access-tokens.md#creating-personal-access-tokens). + +b). In the `Provisioning` tab, configure the DataHub-SCIM endpoint as shown in the below image: + +

+ +

+ +**Note**: Set the value of the `Bearer` field to the personal access token obtained in step (a) above. + +c). Configure the `To App` section as shown below: + +

+ +

+ +**Note**: We are not pushing passwords to DataHub over SCIM, since we are assuming SSO with OIDC as mentioned earlier. + +### 3. Assign users & groups to the app + +Assign users and groups to the app from the `Assignments` tab: + +

+ +

+ +### The provisioning setup is now complete + +Once the above steps are completed, user assignments/unassignments to the DataHub-SCIM-SWA app in Okta will get reflected in DataHub automatically. + +> #### A note on user deletion +> +> Note that when users are unassigned or deactivated in Okta, the corresponding users in DataHub are also deactivated (marked "suspended"). +> But when a user is _deleted_ in Okta, the corresponding user in DataHub does _not_ get deleted. +> Refer the Okta documentation on [Delete (Deprovision)](https://developer.okta.com/docs/concepts/scim/#delete-deprovision) for more details. + +### 5. (Recommended): Configure push groups + +When groups are assigned to the app, Okta pushes the group-members as users to DataHub, but the group itself isn't pushed. +To push group information to DataHub, configure the `Push Groups` tab accordingly as shown below: + +

+ +

+ +When you assign roles or policies to groups in DataHub, all users in those groups automatically inherit the same roles or policies +Refer to the Okta [Group Push](https://help.okta.com/en-us/content/topics/users-groups-profiles/app-assignments-group-push.htm) documentation for more details. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/entity-events-api.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/entity-events-api.md new file mode 100644 index 00000000..3af4f4e1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/entity-events-api.md @@ -0,0 +1,909 @@ +--- +description: >- + This guide details the Entity Events API, which allows you to take action when + things change on DataHub. +title: Entity Events API +slug: /managed-datahub/datahub-api/entity-events-api +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/datahub-api/entity-events-api.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Entity Events API + + + +## Introduction + +The Events API allows you to integrate changes happening on the DataHub Metadata Graph in real time into a broader event-based architecture. + +### Supported Integrations + +- [AWS EventBridge](docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md) +- [DataHub Cloud Event Source](docs/actions/sources/datahub-cloud-event-source.md) + +### Use Cases + +Real-time use cases broadly fall into the following categories: + +- **Workflow Integration:** Integrate DataHub flows into your organization's internal workflow management system. For example, create a Jira ticket when specific Tags or Terms are proposed on a Dataset. +- **Notifications**: Generate organization-specific notifications when a change is made on DataHub. For example, send an email to the governance team when a "PII" tag is added to any data asset. +- **Metadata Enrichment**: Trigger downstream metadata changes when an upstream change occurs. For example, propagating glossary terms or tags to downstream entities. +- **Synchronization**: Syncing changes made in DataHub into a 3rd party system. For example, reflecting Tag additions in DataHub into Snowflake. +- **Auditing:** Audit \*\*\*\* _who_ is making _what changes_ on DataHub through time. + +## Event Structure + +Each entity event is serialized to JSON & follows a common base structure. + +**Common Fields** + +| Name | Type | Description | Optional | +| -------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------- | +| **entityUrn** | String | The unique identifier for the Entity being changed. For example, a Dataset's urn. | Fals**e** | +| **entityType** | String | The type of the entity being changed. Supported values include `dataset`, `chart`, `dashboard`, `dataFlow (Pipeline)`, `dataJob` (Task), `domain`, `tag`, `glossaryTerm`, `corpGroup`, & `corpUser.` | False | +| **category** | String | The category of the change, related to the kind of operation that was performed. Examples include `TAG`, `GLOSSARY_TERM`, `DOMAIN`, `LIFECYCLE`, and more. | False | +| **operation** | String | The operation being performed on the entity given the category. For example, `ADD` ,`REMOVE`, `MODIFY`. For the set of valid operations, see the full catalog below. | False | +| **modifier** | String | The modifier that has been applied to the entity. The value depends on the category. An example includes the URN of a tag being applied to a Dataset or Schema Field. | True | +| **parameters** | Dict | Additional key-value parameters used to provide specific context. The precise contents depends on the category + operation of the event. See the catalog below for a full summary of the combinations. | True | +| **auditStamp.actor** | String | The urn of the actor who triggered the change. | False | +| **auditStamp.time** | Number | The timestamp in milliseconds corresponding to the event. | False | + +For example, an event indicating that a Tag has been added to a particular Dataset would populate each of these fields: + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TAG", + "operation": "ADD", + "modifier": "urn:li:tag:PII", + "parameters": { + "tagUrn": "urn:li:tag:PII" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +In the following sections, we'll take a closer look at the purpose and structure of each supported event type. + +## Event Types + +Below, we will review the catalog of events available for consumption. + +### Add Tag Event + +This event is emitted when a Tag has been added to an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
TAGADDdataset, dashboard, chart, dataJob, container, dataFlow , schemaField
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| tagUrn | String | The urn of the tag that has been added. | False | +| fieldPath | String | The path of the schema field which the tag is being added to. This field is **only** present if the entity type is `schemaField`. | True | +| parentUrn | String | The urn of a parent entity. This field is only present if the entity type is `schemaField`, and will contain the parent Dataset to which the field belongs. | True | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TAG", + "operation": "ADD", + "modifier": "urn:li:tag:PII" + "parameters": { + "tagUrn": "urn:li:tag:PII" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Tag Event + +This event is emitted when a Tag has been removed from an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
TAGREMOVEdataset, dashboard, chart, dataJob, container, dataFlow, schemaField
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| tagUrn | String | The urn of the tag that has been removed. | False | +| fieldPath | String | The path of the schema field which the tag is being removed from. This field is **only** present if the entity type is `schemaField`. | True | +| parentUrn | String | The urn of a parent entity. This field is only present if the entity type is `schemaField`, and will contain the parent Dataset to which the field belongs. | True | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TAG", + "operation": "REMOVE", + "modifier": "urn:li:tag:PII", + "parameters": { + "tagUrn": "urn:li:tag:PII" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Glossary Term Event + +This event is emitted when a Glossary Term has been added to an entity on DataHub. + +**Header** + +
CategoryOperationEntity Types
GLOSSARY_TERMADDdataset, dashboard, chart, dataJob, container, dataFlow , schemaField
+ +#### Parameters + +| Name | | Type | Description | Optional | +| --------- | --- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| termUrn | | String | The urn of the glossary term that has been added. | False | +| fieldPath | | String | The path of the schema field to which the term is being added. This field is **only** present if the entity type is `schemaField`. | True | +| parentUrn | | String | The urn of a parent entity. This field is only present if the entity type is `schemaField`, and will contain the parent Dataset to which the field belongs. | True | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "GLOSSARY_TERM", + "operation": "ADD", + "modifier": "urn:li:glossaryTerm:ExampleNode.ExampleTerm", + "parameters": { + "termUrn": "urn:li:glossaryTerm:ExampleNode.ExampleTerm" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Glossary Term Event + +This event is emitted when a Glossary Term has been removed from an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
GLOSSARY_TERMREMOVEdataset, dashboard, chart, dataJob, container, dataFlow , schemaField
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| termUrn | String | The urn of the glossary term that has been removed. | False | +| fieldPath | String | The path of the schema field from which the term is being removed. This field is **only** present if the entity type is `schemaField`. | True | +| parentUrn | String | The urn of a parent entity. This field is only present if the entity type is `schemaField`, and will contain the parent Dataset to which the field belongs. | True | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "GLOSSARY_TERM", + "operation": "REMOVE", + "modifier": "urn:li:glossaryTerm:ExampleNode.ExampleTerm", + "parameters": { + "termUrn": "urn:li:glossaryTerm:ExampleNode.ExampleTerm" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Domain Event + +This event is emitted when Domain has been added to an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
DOMAINADDdataset, dashboard, chart, dataJob, container, dataFlow
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------ | ------------------------------------------ | -------- | +| domainUrn | String | The urn of the domain that has been added. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DOMAIN", + "operation": "ADD", + "modifier": "urn:li:domain:ExampleDomain", + "parameters": { + "domainUrn": "urn:li:domain:ExampleDomain" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Domain Event + +This event is emitted when Domain has been removed from an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
DOMAINREMOVEdataset, dashboard, chart, dataJob, container ,dataFlow
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------ | -------------------------------------------- | -------- | +| domainUrn | String | The urn of the domain that has been removed. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DOMAIN", + "operation": "REMOVE", + "modifier": "urn:li:domain:ExampleDomain", + "parameters": { + "domainUrn": "urn:li:domain:ExampleDomain" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Owner Event + +This event is emitted when a new owner has been assigned to an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
OWNERADDdataset, dashboard, chart, dataJob, dataFlow , container, glossaryTerm, domain, tag
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------ | ------------------------------------------------------------------------------------------------------------ | -------- | +| ownerUrn | String | The urn of the owner that has been added. | False | +| ownerType | String | The type of the owner that has been added. `TECHNICAL_OWNER`, `BUSINESS_OWNER`, `DATA_STEWARD`, `NONE`, etc. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "OWNER", + "operation": "ADD", + "modifier": "urn:li:corpuser:jdoe", + "parameters": { + "ownerUrn": "urn:li:corpuser:jdoe", + "ownerType": "BUSINESS_OWNER" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Owner Event + +This event is emitted when an existing owner has been removed from an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
OWNERREMOVEdataset, dashboard, chart, dataJob, container ,dataFlow , glossaryTerm, domain, tag
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------ | -------------------------------------------------------------------------------------------------------------- | -------- | +| ownerUrn | String | The urn of the owner that has been removed. | False | +| ownerType | String | The type of the owner that has been removed. `TECHNICAL_OWNER`, `BUSINESS_OWNER`, `DATA_STEWARD`, `NONE`, etc. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "OWNER", + "operation": "REMOVE", + "modifier": "urn:li:corpuser:jdoe", + "parameters": { + "ownerUrn": "urn:li:corpuser:jdoe", + "ownerType": "BUSINESS_OWNER" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Description Event + +This event is emitted when a description has been added to an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
DOCUMENTATIONADDdataset, dashboard, chart, dataJob, dataFlow , container, glossaryTerm, domain, tag, schemaField
+ +#### Parameters + +| Name | Type | Description | Optional | +| ----------- | ------ | ------------------------------------ | -------- | +| description | String | The description that has been added. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DOCUMENTATION", + "operation": "ADD", + "parameters": { + "description": "This is a new description" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1706646452982 + } +} +``` + +### Remove Description Event + +This event is emitted when an existing description has been removed from an entity on DataHub. + +#### Header + +
CategoryOperationEntity Types
DOCUMENTATIONREMOVEdataset, dashboard, chart, dataJob, container ,dataFlow , glossaryTerm, domain, tag, schemaField
+ +#### Parameters + +| Name | Type | Description | Optional | +| ----------- | ------ | -------------------------------------- | -------- | +| description | String | The description that has been removed. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DOCUMENTATION", + "operation": "REMOVE", + "parameters": { + "description": "This is the removed description" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1706646452982 + } +} +``` + +### Modify Deprecation Event + +This event is emitted when the deprecation status of an entity has been modified on DataHub. + +#### Header + +
CategoryOperationEntity Types
DEPRECATIONMODIFYdataset, dashboard, chart, dataJob, dataFlow , container
+ +#### Parameters + +| Name | Type | Description | Optional | +| ------ | ------ | -------------------------------------------------------------------------- | -------- | +| status | String | The new deprecation status of the entity, either `DEPRECATED` or `ACTIVE`. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "DEPRECATION", + "operation": "MODIFY", + "modifier": "DEPRECATED", + "parameters": { + "status": "DEPRECATED" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Add Dataset Schema Field Event + +This event is emitted when a new field has been added to a **Dataset** **Schema**. + +#### Header + +
CategoryOperationEntity Types
TECHNICAL_SCHEMAADDdataset
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| fieldUrn | String | The urn of the new schema field. | False | +| fieldPath | String | The path of the new field. For more information about field paths, check out [Dataset Field Paths Explained](docs/generated/metamodel/entities/dataset.md#field-paths-explained) | False | +| nullable | Boolean | Whether the new field is nullable. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TECHNICAL_SCHEMA", + "operation": "ADD", + "modifier": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "parameters": { + "fieldUrn": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "fieldPath": "newFieldName", + "nullable": false + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Remove Dataset Schema Field Event + +This event is emitted when a new field has been remove from a **Dataset** **Schema**. + +#### Header + +
CategoryOperationEntity Types
TECHNICAL_SCHEMAREMOVEdataset
+ +#### Parameters + +| Name | Type | Description | Optional | +| --------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | +| fieldUrn | String | The urn of the removed schema field. | False | +| fieldPath | String | The path of the removed field. For more information about field paths, check out [Dataset Field Paths Explained](docs/generated/metamodel/entities/dataset.md#field-paths-explained) | False | +| nullable | Boolean | Whether the removed field is nullable. | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TECHNICAL_SCHEMA", + "operation": "REMOVE", + "modifier": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "parameters": { + "fieldUrn": "urn:li:schemaField:(urn:li:dataset:abc,newFieldName)", + "fieldPath": "newFieldName", + "nullable": false + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Entity Create Event + +This event is emitted when a new entity has been created on DataHub. + +#### Header + +
CategoryOperationEntity Types
LIFECYCLECREATEdataset, dashboard, chart, dataJob, dataFlow , glossaryTerm, domain, tag, container
+ +#### Parameters + +_None_ + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "LIFECYCLE", + "operation": "CREATE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Entity Soft-Delete Event + +This event is emitted when a new entity has been soft-deleted on DataHub. + +#### Header + +
CategoryOperationEntity Types
LIFECYCLESOFT_DELETEdataset, dashboard, chart, dataJob, dataFlow , glossaryTerm, domain, tag, container
+ +#### Parameters + +_None_ + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "LIFECYCLE", + "operation": "SOFT_DELETE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Entity Hard-Delete Event + +This event is emitted when a new entity has been hard-deleted on DataHub. + +#### Header + +
CategoryOperationEntity Types
LIFECYCLEHARD_DELETEdataset, dashboard, chart, dataJob, dataFlow , glossaryTerm, domain, tag, container
+ +#### Parameters + +_None_ + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "LIFECYCLE", + "operation": "HARD_DELETE", + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Completed Assertion Run Event + +This event is emitted when an Assertion has been run has succeeded on DataHub. + +#### Header + +
CategoryOperationEntity Types
RUNCOMPLETEDassertion
+ +#### Parameters + +| Name | Type | Description | Optional | +| ---------- | ------ | ----------------------------------------------------- | -------- | +| runResult | String | The result of the run, either `SUCCESS` or `FAILURE`. | False | +| runId | String | Native (platform-specific) identifier for this run. | False | +| aserteeUrn | String | Urn of entity on which the assertion is applicable. | False | + +#### + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:assertion:abc", + "entityType": "assertion", + "category": "RUN", + "operation": "COMPLETED", + "parameters": { + "runResult": "SUCCESS", + "runId": "123", + "asserteeUrn": "urn:li:dataset:def" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Started Data Process Instance Run Event + +This event is emitted when a Data Process Instance Run has STARTED on DataHub. + +#### Header + +
CategoryOperationEntity Types
RUNSTARTEDdataProcessInstance
+ +#### Parameters + +| Name | Type | Description | Optional | +| ----------------- | ------- | ----------------------------------------------------------------------------------------------- | -------- | +| attempt | Integer | The number of attempts that have been made. | True | +| dataFlowUrn | String | The urn of the associated Data Flow. Only filled in if this run is associated with a Data Flow. | True | +| dataJobUrn | String | The urn of the associated Data Flow. Only filled in if this run is associated with a Data Job. | True | +| parentInstanceUrn | String | Urn of the parent DataProcessInstance (if there is one). | True | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataProcessInstance:abc", + "entityType": "dataProcessInstance", + "category": "RUN", + "operation": "STARTED", + "parameters": { + "dataFlowUrn": "urn:li:dataFlow:def", + "attempt": "1", + "parentInstanceUrn": ""urn:li:dataProcessInstance:ghi" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Completed Data Process Instance Run Event + +This event is emitted when a Data Process Instance Run has been COMPLETED on DataHub. + +#### Header + +
CategoryOperationEntity Types
RUNCOMPLETEDdataProcessInstance
+ +#### Parameters + +| Name | Type | Description | Optional | +| ----------------- | ------- | ----------------------------------------------------------------------------------------------- | -------- | +| runResult | String | The result of the run, one of `SUCCESS` , `FAILURE`, `SKIPPED`, or `UP_FOR_RETRY` . | False | +| attempt | Integer | The number of attempts that have been made. | True | +| dataFlowUrn | String | The urn of the associated Data Flow. Only filled in if this run is associated with a Data Flow. | True | +| dataJobUrn | String | The urn of the associated Data Flow. Only filled in if this run is associated with a Data Job. | True | +| parentInstanceUrn | String | Urn of the parent DataProcessInstance. | True | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:dataProcessInstance:abc", + "entityType": "dataProcessInstance", + "category": "RUN", + "operation": "COMPLETED", + "parameters": { + "runResult": "SUCCESS" + "attempt": "2", + "dataFlowUrn": "urn:li:dataFlow:def", + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Action Request Created Event + +This event is emitted when a new Action Request (Metadata Proposal) has been created. + +#### Header + +
CategoryOperationEntity Types
LIFECYCLECREATEDactionRequest
+ +#### Parameters + +These are the common parameters for all Action Request create events. + +| Name | Type | Description | Optional | +| ----------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| actionRequestType | String | The type of Action Request. One of `TAG_ASSOCIATION`, `TERM_ASSOCIATION`, `CREATE_GLOSSARY_NODE`, `CREATE_GLOSSARY_TERM`, or `UPDATE_DESCRIPTION.` | False | +| resourceType | String | The type of entity this Action Request is applied on, such as `dataset`. | True | +| resourceUrn | String | The entity this Action Request is applied on. | True | +| subResourceType | String | Filled if this Action Request is applied on a sub-resource, such as a `schemaField`. | True | +| subResource | String | Identifier of the sub-resource if this proposal is applied on one. | True | + +Parameters specific to different proposal types are listed below. + +#### Tag Association Proposal Specific Parameters and Sample Event + +| Name | Type | Description | Optional | +| ------ | ------ | ----------------------------------------- | -------- | +| tagUrn | String | The urn of the Tag that would be applied. | False | + +``` +{ + "entityUrn": "urn:li:actionRequest:abc", + "entityType": "actionRequest", + "category": "LIFECYCLE", + "operation": "CREATED", + "parameters": { + "actionRequestType": "TAG_ASSOCIATION", + "resourceType": "dataset", + "resourceUrn": "urn:li:dataset:snowflakeDataset, + "tagUrn": "urn:li:tag:Classification" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +#### Term Association Proposal Specific Parameters and Sample Event + +| Name | Type | Description | Optional | +| ------- | ------ | --------------------------------------------------- | -------- | +| termUrn | String | The urn of the Glossary Term that would be applied. | False | + +``` +{ + "entityUrn": "urn:li:actionRequest:abc", + "entityType": "actionRequest", + "category": "LIFECYCLE", + "operation": "CREATED", + "parameters": { + "actionRequestType": "TERM_ASSOCIATION", + "resourceType": "dataset", + "resourceUrn": "urn:li:dataset:snowflakeDataset, + "termUrn": "urn:li:glossaryTerm:Classification" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +#### Create Glossary Node/Term Proposal Specific Parameters and Sample Event + +| Name | Type | Description | Optional | +| ------------------ | ------ | --------------------------------------------------------------------------------- | -------- | +| glossaryEntityName | String | The name of the Glossary Entity that would be created. | False | +| parentNodeUrn | String | The urn of the Parent Node that would be associated with the new Glossary Entity. | True | +| description | String | The description of the new Glossary Entity. | True | + +``` +{ + "entityUrn": "urn:li:actionRequest:abc", + "entityType": "actionRequest", + "category": "LIFECYCLE", + "operation": "CREATED", + "parameters": { + "actionRequestType": "CREATE_GLOSSARY_TERM", + "resourceType": "glossaryNode", + "glossaryEntityName": "PII", + "parentNodeUrn": "urn:li:glossaryNode:Classification", + "description": "Personally Identifiable Information" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +#### Update Description Proposal Specific Parameters + +| Name | Type | Description | Optional | +| ----------- | ------ | --------------------------------- | -------- | +| description | String | The proposed updated description. | False | + +``` +{ + "entityUrn": "urn:li:actionRequest:abc", + "entityType": "actionRequest", + "category": "LIFECYCLE", + "operation": "CREATED", + "parameters": { + "actionRequestType": "UPDATE_DESCRIPTION", + "resourceType": "glossaryNode", + "description": "Personally Identifiable Information" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Action Request Status Change Event + +This event is emitted when an existing Action Request (proposal) changes status. For example, this event will be emitted when an Action Request transitions from pending to completed. + +#### Header + +
CategoryOperationEntity Types
LIFECYCLEPENDING, COMPLETEDactionRequest
+ +#### Parameters + +These are the common parameters for all parameters. + +| Name | Type | Description | Optional | +| ------------------- | ------ | ----------------------------------------------------------------------------------------- | -------- | +| actionRequestStatus | String | The status of the Action Request. | False | +| actionRequestResult | String | Only filled if the `actionRequestStatus` is `COMPLETED`. Either `ACCEPTED` or `REJECTED`. | True | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:actionRequest:abc", + "entityType": "actionRequest", + "category": "LIFECYCLE", + "operation": "COMPLETED", + "parameters": { + "actionRequestStatus": "COMPLETED", + "actionRequestResult": "ACCEPTED" + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` + +### Incident Change Event + +This event is emitted when an Incident has been created or it's status changes. + +#### Header + +
CategoryOperationEntity Types
INCIDENTACTIVE, RESOLVEDincident
+ +#### Parameters + +| Name | Type | Description | Optional | +| -------- | ------ | ------------------------------------------------- | -------- | +| entities | String | The list of entities associated with the incident | False | + +#### Sample Event + +``` +{ + "entityUrn": "urn:li:incident:16ff200a-0ac5-4a7d-bbab-d4bdb4f831f9", + "entityType": "incident", + "category": "INCIDENT", + "operation": "ACTIVE", + "parameters": { + "entities": "[urn:li:dataset:abc, urn:li:dataset:abc2]", + }, + "auditStamp": { + "actor": "urn:li:corpuser:jdoe", + "time": 1649953100653 + } +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/graphql-api/getting-started.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/graphql-api/getting-started.md new file mode 100644 index 00000000..59c79a64 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/datahub-api/graphql-api/getting-started.md @@ -0,0 +1,48 @@ +--- +description: Getting started with the DataHub GraphQL API. +title: Getting Started +slug: /managed-datahub/datahub-api/graphql-api/getting-started +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/datahub-api/graphql-api/getting-started.md +--- + +# Getting Started + +The DataHub Cloud GraphQL API is an extension of the open source [DataHub GraphQL API.](docs/api/graphql/overview.md) + +For a full reference to the Queries & Mutations available for consumption, check out [Queries](graphql/queries.md) & [Mutations](graphql/mutations.md). + +### Connecting to the API + +

+ +

+ +When you generate the token you will see an example of `curl` command which you can use to connect to the GraphQL API. + +Note that there is a single URL mentioned there but it can be any of these + +- https://`your-account`.acryl.io/api/graphql +- https://`your-account`.acryl.io/api/gms/graphql + +If there is any example that requires you to connect to GMS then you can use the second URL and change the endpoints. + +e.g. to get configuration of your GMS server you can use + +``` +curl -X GET 'https://your-account.acryl.io/api/gms/config' --header +``` + +e.g. to connect to ingestion endpoint for doing ingestion programmatically you can use the below URL + +- https://your-account.acryl.io/api/gms/aspects?action=ingestProposal + +### Exploring the API + +The entire GraphQL API can be explored & [introspected](https://graphql.org/learn/introspection/) using GraphiQL, an interactive query tool which allows you to navigate the entire DataHub Cloud GraphQL schema as well as craft & issue using an intuitive UI. + +[GraphiQL](https://www.gatsbyjs.com/docs/how-to/querying-data/running-queries-with-graphiql/) is available for each DataHub Cloud deployment, locating at `https://your-account.acryl.io/api/graphiql`. + +### Querying the API + +Currently, we do not offer language-specific SDKs for accessing the DataHub GraphQL API. For querying the API, you can make use of a variety of per-language client libraries. For a full list, see [GraphQL Code Libraries, Tools, & Services](https://graphql.org/code/). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/aws-privatelink.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/aws-privatelink.md new file mode 100644 index 00000000..521902fa --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/aws-privatelink.md @@ -0,0 +1,38 @@ +--- +title: AWS PrivateLink +slug: /managed-datahub/integrations/aws-privatelink +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/integrations/aws-privatelink.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# AWS PrivateLink + + + +If you require a private connection between the provisioned DataHub instance and your own existing AWS account, DataHub Cloud supports using AWS PrivateLink in order to complete this private connection. + +In order to complete this connection, the DataHub integrations team will require the AWS ARN for a user or role that can accept and complete the connection to your AWS account. + +Once that team reports the PrivateLink has been created, the team will give you a VPC Endpoint Service Name to use. + +In order to complete the connection, you will have to create a VPC Endpoint in your AWS account. To do so, please follow these instructions: + +:::info +Before following the instructions below, please create a VPC security group with ports 80, and 443 (Both TCP) and any required CIDR blocks or other sources as an inbound rule +::: + +1. Open the AWS console to the region that the VPC Endpoint Service is created (Generally this will be in `us-west-2 (Oregon)` but will be seen in the service name itself) +2. Browse to the **VPC** Service and click on **Endpoints** +3. Click on **Create Endpoint** in the top right corner +4. Give the endpoint a name tag (such as _datahub-pl_) +5. Click on the **Other endpoint services** radio button +6. In the **Service setting**, copy the service name that was given to you by the integrations team into the **Service name** field and click **Verify Service** +7. Now select the VPC from the dropdown menu where the endpoint will be created. +8. A list of availability zones will now be shown in the **Subnets** section. Please select at least 1 availability zone and then a corresponding subnet ID from the drop down menu to the right of that AZ. +9. Choose **IPv4** for the **IP address type** +10. Choose an existing security group (or multiple) to use on this endpoint +11. (Optional) For **Policy,** you can keep it on **Full access** or **custom** if you have specific access requirements +12. (Optional) Create any tags you wish to add to this endpoint +13. Click **Create endpoint** +14. Once it has been created, DataHub Cloud will need to accept the incoming connection from your AWS account; the integrations team will advise you when this has been completed. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/oidc-sso-integration.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/oidc-sso-integration.md new file mode 100644 index 00000000..f147b6d7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/integrations/oidc-sso-integration.md @@ -0,0 +1,50 @@ +--- +description: >- + This page will help you set up OIDC SSO with your identity provider to log + into DataHub +title: Enable OIDC SSO +slug: /managed-datahub/integrations/oidc-sso-integration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/integrations/oidc-sso-integration.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Enable OIDC SSO + + + +This guide will walk you through configuring OIDC Single Sign-On in DataHub Cloud. + +### Step 1. Complete OIDC Prerequisites + +You will need the following in order to enable OIDC SSO in DataHub Cloud: + +- Gather the **Client ID**, **Client Secret**, and **Discovery URI** for your OIDC provider, as detailed in [this guide](../../authentication/guides/sso/initialize-oidc.md). +- Confirm you have the `Manage Platform Settings` privilege in DataHub. + +### Step 2. Enable OIDC SSO + +1. In DataHub Cloud, navigate to **Settings > Platform > SSO** and choose **OIDC**. + +

+ +

+ +2. Enter the **Client ID**, **Client Secret**, and **Discovery URI** from Step 1. +3. Confirm your preferred **User Provisioning Strategy**: + + - **Just-in-Time (JIT) Provisioning** is enabled by default, automatically creating a DataHub User on login if one does not exist. + - **Pre-Provisioning DataHub Users** will only allow login for pre-provisioned DataHub Users. _Requires configuring SSO Ingestion._ + +4. Optionally enable **Extract Groups** to extract group memberships in the OIDC profile by default. _Requires JIT Provisioning._ +5. Click **Connect**. +6. Log out and log back in through SSO to confirm connection succeeded. + +

+ +

+ +:::note +We do not yet support LDAP or SAML authentication. Please let us know if either of these integrations would be useful for your organization. +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/managed-datahub-overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/managed-datahub-overview.md new file mode 100644 index 00000000..747f6356 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/managed-datahub-overview.md @@ -0,0 +1,77 @@ +--- +title: 'OSS vs Cloud: Comparison Guide' +sidebar_label: 'OSS vs Cloud: Comparison Guide' +slug: /managed-datahub/managed-datahub-overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/managed-datahub-overview.md +--- + +# OSS vs Cloud: Comparison Guide + +This guide compares DataHub Open Source (OSS) and DataHub Cloud features and platform differences. DataHub Cloud builds on the OSS foundation with enterprise-grade capabilities including AI automation, advanced governance, operational reliability, and production support for mid-to-large organizations. Cloud also offers a fully managed service with 99.5%+ SLA-backed availability, dedicated support, enhanced security, training services, and flexible deployment options. + +## Discovery & Search + +| Feature Name | OSS | Cloud | Business Value | Link | +| :-------------------------------------------- | :-: | :---: | :-------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------: | +| **70+ Source Connectors with Unified Search** | ✔ | ✔ | Connect entire data ecosystem | [Docs](/integrations) | +| **Ask DataHub AI Agent** | ❌ | ✔ |
  • Find trustworthy data metrics
  • Generate Accurate SQL
  • Debug data quality issues
  • Understand impact of data changes
| [Docs](/docs/features/feature-guides/ask-datahub) | +| **DataHub Hosted MCP Server** | ❌ | ✔ | Connect AI tools directly to your data catalog | [Docs](/docs/features/feature-guides/mcp) | +| **Enhanced Usage-Aware Search Ranking** | ❌ | ✔ | Surface most relevant data first | [Docs](/docs/how/search#example-1-ranking-by-tagsterms) | +| **Column-Level Lineage & Impact Analysis** | ✔ | ✔ | Understand data dependencies | [Docs](/docs/features/feature-guides/lineage) | +| **Lineage-Based Propagation** | ❌ | ✔ | Auto-enrich downstream datasets | [Docs](/docs/automations/docs-propagation#introduction) | +| **Context Documents** | ✔ | ✔ | Create & semantically search across unstructured docs | [Docs](/docs/features/feature-guides/context/context-documents) | +| **AI Documentation Generation** | ❌ | ✔ | Auto-document tables & columns | [Docs](/docs/automations/ai-docs) | +| **Personalized Home and Asset Views** | ❌ | ✔ | Customize home page and asset summaries for a personalized data experience | [Docs](/docs/features/feature-guides/custom-asset-summaries#custom-modules) | +| **Multi-Channel Notifications** | ❌ | ✔ | Stay informed where you work (Email, Slack, & Teams) | [Docs](/docs/incidents/incidents/#enabling-slack-notifications-datahub-cloud-only) | + +## Data Observability + +| Feature Name | OSS | Cloud | Business Value | Link | +| :------------------------------------------------------------------- | :-: | :---: | :------------------------------------------------------------ | :-------------------------------------------------------------------------------------------------: | +| **Quality & Health Status on Asset Profiles** | ✔ | ✔ | See quality at a glance | | +| **AI Anomaly Detection (Smart Assertions)** | ❌ | ✔ | Catch issues automatically | [Docs](/docs/managed-datahub/observe/smart-assertions) | +| **Freshness, Volume, Schema & Column Monitoring, Custom SQL Checks** | ❌ | ✔ | Ensure timely data | [Docs](/docs/managed-datahub/observe/freshness-assertions) | +| **Data Contracts** | ✔ | ✔ | Define quality expectations | [Docs](/docs/managed-datahub/observe/data-contract#what-is-a-data-contract) | +| **Data Health Dashboard** | ❌ | ✔ | Quality overview at scale | [Docs](/docs/managed-datahub/observe/data-health-dashboard) | +| **Notifications for Data Assertions** | ❌ | ✔ | Real-time quality alerts | [Docs](/docs/managed-datahub/subscription-and-notification) | +| **Secure In-VPC Quality Validation** | ❌ | ✔ | Metadata never leaves your network | | +| **Pipeline Circuit Breakers (API)** | ❌ | ✔ | Validate data quality programmatically before reads or writes | [Docs](/docs/managed-datahub/observe/data-contract#api) | + +## Data Governance + +| Feature Name | OSS | Cloud | Business Value | Link | +| :------------------------------------------------------------------------------- | :-: | :---: | :---------------------------- | :------------------------------------------------------------------------------------------------------: | +| **Data Ownership Management** | ✔ | ✔ | Clear accountability | [Docs](/docs/metadata-integration/java/docs/sdk-v2/dataset-entity#owners) | +| **Business Glossary** | ✔ | ✔ | Common data language | [Docs](/learn/business-glossary) | +| **AI Data Classification** | ❌ | ✔ | Auto-tag sensitive data | [Docs](/docs/automations/ai-term-suggestion) | +| **Bi-Directional Metadata Sync** | ❌ | ✔ | Keep metadata current | [Docs](/docs/automations/bigquery-metadata-sync) | +| **Compliance Forms and Workflow Engine** | ❌ | ✔ | Track regulatory compliance | [Docs](/docs/features/feature-guides/compliance-forms/analytics) | +| **Metadata Tests** | ❌ | ✔ | Validate governance rules | [Docs](/docs/tests/metadata-tests) | +| **Approval Workflows: Documentation, Glossary, Tags, Terms, and Data Ownership** | ❌ | ✔ | Controlled vocabulary changes | [Docs](/docs/managed-datahub/change-proposals#proposing-tags-and-glossary-terms) | +| **Access Request Workflows** | ❌ | ✔ | Self-service data access | [Docs](/docs/managed-datahub/workflows/access-workflows#faq-and-troubleshooting) | + +## Enterprise & Security + +| Feature Name | OSS Available | Cloud Available | Business Value | +| :-------------------------------- | :-----------: | :-------------: | :---------------------- | +| **99.5% Uptime SLA** | ❌ | ✔ | Guaranteed availability | +| **Fine-grained Access Control** | ❌ | ✔ | Secure by default | +| **AWS PrivateLink Support** | ❌ | ✔ | Network isolation | +| **IP Address Restrictions** | ❌ | ✔ | Access control | +| **In-VPC Remote Ingestion Agent** | ❌ | ✔ | Data security control | + +## Implementation & Support + +| Feature Name | OSS Available | Cloud Available | Business Value | +| :------------------------------------- | :-----------: | :-------------: | :----------------------------------------------------------- | +| **Fully Managed Cloud Deployment** | ❌ | ✔ | Zero maintenance cloud-hosted instance | +| **Dedicated Customer Success** | ❌ | ✔ | Expert guidance | +| **Guided Implementation & Onboarding** | ❌ | ✔ | Smooth rollout | +| **Private Slack Support Channel** | ❌ | ✔ | Direct access to experts | +| **Community Support** | ✔ | ✔ | Peer assistance | +| **OSS Contribution Fast-Track** | ❌ | ✔ | Community Contribution Support to DataHub Apache 2.0 Project | + + +See DataHub Cloud In Action + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-backfill.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-backfill.md new file mode 100644 index 00000000..9390a89c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-backfill.md @@ -0,0 +1,196 @@ +--- +description: >- + This page provides an overview of Assertion Backfill (Historical Data + Bootstrapping) +title: Backfill Assertion History +slug: /managed-datahub/observe/assertion-backfill +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/assertion-backfill.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Backfill Assertion History + + + +> The **Backfill Assertion History** feature is available as part of the **DataHub Cloud Observe** module of DataHub Cloud. +> If you are interested in learning more about **DataHub Cloud Observe** or trying it out, please [visit our website](https://datahub.com/products/data-observability/). + +
+ +## Introduction + +When you create a new [Smart Assertion](./smart-assertions.md), it needs historical data to learn what "normal" looks like before it can start making accurate predictions. Without historical context, the assertion's AI model has nothing to train on, meaning it will take days or weeks of real-time evaluations before it can reliably detect anomalies. + +**Backfill Assertion History** solves this by running the assertion against historical data at the time of creation. Instead of waiting for the model to accumulate enough data points through scheduled evaluations, the system queries your warehouse for past data and populates the assertion's metrics history in one go. This means you get accurate anomaly detection thresholds from day one, with full awareness of daily, weekly, or monthly seasonality in your data. + +Backfill is available for the following assertion types: + +| Assertion Type | Backfill Support | +| --------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| **Smart Volume Assertion** | Yes (requires [time-series bucketing](./volume-assertions.md#time-series-bucketing)) | +| **Smart Column Metric Assertion** | Yes (requires [time-series bucketing](./column-assertions.md#time-series-bucketing-for-column-metric-assertions)) | +| **Freshness Assertion** | No | +| **Schema Assertion** | No | +| **Custom SQL Assertion** | No | + +## How Backfill Works + +When you create a bucketed assertion with backfill enabled, the following process occurs: + +1. **Job creation**: A backfill job is created in `PENDING` state and queued for execution. +2. **Job scheduling**: A background fetcher runs every few minutes, picks up pending backfill jobs, and submits them for execution. To avoid overloading your warehouse, a maximum of 4 backfill jobs run concurrently. +3. **Chunked execution**: The backfill queries your warehouse in chunks (approximately one month of data per chunk) using efficient `GROUP BY` queries. This balances query cost against resilience — if a single chunk fails, only that chunk needs to be retried rather than the entire backfill. +4. **Progress tracking**: After each chunk completes, progress is recorded (percentage complete and the last evaluated bucket). If the job is interrupted, it will resume from where it left off. +5. **Completion**: Once all historical buckets are populated, the assertion's AI model trains on the backfilled data and begins generating predictions. + +### Backfill Limits + +The maximum amount of historical data that can be backfilled depends on the bucket interval: + +| Bucket Interval | Maximum Lookback | +| --------------- | ------------------- | +| **Daily** | 365 days (1 year) | +| **Weekly** | 156 weeks (3 years) | + +This is lookback window is relative to the assertion's creation date + +### Backfill Statuses + +You can track the progress of a backfill from the assertion detail page. A backfill job will be in one of the following states: + +- **Pending**: The backfill job is queued and waiting to be picked up by the executor. +- **Submitted**: The job is about to start. +- **Running**: The backfill queries are actively executing. +- **Complete**: The backfill finished successfully. +- **Failed**: The backfill encountered an error. You can retry the backfill (see below). +- **Rejected**: The backfill was not attempted because it did not meet eligibility requirements (e.g., the assertion type does not support backfill) or backfill was disabled. + +:::info +Backfilling large tables (> 1 TB) can be expensive in terms of warehouse compute. Consider starting with a shorter lookback period and extending it if needed. +::: + +## Configuring Backfill + +### Via the UI + +When creating a new smart assertion with [time-series bucketing](./volume-assertions.md#time-series-bucketing) enabled: + +1. Toggle **Backfill historical data** to on (this defaults to on when bucketing is enabled for smart assertions). +2. Select a **backfill start date** using the date picker. The date picker enforces the maximum lookback constraints (365 days for daily, 156 weeks for weekly bucketing). +3. Complete the rest of the assertion configuration and click **Save**. + +

+ +

+ +After creation, you can update the backfill start date, but you cannot change the bucketing configuration (timestamp column, bucket interval, or timezone). + +### Via the Python SDK + +You can configure backfill using the `backfill_config` parameter on the `sync_smart_volume_assertion` and `sync_smart_column_metric_assertion` methods. + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +client = DataHubClient(server="", token="") +dataset_urn = DatasetUrn.from_string( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)" +) + +# Smart volume assertion with daily bucketing and 6-month backfill +assertion = client.assertions.sync_smart_volume_assertion( + dataset_urn=dataset_urn, + display_name="Daily Volume Anomaly Monitor", + detection_mechanism="information_schema", + sensitivity="medium", + time_bucketing_strategy={ + "timestamp_field_path": "created_at", + "bucket_interval": {"unit": "DAY", "multiple": 1}, + "timezone": "America/Los_Angeles", + }, + backfill_config={ + "backfill_start_date_ms": 1688169600000, # 2023-07-01T00:00:00Z + }, + tags=["automated", "volume"], + enabled=True, +) +``` + +The `backfill_config` parameter accepts: + +- A dict with `backfill_start_date_ms` (epoch milliseconds) +- A `BackfillConfig` Pydantic model (supports `datetime` objects) +- A raw `AssertionMonitorBootstrapConfigClass` GMS model + +```python +from datetime import datetime +from acryl_datahub_cloud.sdk import BackfillConfig + +# Using a BackfillConfig with a datetime object +backfill = BackfillConfig(backfill_start_date_ms=datetime(2024, 1, 1)) + +assertion = client.assertions.sync_smart_column_metric_assertion( + dataset_urn=dataset_urn, + column_name="user_id", + metric_type="null_count", + display_name="Smart Null Count - user_id", + detection_mechanism="all_rows_query_datahub_dataset_profile", + sensitivity="medium", + time_bucketing_strategy={ + "timestamp_field_path": "created_at", + "bucket_interval": {"unit": "WEEK", "multiple": 1}, + }, + backfill_config=backfill, + enabled=True, +) +``` + +:::note +`backfill_config` requires `time_bucketing_strategy` to also be set. If you provide `backfill_config` without `time_bucketing_strategy` on a column metric assertion the configuration will be rejected, and the assertion will not be created. +::: + +## Retrying a Failed Backfill + +If a backfill fails (due to a warehouse timeout, network error, etc.), you can retry it from the assertion detail page. There are two retry modes: + +- **Soft retry** (default): Resumes the backfill from the last successfully evaluated bucket. Only the remaining gaps are filled in. +- **Hard reset**: Restarts the backfill from scratch, re-evaluating all buckets and overwriting any previously collected metrics. Use this if you suspect the existing backfilled data is incorrect. This currently only available via the GraphQL API. + +### Via the UI + +Navigate to the assertion detail page. The backfill status will appear near the top of the page, alongside the error encountered. Click **Retry**. + +### Via GraphQL + +```graphql +mutation retryMonitorBackfill { + retryMonitorBackfill( + input: { monitorUrn: "urn:li:monitor:your-monitor-id", hardReset: false } + ) +} +``` + +Set `hardReset: true` to perform a full re-backfill from scratch. This is useful if you recently ran a job that added/updated entries and backdated them. + +## Prerequisites + +- **Remote Executor**: Backfill requires the Remote Executor at version **v0.3.17-acryl** or later. +- **Warehouse connection**: An Ingestion Source must be configured for your data platform (Snowflake, BigQuery, Redshift, or Databricks) under the **Integrations** tab. +- **Permissions**: The actor must have `Edit Assertions` and `Edit Monitors` privileges for the target dataset. + +## FAQ + +**Q: Does backfill run queries against my warehouse?** +Yes. For bucketed assertions, the backfill process issues `GROUP BY` queries against your warehouse to compute historical metrics. Queries are batched in chunks (approximately 28 days per chunk) to balance cost and resilience. + +**Q: Can I change the backfill start date after creation?** +Yes. The backfill start date can be updated after creation. However, the corresponding bucketing parameters (timestamp column, bucket interval, timezone) cannot be changed without recreating the assertion. + +**Q: What happens if my warehouse goes down during a backfill?** +The backfill will fail and can be retried. Because progress is tracked per-chunk, a soft retry will resume from the last successful chunk rather than starting over. + +**Q: Does backfill affect my scheduled assertion evaluations?** +No. Backfill runs do not interfere with your normal assertion evaluation schedule. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-notes.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-notes.md new file mode 100644 index 00000000..1e6e4fe3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-notes.md @@ -0,0 +1,37 @@ +--- +description: This page provides an overview of using Assertion Notes +title: Assertion Notes +slug: /managed-datahub/observe/assertion-notes +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/assertion-notes.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Assertion Notes + + + +> The **Assertion Notes** feature is available as part of the **DataHub Cloud Observe** module of DataHub Cloud. +> If you are interested in learning more about **DataHub Cloud Observe** or trying it out, please [visit our website](https://datahub.com/products/data-observability/). + +## Introduction + +The Assertion notes feature aims to solve two key use cases: + +1. Surfacing useful tips for engineers to troubleshoot and resolve data quality failures +2. Documenting the purpose of a given check, and implications of its failiure; for instance, some checks may circuit-break pipelines. + +### For Troubleshooting + +As you scale your data quality coverage across a large data landscape, you will often find that the engineers who are troubleshooting and resolving an assertion failure are not the same people who created the check. +Oftentimes, it's useful to provide troubleshooting instructions or notes with context about how to resolve the problem when a check fails. + +- If the check was manually set up, it may be worthwhile for the creator to add notes for future on-call engineers +- If it was an AI check, whoever is first to investigate the failure may want to document what they did to fix it. + +### For Documenting + +Adding notes to Assertions is useful for documenting your Assertions. This is particularly relevant for Custom SQL checks, where understanding the logic from the query statements can be difficult. By adding documentation in the notes tab, others can understand exactly what is being monitored and how to resolve issues in event of failure. + + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-query-attribution.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-query-attribution.md new file mode 100644 index 00000000..0104b62a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertion-query-attribution.md @@ -0,0 +1,41 @@ +--- +description: Learn how to attribute queries to DataHub Cloud Observe to better track costs +title: Assertion Query Attribution +slug: /managed-datahub/observe/assertion-query-attribution +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/assertion-query-attribution.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Assertion Query Attribution + + + +For certain Assertions, like Freshness and Volume, DataHub will issue queries against your Data Warehouse (ex. Snowflake) to determine if that assertion passed or failed. This can result in many additional queries against your warehouse every day, depending on how many assertions you have set up. In order to help you track and understand all of the queries coming from DataHub Cloud Observe, tagging has been added to issued queries. + +## SQL Comments + +For all platforms, a SQL comment is added to the top of all queries indicating the query source is DataHub and also including the URN of the assertion that issued the query. For example: + +```sql +/* query_source=datahub_observe assertion_urn=urn:li:assertion:507e3dec-8fed-4809-9cdd-cf2a4a06a249 */ +SELECT * +FROM users +``` + +## Snowflake Query Tag + +For queries issued against Snowflake, a [Snowflake Query Tag](https://select.dev/posts/snowflake-query-tags) is added to the SQL statement. + +```sql +ALTER SESSION SET query_tag='{"query_source": "datahub_observe", "assertion_urn": "urn:li:assertion:507e3dec-8fed-4809-9cdd-cf2a4a06a249"}' +``` + +## BigQuery Job Labels + +BigQuery support attribution through [job labels](https://docs.cloud.google.com/bigquery/docs/adding-labels#job-label) which are automatically included in your billing data. Unfortunately, due to length and character limits on labels, the assertion URN is not included in the job label. + +```python +labels = {"datahub_observe": "true"} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertions.md new file mode 100644 index 00000000..7967a148 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/assertions.md @@ -0,0 +1,113 @@ +--- +title: Assertions +slug: /managed-datahub/observe/assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/assertions.md +--- +# Assertions + +:::note Supported Data Platforms +Currently we support monitoring data on Snowflake, Redshift, BigQuery, and Databricks as part of DataHub Cloud Observe. +DataHub Cloud Observe can still monitor assertions for other data platforms against dataset metrics (such as row count, or column nullness) and dataset freshness by using the [ingested statistics](/metadata-ingestion/docs/dev_guides/sql_profiles.md). +::: + +An assertion is **a data quality test that finds data that violates a specified rule.** +Assertions serve as the building blocks of [Data Contracts](/docs/managed-datahub/observe/data-contract.md) – this is how we verify the contract is met. + +## How to Create and Run Assertions + +Data quality tests (a.k.a. assertions) can be created and run by DataHub Cloud or ingested from a 3rd party tool. + +### DataHub Cloud Assertions + +For DataHub-provided assertion runners, we can deploy an agent in your environment to hit your sources and DataHub. DataHub Cloud Observe offers out-of-the-box evaluation of the following kinds of assertions: + +- [Freshness](/docs/managed-datahub/observe/freshness-assertions.md) (SLAs) +- [Volume](/docs/managed-datahub/observe/volume-assertions.md) +- [Custom SQL](/docs/managed-datahub/observe/custom-sql-assertions.md) +- [Column](/docs/managed-datahub/observe/column-assertions.md) +- [Schema](/docs/managed-datahub/observe/schema-assertions.md) + +#### Monitoring Rules — Assertions at Scale + +[Monitoring Rules](/docs/managed-datahub/observe/data-health-dashboard.md#monitoring-rules) let you automatically apply [Smart Assertions](/docs/managed-datahub/observe/smart-assertions.md) (AI Anomaly Monitors) across your data landscape. Define a search predicate — such as a DataHub Domain, data platform, or schema — and DataHub will create Freshness, Volume, and Schema anomaly monitors on all matching datasets. As your data landscape evolves, new datasets that match the predicate are automatically covered, and datasets that no longer match have their monitors removed. + +You can create and manage Monitoring Rules from the [Data Health Dashboard](/docs/managed-datahub/observe/data-health-dashboard.md). + +To create column metric anomaly monitors for multiple columns on a single dataset, see the **Anomaly Detection** section of [Column Assertions](/docs/managed-datahub/observe/column-assertions#anomaly-detection-with-smart-assertions-). + +### Detecting Anomalies Across Massive Data Landscapes + +There are many cases where either you do not have the time to figure out what a good rule for an assertion is, or strict rules simply do not suffice for your data validation needs. Traditional rule-based assertions can become inadequate when dealing with complex data patterns or large-scale operations. + +**Common Scenarios** + +Here are some typical situations where manual assertion rules fall short: + +- **Seasonal data patterns** - A table whose row count changes exhibit weekly seasonality may need a different set of assertions for each day of the week, making static rules impractical to maintain. + +- **Statistical complexity across large datasets** - Figuring out what the expected standard deviation is for each column can be incredibly time consuming and not feasible across hundreds of tables, especially when each table has unique characteristics. + +- **Dynamic data environments** - When data patterns evolve over time, manually updating assertion rules becomes a maintenance burden that can lead to false positives or missed anomalies. + +### The AI Smart Assertion Solution + +In these scenarios, you may want to consider creating a [Smart Assertion](./smart-assertions.md) to let machine learning automatically detect the normal patterns in your data and alert you when anomalies occur. This approach allows for more flexible and adaptive data quality monitoring without the overhead of manual rule maintenance. + +Both traditional and smart assertions can be defined through the DataHub API or the UI. + +### Time-Series Bucketing & Historical Backfill + +For assertions that need to evaluate data at a specific time granularity (e.g., daily or weekly), you can enable **time-series bucketing** on [Volume](/docs/managed-datahub/observe/volume-assertions.md#time-series-bucketing) and [Column Metric](/docs/managed-datahub/observe/column-assertions.md#time-series-bucketing-for-column-metric-assertions) assertions. This partitions your data into time buckets using a timestamp column and evaluates each bucket independently. + +For Smart Assertions with bucketing enabled, you can also configure **historical backfill** to immediately populate the assertion's metrics history, so the AI model can start making accurate predictions from day one. See [Backfill Assertion History](/docs/managed-datahub/observe/assertion-backfill.md) for details. + +

+ +

+ +### Reporting from 3rd Party tools + +You can integrate 3rd party tools as follows: + +- [DBT Test](/docs/generated/ingestion/sources/dbt.md#integrating-with-dbt-test) +- [Great Expectations](../../../metadata-ingestion/integration_docs/great-expectations.md) +- [Custom Assertions](../../api/tutorials/custom-assertions.md) + +If you opt for a 3rd party tool, it will be your responsibility to ensure the assertions are run based on the Data Contract spec stored in DataHub. With 3rd party runners, you can get the Assertion Change events by subscribing to our Kafka topic using the [DataHub Actions Framework](/docs/actions/README.md). + +## Alerts + +Beyond the ability to see the results of the assertion checks (and history of the results) both on the physical asset’s page in the DataHub UI and as the result of DataHub API calls, you can also get notified via [Slack messages](/docs/managed-datahub/slack/saas-slack-setup.md) (DMs or to a team channel) based on your [subscription](https://youtu.be/VNNZpkjHG_I?t=79) to an assertion run event, or when an [incident](../../incidents/incidents.md) is raised or resolved. In the future, we’ll also provide the ability to subscribe directly to contracts. + +With DataHub Cloud Observe, you can react to the Assertion Run Event by listening to API events via [AWS EventBridge](/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md) (the availability and simplicity of setup of each solution dependent on your current DataHub Cloud setup – chat with your DataHub Cloud representative to learn more). + +## Sifting through the noise & Data Health Reporting + +Sometimes alerts can get noisy, and it's hard to sift through slack notifications to figure out what's important. Sometimes you need to figure out which of the tables your team owns actually have data quality checks running on them. +The [Data Health Dashboard](./data-health-dashboard.md) provides a birds-eye view of the health of your data landscape. You can slice and dice the data to find the exact answers you're looking for. + +## Cost + +We provide a plethora of ways to run your assertions, aiming to allow you to use the cheapest possible means to do so and/or the most accurate means to do so, depending on your use case. For example, for Freshness (SLA) assertions, it is relatively cheap to use either their Audit Log or Information Schema as a means to run freshness checks, and we support both of those as well as Last Modified Column, High Watermark Column, and DataHub Operation ([see the docs for more details](/docs/managed-datahub/observe/freshness-assertions.md#3-change-source)). + +## Execution details - Where and How + +There are a few ways DataHub Cloud assertions can be executed: + +1. Directly query the source system: + a. `Information Schema` tables are used by default to power cheap, fast checks on a table's freshness or row count. + b. `Audit log` or `Operation log` tables can be used to granularly monitor table operations. + c. The table itself can also be queried directly. This is useful for freshness checks referencing `last_updated` columns, row count checks targetting a subset of the data, and column value checks. We offer several optimizations to reduce query costs for these checks. +2. Reference DataHub metadata + a. [Operations](/docs/api/tutorials/operations.md) that are reported via ingestion or our SDKs can power monitoring table freshness. + b. `DatasetProfile` and `SchemaFieldProfile` ingested or reported via SDKs can power monitoring table metrics and column metrics. + +### Privacy: Execute In-Network, avoid exposing data externally + +As a part of DataHub Cloud, we offer a [Remote Executor](/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md) deployment model. If this model is used, assertions will execute within your network, and only the results will be sent back to DataHub Cloud. Neither your actual credentials, nor your source data will leave your network. + +### Source system selection + +Assertions will execute queries using the same source system that was used to initially ingest the table. +There are some scenarios where customers may have multiple ingestion sources for, i.e. a BigQuery table. In this case, by default the executor will take the ingestion source that was used to ingest the table's `DatasetProperties`. This behavior can be modified by your customer success rep. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/column-assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/column-assertions.md new file mode 100644 index 00000000..13e6bd48 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/column-assertions.md @@ -0,0 +1,520 @@ +--- +description: This page provides an overview of working with DataHub Column Assertions +title: Column Assertions +slug: /managed-datahub/observe/column-assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/column-assertions.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Column Assertions + + + +> The **Column Assertions** feature is available as part of the **DataHub Cloud Observe** module of DataHub Cloud. +> If you are interested in learning more about **DataHub Cloud Observe** or trying it out, please [visit our website](https://datahub.com/products/data-observability/). + +## Introduction + +Can you remember a time when an important warehouse table column changed dramatically, with little or no notice? Perhaps the number of null values suddenly spiked, or a new value was added to a fixed set of possible values. If the answer is yes, how did you initially find out? We'll take a guess - someone looking at an internal reporting dashboard or worse, a user using your your product, sounded an alarm when a number looked a bit out of the ordinary. + +There are many reasons why important columns in your Snowflake, Redshift, BigQuery, or Databricks tables may change - application code bugs, new feature rollouts, etc. Oftentimes, these changes break important assumptions made about the data used in building key downstream data products like reporting dashboards or data-driven product features. + +What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data issues before anyone else? With DataHub Cloud Column Assertions, you can. + +With DataHub Cloud, you can define **Column Value** assertions to ensure each value in a column matches specific constraints, and **Column Metric** assertions to ensure that computed metrics from columns align with your expectations. As soon as things go wrong, your team will be the first to know, before the data issue becomes a larger data incident. + +In this guide, we'll cover the basics of Column Assertions - what they are, how to configure them, and more - so that you and your team can start building trust in your most important data assets. + +Let's dive in! + +## Support + +Column Assertions are currently supported for: + +1. Snowflake +2. Redshift +3. BigQuery +4. Databricks +5. DataHub Dataset Profile Metrics (collected via ingestion) + +Note that an Ingestion Source _must_ be configured with the data platform of your choice in +DataHub Cloud's **Ingestion** tab. + +> Note that Column Assertions are not yet supported if you are connecting to your warehouse +> using the DataHub CLI. + +## What is a Column Assertion? + +A **Column Assertion** is a highly configurable Data Quality rule used to monitor specific columns of a Data Warehouse table for unexpected changes. + +Column Assertions are defined to validate a specific column, and can be used to + +1. Validate that the values of the column match some constraints (regex, allowed values, max, min, etc) across rows OR +2. Validate that specific column aggregation metrics match some expectations across rows. + +Column Assertions can be particularly useful for documenting and enforcing column-level "contracts", i.e. formal specifications about the expected contents of a particular column that can be used for coordinating among producers and consumers of the data. + +### Anatomy of Column Assertion + +Column Assertions can be divided into two main types: **Column Value** and **Column Metric** Assertions. + +A **Column Value Assertion** is used to monitor the value of a specific column in a table, and ensure that every row +adheres to a specific condition. In comparison, a **Column Metric Assertion** is used to compute a metric for that column, +and ensure that the value of that metric adheres to a specific condition. + +At the most basic level, both types consist of a few important parts: + +1. An **Evaluation Schedule** +2. A **Column Selection** +3. A **Evaluation Criteria** +4. A **Row Evaluation Type** + +In this section, we'll give an overview of each. + +#### 1. Evaluation Schedule + +The **Evaluation Schedule**: This defines how often to evaluate the Column Assertion against the given warehouse table. +This should usually be configured to match the expected change frequency of the table, although it can also be less +frequently depending on your requirements. You can also specify specific days of the week, hours in the day, or even +minutes in an hour. + +#### 2. Column Selection + +The **Column Selection**: This defines the column that should be monitored by the Column Assertion. You can choose from +any of the columns from the table listed in the dropdown. Note that columns of struct / object type are not currently supported. + +#### 3. Evaluation Criteria + +The **Evaluation Criteria**: This defines the condition that must be satisfied in order for the Column +Assertion to pass. + +For **Column Value Assertions**, you will be able to choose from a set of operators that can be applied to the column +value. The options presented will vary based on the data type of the selected column. For example, if you've selected a numeric column, you +can verify that the column value is greater than a particular value. For string types, you can check that the column value +matches a particular regex pattern. Additionally, you are able to control the behavior of the check in the presence of NULL values. If the +**Allow Nulls** option is _disabled_, then any null values encountered will be reported as a failure when evaluating the +assertion. If **Allow Nulls** is enabled, then nulls will be ignored; the condition will be evaluated for rows where the column value is non-null. + +For **Column Metric Assertions**, you will be able to choose from a list of common column metrics - MAX, MIN, MEAN, NULL COUNT, etc - and then compare these metric values to an expected value. The list of metrics will vary based on the type of the selected column. For example +if you've selected a numeric column, you can choose to compute the MEAN value of the column, and then assert that it is greater than a +specific number. For string types, you can choose to compute the MAX LENGTH of the string across all column values, and then assert that it +is less than a specific number. + +#### 4. Row Selection Set + +The **Row Selection Set**: This defines which rows in the table the Column Assertion will be evaluated across. You can choose +from the following options: + +- **All Table Rows**: Evaluate the Column Assertion across all rows in the table. This is the default option. Note that + this may not be desirable for large tables. + +- **Only Rows That Have Changed**: Evaluate the Column Assertion only against rows that have changed since the last + evaluation of the assertion. If you choose this option, you will need to specify a **High Watermark Column** to help determine which rows + have changed. A **High Watermark Column** is a column that contains a constantly incrementing value - a date, a time, or + another always-increasing number - that can be used to find the "new rows" that were added since previous evaluation. When selected, a query will be issued to the table to find only the rows that have changed since the previous assertion evaluation. + +## Creating a Column Assertion + +### Prerequisites + +1. **Permissions**: To create or delete Column Assertions for a specific entity on DataHub, you'll need to be granted the + `Edit Assertions` and `Edit Monitors` privileges for the entity. This will be granted to Entity owners as part of the `Asset Owners - Metadata Policy` + by default. + +2. (Optional) **Data Platform Connection**: In order to create a Column Assertion that queries the data source directly (instead of DataHub metadata), you'll need to have an **Ingestion Source** + configured to your Data Platform: Snowflake, BigQuery, or Redshift under the **Ingestion** tab. + +Once these are in place, you're ready to create your Column Assertions! + +### Steps + +#### 1. Navigate to the Table that you want to monitor + +#### 2. Click the **Quality** tab + +

+ +

+ +#### 3. Click **+ Create Assertion** + +#### 4. Choose **'Column'** + +

+ +

+ +#### 5. Configure the evaluation **schedule**. + +This is the frequency at which the assertion will be evaluated to produce a +pass or fail result, and the times when the column values will be checked. + +#### 6. Configure the **column assertion type**. + +You can choose from **Column Value** or **Column Metric**. +**Column Value** assertions are used to monitor the value of a specific column in a table, and ensure that every row +adheres to a specific condition. **Column Metric** assertions are used to compute a metric for that column, and then compare the value of that metric to your expectations. + +

+ +

+ +#### 7. Configure the **column selection**. + +This defines the column that should be monitored by the Column Assertion. +You can choose from any of the columns from the table listed in the dropdown. + +

+ +

+ +#### 8. Configure the **evaluation criteria**. This step varies based on the type of assertion you chose in the previous step. + +- **Column Value Assertions**: You will be able to choose from a set of operators that can be applied to the column + value. The options presented will vary based on the data type of the selected column. For example with numeric types, you + can check that the column value is greater than a specific value. For string types, you can check that the column value + matches a particular regex pattern. You will also be able to control the behavior of null values in the column. If the + **Allow Nulls** option is _disabled_, any null values encountered will be reported as a failure when evaluating the + assertion. Note, Smart Assertions are not supported for Column Value Assertions today. + + In addition, for the In Set and Not In Set operators, you can now choose how to provide the set of allowed values: + + - Static List: Manually enter a list of values (e.g., city names like "chicago", "new york"). + - Custom SQL: Provide a SQL query that returns a single column of possible values for the set. At evaluation time, DataHub executes this query using your configured data platform connection and compares each row’s column value against the returned set. + + Notes when using Custom SQL for sets: + + - The query must return exactly one column. Use `SELECT DISTINCT` to avoid duplicates if desired. + - The values returned should be comparable to the selected column’s data type (e.g., strings for VARCHAR/STRING columns, numbers for numeric columns). + - The query runs in the same warehouse connection you configured for the dataset. Ensure the account has read access to referenced objects and use fully qualified table names. + - Large or complex queries may impact evaluation latency and cost on your warehouse. + + Example (allowed city values sourced from a reference table): + + ```sql + SELECT DISTINCT city + FROM reference_data.geo.cities + WHERE active = TRUE + ``` + +- **Column Metric Assertions**: You will be able to choose from a list of common metrics and then specify the operator + and value to compare against. The list of metrics will vary based on the data type of the selected column. For example + with numeric types, you can choose to compute the average value of the column, and then assert that it is greater than a + specific number. For string types, you can choose to compute the max length of all column values, and then assert that it + is less than a specific number. You can also select the **Detect with AI** option to use Smart Assertions to detect anomalies in the column metric. Note that the **Detect with AI** option is currently only available for the following metrics: **null_count**, **unique_count**, **empty_count**, **zero_count**, and **negative_count**. For all other metrics (e.g. min, max, mean), use a fixed threshold instead. + +#### 9. Configure the **row evaluation type**. This defines which rows in the table the Column Assertion should evaluate. + +- **All Table Rows**: Evaluate the Column Assertion against all rows in the table. This is the default option. Note that + this may not be desirable for large tables. + +- **Only Rows That Have Changed**: Evaluate the Column Assertion only against rows that have changed since the last + evaluation. If you choose this option, you will need to specify a **High Watermark Column** to help determine which rows + have changed. A **High Watermark Column** is a column that contains a constantly-incrementing value - a date, a time, or + another always-increasing number. When selected, a query will be issued to the table find only the rows which have changed since the last assertion run. + +

+ +

+ +#### 10. (Optional) Click **Advanced** to further customize the Column Assertion. + +The options listed here will vary based on the type of assertion you chose in the previous step. + +- **Invalid Values Threshold**: For **Column Value** assertions, you can configure the number of invalid values + (i.e. rows) that are allowed to fail before the assertion is marked as failing. This is useful if you want to allow a limited number + of invalid values in the column. By default this is 0, meaning the assertion will fail if any rows have an invalid column value. + +- **Source**: For **Column Metric** assertions, you can choose the mechanism that will be used to obtain the column + metric. **Query** will issue a query to the dataset to compute the metric. This issues a query to the table, which can be more expensive than Information Schema. + **DataHub Dataset Profile** will use the DataHub Dataset Profile metadata to compute the metric. This is the cheapest option, but requires that Dataset Profiles are reported to DataHub. By default, Ingestion will report Dataset Profiles to DataHub, which can be and infrequent. You can report Dataset Profiles via the DataHub APIs for more frequent and reliable data. + +- **Additional Filters**: You can choose to add additional filters to the query that will be used to evaluate the + assertion. This is useful if you want to limit the assertion to a subset of rows in the table. Note this option will not + be available if you choose **DataHub Dataset Profile** as the **source**. + +#### 11. Configure actions that should be taken when the Column Assertion passes or fails + +

+ +

+ +- **Raise incident**: Automatically raise a new DataHub `Column` Incident for the Table whenever the Column Assertion is failing. This + may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when + an incident is created due to an Assertion failure. +- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Column Assertion. Note that + any other incidents will not be impacted. + +#### 12. Click **Next** and then **Save**. + +And that's it! DataHub will now begin to monitor your Column Assertion for the table. + +Once your assertion has run, you will begin to see Success or Failure status for the Table + +

+ +

+ +## Anomaly Detection with Smart Assertions ⚡ + +As part of the **DataHub Cloud Observe** module, DataHub Cloud also provides [Smart Assertions](./smart-assertions.md) out of the box. These are dynamic, AI-powered Column Metric Assertions that you can use to monitor anomalies on column metrics of important warehouse Tables, without requiring any manual setup. + +:::note Supported Metrics +Smart Assertions for Column Metrics currently support only: **null_count**, **unique_count**, **empty_count**, **zero_count**, and **negative_count**. Other column metrics (e.g. min, max, mean, median, stddev) are available for standard Column Metric Assertions with fixed thresholds, but cannot be used with the **Detect with AI** option at this time. Any existing Smart Assertions using other metrics will continue to operate normally. +::: + +You can create smart assertions by simply selecting the column and the metric you wish to monitor, and then clicking the `Detect with AI` option in the UI: + +

+ +

+ +**Bulk Creating for Multiple Columns** + +To select several columns on a table to monitor at once, you can use the **Bulk-Create Smart Assertions** button below the column selector in the Column Metric Assertion authoring UI. + + + +## Time-Series Bucketing for Column Metric Assertions + +By default, column metric assertions evaluate a metric (e.g., null count, min, max) across all rows or changed rows in your table. With **time-series bucketing**, you can partition your data into time-based buckets (e.g., daily or weekly) and evaluate column metrics within each bucket. + +This is useful when: + +- You want to monitor column quality at a day or week granularity +- You want to detect issues like "null count spiked for today's data" rather than checking the entire table +- Your column metrics have seasonal patterns that vary by day of week or time of year + +### Bucketing Configuration + +A time-series bucketing strategy for column assertions consists of: + +- **Timestamp column**: The date/time column used to partition rows into buckets (e.g., `created_at`, `updated_at`). +- **Bucket interval**: **Daily** (1 DAY) or **Weekly** (1 WEEK). +- **Timezone**: The IANA timezone for bucket boundaries. Defaults to UTC. +- **Late arrival grace period** (optional): A buffer after the bucket end time before evaluation. + +:::note +When time-series bucketing is enabled, the evaluation schedule is automatically computed from the bucket configuration. Column Value assertions (`FIELD_VALUES` type) do not support time-series bucketing — only Column Metric assertions (`FIELD_METRIC` type) do. +::: + +### Configuring Bucketing in the UI + +When creating a column metric assertion, you will see a **Row Evaluation Type** section with three options: + +

+ +

+ +- **All Table Rows**: Evaluate the assertion across all rows in the table (default). +- **Only Rows That Have Changed**: Use a high watermark column to evaluate only new rows. +- **Rows Within a Time Bucket**: Partition data into daily or weekly time buckets and evaluate each bucket independently. + +When selecting **Rows Within a Time Bucket**, you will configure the timestamp column, bucket size, timezone, and optional grace period. + +### Configuring Bucketing via the Python SDK + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +client = DataHubClient(server="", token="") +dataset_urn = DatasetUrn.from_string( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)" +) + +# Column metric assertion with daily bucketing +column_assertion = client.assertions.sync_column_metric_assertion( + dataset_urn=dataset_urn, + column_name="price", + metric_type="min", + operator="greater_than_or_equal_to", + criteria_parameters=0, + display_name="Daily Price Min Check", + time_bucketing_strategy={ + "timestamp_field_path": "order_date", + "bucket_interval": {"unit": "DAY", "multiple": 1}, + "timezone": "America/Los_Angeles", + }, + tags=["automated", "column_quality"], + enabled=True, +) + +# Smart column metric assertion with weekly bucketing and backfill +smart_column = client.assertions.sync_smart_column_metric_assertion( + dataset_urn=dataset_urn, + column_name="user_id", + metric_type="null_count", + display_name="Weekly Null Count Monitor", + detection_mechanism="all_rows_query_datahub_dataset_profile", + sensitivity="medium", + time_bucketing_strategy={ + "timestamp_field_path": "created_at", + "bucket_interval": {"unit": "WEEK", "multiple": 1}, + }, + backfill_config={"backfill_start_date_ms": 1704067200000}, + enabled=True, +) +``` + +:::info +For smart column metric assertions with bucketing enabled, you can configure **historical backfill** to populate metrics history. See [Backfill Assertion History](./assertion-backfill.md) for details. +::: + +## Stopping a Column Assertion + +In order to temporarily stop the evaluation of the assertion: + +1. Navigate to the **Quality** tab of the Table with the assertion +2. Click **Column** to open the Column Assertion assertions +3. Click the "Stop" button for the assertion you wish to pause. + +

+ +

+ +To resume the assertion, simply click **Start**. + +

+ +

+ +## Creating Column Assertions via API + +Under the hood, DataHub Cloud implements Column Assertion Monitoring using two concepts: + +- **Assertion**: The specific expectation for the column metric. e.g. "The value of an integer column is greater than 10 for all rows in the table." This is the "what". +- **Monitor**: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific + mechanisms. This is the "how". + +Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the +`Edit Assertions` and `Edit Monitors` privileges for it. + +#### GraphQL + +In order to create or update a Column Assertion, you can the `upsertDatasetColumnAssertionMonitor` mutation. + +#### Examples + +Creating a Field Values Column Assertion that runs every 8 hours: + +```graphql +mutation upsertDatasetFieldAssertionMonitor { + upsertDatasetFieldAssertionMonitor( + input: { + entityUrn: "" + type: FIELD_VALUES + fieldValuesAssertion: { + field: { + path: "" + type: "NUMBER" + nativeType: "NUMBER(38,0)" + } + operator: GREATER_THAN + parameters: { value: { type: NUMBER, value: "10" } } + failThreshold: { type: COUNT, value: 0 } + excludeNulls: true + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: ALL_ROWS_QUERY } + mode: ACTIVE + } + ) { + urn + } +} +``` + +To create an AI Smart Column Nullness Metric Assertion: + +```graphql +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + input: { + entityUrn: "" + type: FIELD_METRIC + inferWithAI: true + fieldMetricAssertion: { + field: { + path: "" + type: "NUMBER" + nativeType: "NUMBER(38,0)" + } + metric: NULL_PERCENTAGE + operator: BETWEEN + # you can provide any value for this as it will be overwritten continuously by the AI engine + parameters: { + minValue: { value: "0", type: NUMBER } + maxValue: { value: "0", type: NUMBER } + } + failThreshold: { type: COUNT, value: 0 } + excludeNulls: true + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: ALL_ROWS_QUERY } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can use same endpoint with assertion urn input to update an existing Column Assertion and corresponding Monitor. + +```graphql +mutation upsertDatasetFieldAssertionMonitor { + upsertDatasetFieldAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: FIELD_VALUES + fieldValuesAssertion: { + field: { + path: "" + type: "NUMBER" + nativeType: "NUMBER(38,0)" + } + operator: GREATER_THAN_OR_EQUAL_TO + parameters: { value: { type: NUMBER, value: "10" } } + failThreshold: { type: COUNT, value: 0 } + excludeNulls: true + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: ALL_ROWS_QUERY } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. + +### Tips + +:::info +**Authorization** + +Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows: + +``` +Authorization: Bearer +``` + +**Exploring GraphQL API** + +Also, remember that you can play with an interactive version of the DataHub Cloud GraphQL API at `https://your-account-id.acryl.io/api/graphiql` +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/custom-sql-assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/custom-sql-assertions.md new file mode 100644 index 00000000..405e1797 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/custom-sql-assertions.md @@ -0,0 +1,297 @@ +--- +description: This page provides an overview of working with DataHub SQL Assertions +title: Custom SQL Assertions +slug: /managed-datahub/observe/custom-sql-assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/custom-sql-assertions.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Custom SQL Assertions + + + +> The **Custom SQL Assertions** feature is available as part of the **DataHub Cloud Observe** module of DataHub Cloud. +> If you are interested in learning more about **DataHub Cloud Observe** or trying it out, please [visit our website](https://datahub.com/products/data-observability/). + +## Introduction + +Can you remember a time when the meaning of Data Warehouse Table that you depended on fundamentally changed, with little or no notice? +If the answer is yes, how did you find out? We'll take a guess - someone looking at an internal reporting dashboard or worse, a user using your your product, sounded an alarm when +a number looked a bit out of the ordinary. Perhaps your table initially tracked purchases made on your company's e-commerce web store, but suddenly began to include purchases made +through your company's new mobile app. + +There are many reasons why an important Table on Snowflake, Redshift, BigQuery, or Databricks may change in its meaning - application code bugs, new feature rollouts, +changes to key metric definitions, etc. Often times, these changes break important assumptions made about the data used in building key downstream data products +like reporting dashboards or data-driven product features. + +What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data +issues _before_ anyone else? With DataHub Cloud **Custom SQL Assertions**, you can. + +DataHub Cloud allows users to define complex expectations about a particular warehouse Table through custom SQL queries, and then monitor those expectations over time as the table grows and changes. + +In this article, we'll cover the basics of monitoring Custom SQL Assertions - what they are, how to configure them, and more - so that you and your team can +start building trust in your most important data assets. + +Let's get started! + +## Support + +Custom SQL Assertions are currently supported for: + +1. Snowflake +2. Redshift +3. BigQuery +4. Databricks + +Note that an Ingestion Source _must_ be configured with the data platform of your choice in DataHub Cloud's **Ingestion** +tab. + +> Note that SQL Assertions are not yet supported if you are connecting to your warehouse +> using the DataHub CLI. + +## What is a Custom SQL Assertion? + +A **Custom SQL Assertion** is a highly configurable Data Quality rule used to monitor a Data Warehouse Table +for unexpected or sudden changes in its meaning. Custom SQL Assertions are defined through a raw SQL query that is evaluated against +the Table. You have full control over the SQL query, and can use any SQL features supported by your Data Warehouse. +Custom SQL Assertions can be particularly useful when you have complex tables or relationships +that are used to generate important metrics or reports, and where the meaning of the table is expected to be stable over time. +If you have existing SQL queries that you already use to monitor your data, you may find that Custom SQL Assertions are an easy way to port them +to DataHub Cloud to get started. + +For example, imagine that you have a Table that tracks the number of purchases made on your company's e-commerce web store. +You have a SQL query that you use to calculate the number of purchases made in the past 24 hours, and you'd like to monitor this +metric over time to ensure that it is always greater than 1000. You can use a Custom SQL Assertion to do this! + +### Anatomy of a Custom SQL Assertion + +At the most basic level, **Custom SQL Assertions** consist of a few important parts: + +1. An **Evaluation Schedule** +2. A **Query** +3. A **Condition Type** + +In this section, we'll give an overview of each. + +#### 1. Evaluation Schedule + +The **Evaluation Schedule**: This defines how often to query the given warehouse Table. This should usually +be configured to match the expected change frequency of the Table, although it can also be less frequently depending +on the requirements. You can also specify specific days of the week, hours in the day, or even +minutes in an hour. + +#### 2. Query + +The **Query**: This is the SQL query that will be used to evaluate the Table. The query should return a **single row** containing a **single numeric column** (integers, floats). +The query can be as simple or as complex as you'd like, and can use any SQL features supported by your Data Warehouse. This requires that the configured user account has read access to the asset. Make sure to use the fully qualified name of the Table in your query. + +Use the "Try it out" button to test your query and ensure that it returns a single row with a single column. The query will be run against the Table in the context of the configured user account, so ensure that the user has read access to the Table. + +#### 3. Condition Type + +The **Condition Type**: This defines the conditions under which the Assertion will **fail**. The list of supported operations is: + +- **Is Equal To**: The assertion will fail if the query result is equal to the configured value +- **Is Not Equal To**: The assertion will fail if the query result is not equal to the configured value +- **Is Greater Than**: The assertion will fail if the query result is greater than the configured value +- **Is Less Than**: The assertion will fail if the query result is less than the configured value +- **Is Outside a Range**: The assertion will fail if the query result is outside the configured range +- **Grows More Than**: The assertion will fail if the query result grows more than the configured range. This can be either a percentage (**Percentage**) or a number (**Value**). +- **Grows Less Than**: The assertion will fail if the query result grows less than the configured percentage. This can be either a percentage (**Percentage**) or a number (**Value**). +- **Growth is outside a range**: The assertion will fail if the query result growth is outside the configured range. This can be either a percentage (**Percentage**) or a number (**Value**). + +Custom SQL Assertions also have an off switch: they can be started or stopped at any time with the click of button. + +## Creating a Custom SQL Assertion + +### Prerequisites + +1. **Permissions**: To create or delete Custom SQL Assertions for a specific entity on DataHub, you'll need to be granted the + `Edit Assertions`, `Edit Monitors`, **and the additional `Edit SQL Assertion Monitors`** privileges for the entity. This will be granted to Entity owners as part of the `Asset Owners - Metadata Policy` + by default. + +2. **Data Platform Connection**: In order to create a Custom SQL Assertion, you'll need to have an **Ingestion Source** configured to your + Data Platform: Snowflake, BigQuery, Redshift, or Databricks under the **Integrations** tab. + +Once these are in place, you're ready to create your Custom SQL Assertions! + +### Steps + +1. Navigate to the Table you want to monitor +2. Click the **Quality** tab + +

+ +

+ +3. Click **+ Create Assertion** + +

+ +

+ +4. Choose **Custom** + +5. Configure the evaluation **schedule**. This is the frequency at which the assertion will be evaluated to produce a pass or fail result, and the times + when the query will be executed. + +6. Provide a SQL **query** that will be used to evaluate the Table. The query should return a single row with a single column. Currently only numeric values are supported (integer and floats). The query can be as simple or as complex as you'd like, and can use any SQL features supported by your Data Warehouse. Make sure to use the fully qualified name of the Table in your query. + +

+ +

+ +7. Configure the evaluation **condition type**. This determines the cases in which the new assertion will fail when it is evaluated. + +

+ +

+ +8. Configure actions that should be taken when the Custom SQL Assertion passes or fails + +

+ +

+ +- **Raise incident**: Automatically raise a new DataHub Incident for the Table whenever the Custom SQL Assertion is failing. This + may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when + an incident is created due to an Assertion failure. + +- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Custom SQL Assertion. Note that + any other incidents will not be impacted. + +9. (Optional) Use the **Try it out** button to test your query and ensure that it returns a single row with a single column, and passes the configured condition type. + +

+ +

+ +10. Click **Next** and then add a description. + +11. Click **Save** + +And that's it! DataHub will now begin to monitor your Custom SQL Assertion for the table. + +Once your assertion has run, you will begin to see Success or Failure status for the Table + +

+ +

+ +## Stopping a Custom SQL Assertion + +In order to temporarily stop the evaluation of the assertion: + +1. Navigate to the **Quality** tab of the Table with the assertion +2. Click **Custom SQL** to open the SQL Assertion assertions +3. Click the "Stop" button for the assertion you wish to pause. + +

+ +

+ +To resume the assertion, simply click **Start**. + +

+ +

+ +## Anomaly Detection with Smart Assertions ⚡ + +As part of the **DataHub Cloud Observe** module, DataHub Cloud also provides [Smart Assertions](./smart-assertions.md) out of the box. These are +dynamic, AI-powered Custom SQL Assertions that you can use to monitor a metric computed by your SQL query, without +requiring any manual setup. + +You can create smart assertions by simply selecting the `Detect with AI` option in the UI: + +

+ +

+ +## Creating Custom SQL Assertions via API + +Under the hood, DataHub Cloud implements Custom SQL Assertion Monitoring using two concepts: + +- **Assertion**: The specific expectation for the custom assertion, e.g. "The table was changed in the past 7 hours" + or "The table is changed on a schedule of every day by 8am". This is the "what". + +- **Monitor**: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific + mechanisms. This is the "how". + +Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the +`Edit Assertions` and `Edit Monitors` privileges for it. + +#### GraphQL + +In order to create or update a Custom SQL Assertion, you can use the `upsertDatasetSqlAssertionMonitor` mutation. + +##### Examples + +To create a Custom SQL Assertion Entity that checks whether a query result is greater than 100 that runs every 8 hours: + +```graphql +mutation upsertDatasetSqlAssertionMonitor { + upsertDatasetSqlAssertionMonitor( + input: { + entityUrn: "" + type: METRIC + description: "" + statement: "" + operator: GREATER_THAN_OR_EQUAL_TO + parameters: { value: { value: "100", type: NUMBER } } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can use same endpoint with assertion urn input to update an existing Custom SQL Assertion and corresponding Monitor. + +```graphql +mutation upsertDatasetSqlAssertionMonitor { + upsertDatasetSqlAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: METRIC + description: "" + statement: "" + operator: GREATER_THAN_OR_EQUAL_TO + parameters: { value: { value: "100", type: NUMBER } } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. + +### Tips + +:::info +**Authorization** + +Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows: + +``` +Authorization: Bearer +``` + +**Exploring GraphQL API** + +Also, remember that you can play with an interactive version of the DataHub Cloud GraphQL API at `https://your-account-id.acryl.io/api/graphiql` +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-contract.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-contract.md new file mode 100644 index 00000000..125eb6f6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-contract.md @@ -0,0 +1,95 @@ +--- +title: Data Contracts +slug: /managed-datahub/observe/data-contract +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/data-contract.md +--- +# Data Contracts + +## What Is a Data Contract + +A Data Contract is **an agreement between a data asset's producer and consumer**, serving as a promise about the quality of the data. +It often includes [assertions](assertions.md) about the data’s schema, freshness, and data quality. + +Some of the key characteristics of a Data Contract are: + +- **Verifiable** : based on the actual physical data asset, not its metadata (e.g., schema checks, column-level data checks, and operational SLA-s but not documentation, ownership, and tags). +- **A set of assertions** : The actual checks against the physical asset to determine a contract’s status (schema, freshness, volume, custom, and column) +- **Producer oriented** : One contract per physical data asset, owned by the producer. + +
+Consumer Oriented Data contracts +We’ve gone with producer-oriented contracts to keep the number of contracts manageable and because we expect consumers to desire a lot of overlap in a given physical asset’s contract. Although, we've heard feedback that consumer-oriented data contracts meet certain needs that producer-oriented contracts do not. For example, having one contract per consumer all on the same physical data asset would allow each consumer to get alerts only when the assertions they care about are violated.We welcome feedback on this in slack! +
+ +Below is a screenshot of the Data Contracts UI in DataHub. + +

+ +

+ +## Data Contract and Assertions + +Another way to word our vision of data contracts is **A bundle of verifiable assertions on physical data assets representing a public producer commitment.** +These can be all the assertions on an asset or only the subset you want publicly promised to consumers. Data Contracts allow you to **promote a selected group of your assertions** as a public promise: if this subset of assertions is not met, the Data Contract is failing. + +See docs on [assertions](/docs/managed-datahub/observe/assertions.md) for more details on the types of assertions and how to create and run them. + +:::note Ownership +The owner of the physical data asset is also the owner of the contract and can accept proposed changes and make changes themselves to the contract. +::: + +## How to Create Data Contracts + +Data Contracts can be created via DataHub API, or UI. + +### UI + +1. Navigate to the Dataset Profile for the dataset you wish to create a contract for +2. Under the **Quality** > **Data Contracts** tab, click **Create**. + +

+ +

+ +3. Select the assertions you wish to be included in the Data Contract. + +

+ +

+ +:::note Create Data Contracts via UI +When creating a Data Contract via UI, the Freshness, Schema, and Data Quality assertions must be created first. +::: + +4. Now you can see it in the UI. + +

+ +

+ +### API + +_API guide on creating data contract is coming soon!_ + +## How to Run Data Contracts + +Running Data Contracts is dependent on running the contract’s assertions and getting the results on DataHub. Using DataHub Cloud Observe (available on SAAS), you can schedule assertions on DataHub itself. Otherwise, you can run your assertions outside of DataHub and have the results published back to DataHub. + +DataHub integrates nicely with DBT Test and Great Expectations, as described below. For other 3rd party assertion runners, you’ll need to use our APIs to publish the assertion results back to our platform. + +### DBT Test + +During DBT Ingestion, we pick up the dbt `run_results` file, which contains the dbt test run results, and translate it into assertion runs. [See details here.](/docs/generated/ingestion/sources/dbt.md#module-dbt) + +

+ +

+ +### Great Expectations + +For Great Expectations, you can integrate the **DataHubValidationAction** directly into your Great Expectations Checkpoint in order to have the assertion (aka. expectation) results to DataHub. [See the guide here](../../../metadata-ingestion/integration_docs/great-expectations.md). + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-health-dashboard.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-health-dashboard.md new file mode 100644 index 00000000..0c11c599 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/data-health-dashboard.md @@ -0,0 +1,103 @@ +--- +description: This page provides an overview of the Data Health Dashboard +title: Data Health Dashboard +slug: /managed-datahub/observe/data-health-dashboard +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/data-health-dashboard.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Data Health Dashboard + + + +## What Is the Data Health Dashboard + +The Data Health Dashboard aims to solve two critical use cases: + +1. Triaging Data Quality Issues +2. Understanding Broader Data Quality Coverage and Trends + +You can access it via the Sidebar Nav. It can be found under the _Observe_ section. + +

+ +

+ +## How to use it + +### Assertions Tab + +There are two ways to slice assertions: + +1. By Assertion +2. By Table + +**When to use `By Assertion`** + +This view presents an activity log of assertion runs, sorted by the last time a given assertion ran. This is incredibly valuable for triaging and detecting trends in the data quality checks. + +For instance, by applying a time range filter (i.e., `Last 7 Days`), and setting `Results` to `At least one failure`, you can quickly see which assertions have failed at least once in the last 7 days. Furthermore, you'll be able to see how often their failing, relative to how often they're running, enabling you to quickly find and investigate flaky checks. + +**When to use `By Table`** + +This view presents a list of tables that have at least one assertion that has ran on it. It is sorted by the last time any assertion ran on that table. The health dots indicate the last status of an assertion of that given type on the table. + +This view is incredibly useful for understanding monitoring coverage across your team's tables. + +### Incidents Tab + +The incidents tab presents the tables that have active incidents open against them. It is sorted by the last time an incident activity was reported on the given table. + +At a glance, you can grasp how many incidents are open against any given table, see which incident last had updates on that table, and who owns it. + +**Coming soon:** In the future we'll be introducing high-level visual cards giving useful statistics on table coverage, time to resolution, and more. +We will also be introducing a timeline view of assertion failures over a given time period. Our hope is to make it even easier to detect trends in data quality failures at a single glance. + +## Personalizing the Dashboard + +We understand that each team, and perhaps even an individual may care about a different subset of data than others. +For this reason, we have included a broad range of filters to make it easy to drill down the Dashboard to the specific subset of data you care about. You can filter by: + +1. Dataset Owner +2. Dataset Domain +3. Dataset Tags + ...and much more. + +In addition, both the `By Tables` tab and the `Incidents` tab will apply your global `View` (managed via the search bar on the very top of DataHub's navigation). So if you already have a view created for your team, these tabs will automatically filter the reports down to the subset of data only you care about. + +

+ +

+ +## Monitoring Rules + +Monitoring Rules let you automatically apply [Smart Assertions](./smart-assertions.md) (AI anomaly monitors) across your data landscape using search-based predicates. Instead of manually creating assertions on individual tables, you define a rule that describes _which_ datasets should be monitored and _what_ to monitor, and DataHub takes care of the rest. + +### Prerequisites + +To create and manage Monitoring Rules, you must have the **`Manage Tests`** platform privilege. Contact your DataHub admin if you do not have this privilege. + +### How It Works + +1. **Define a search predicate** — specify the datasets you want to monitor using filters such as DataHub Domain, data platform, schema, tags, or any combination of search criteria. +2. **Choose assertion types** — enable one or more of Freshness, Volume, and Schema anomaly monitoring for matching datasets. +3. **Configure subscriptions** — set up alert subscriptions so you or your team are notified when anomalies are detected. +4. **Save the rule** — DataHub will automatically create Smart Assertions on all datasets that currently match the predicate. + +You can create and manage Monitoring Rules from the **Data Health Dashboard** by clicking the **Monitoring Rules** button. + +
+ +### Automatic Lifecycle Management + +Monitoring Rules are continuously evaluated as your data landscape evolves: + +- **New datasets that match** the predicate will automatically have Smart Assertions created for them. +- **Datasets that no longer match** the predicate will have their Smart Assertions stopped and removed by the rule. +- **Stopping a rule** will stop all Smart Assertions that were created by that rule. + +:::note Subscription behavior +Subscriptions created by a Monitoring Rule are **not** removed when a dataset stops matching the predicate or when the rule is stopped. This ensures you retain visibility into any in-flight alerts or ongoing incidents even after the monitoring scope changes. You can manage subscriptions independently if needed. +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/freshness-assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/freshness-assertions.md new file mode 100644 index 00000000..cdec809f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/freshness-assertions.md @@ -0,0 +1,382 @@ +--- +description: This page provides an overview of working with DataHub Freshness Assertions +title: Freshness Assertions +slug: /managed-datahub/observe/freshness-assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/freshness-assertions.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Freshness Assertions + + + +> The **Freshness Assertions** feature is available as part of the **DataHub Cloud Observe** module of DataHub Cloud. +> If you are interested in learning more about **DataHub Cloud Observe** or trying it out, please [visit our website](https://datahub.com/products/data-observability/). + +## Introduction + +Can you remember a time when a Data Warehouse Table that you depended on went days, weeks, or even +months without being updated with fresh data? + +Perhaps a bug had been introduced into an upstream Airflow DAG +or worse, the person in charge of maintaining the Table has departed from your organization entirely. +There are many reasons why an important Table on Snowflake, Redshift, BigQuery, or Databricks may fail to be updated as often as expected. + +What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data +issues _before_ anyone else? What if you could communicate commitments about the freshness or change frequency +of a table? With DataHub Cloud Freshness Assertions, you can. + +DataHub Cloud allows users to define expectations about when a particular Table in the warehouse +should change, and then monitor those expectations over time, with the ability to be notified when things go wrong. + +In this article, we'll cover the basics of monitoring Freshness Assertions - what they are, how to configure them, and more - so that you and your team can +start building trust in your most important data assets. + +Let's get started! + +## Support + +Freshness Assertions are currently supported for: + +1. Snowflake +2. Redshift +3. BigQuery +4. Databricks +5. DataHub Operations (collected via ingestion) + +Note that an Ingestion Source _must_ be configured with the data platform of your choice in DataHub Cloud's **Ingestion** +tab. + +> Note that Freshness Assertions are not yet supported if you are connecting to your warehouse +> using the DataHub CLI. + +## What is a Freshness Assertion? + +A **Freshness Assertion** is a configurable Data Quality rule used to determine whether a Table +on the Data Warehouse has been updated within a given period of time. Freshness Assertions are particularly useful when you have frequently-changing +Tables. + +For example, imagine that we work for a company with a Snowflake Table that stores user clicks collected from our e-commerce website. +This table is updated with new data on a specific cadence: once per hour (In practice, daily or even weekly are also common). +In turn, there is a downstream Business Analytics Dashboard in Looker that shows important metrics like +the number of people clicking our "Daily Sale" banners, and this dashboard is generated from data stored in our "clicks" table. +It is important that our clicks Table continues to be updated each hour because if it stops being updated, it could mean +that our downstream metrics dashboard becomes incorrect. And the risk of this situation is obvious: our organization +may make bad decisions based on incomplete information. + +In such cases, we can use a **Freshness Assertion** that checks whether the Snowflake "clicks" Table is being updated with +fresh data each and every hour as expected. If an hour goes by without any changes, we can immediately notify our team, to prevent any +negative impacts. + +### Anatomy of a Freshness Assertion + +At the most basic level, **Freshness Assertions** consist of a few important parts: + +1. An **Evaluation Schedule** +2. A **Change Window** +3. A **Change Source** + +In this section, we'll give an overview of each. + +#### 1. Evaluation Schedule + +The **Evaluation Schedule**: This defines how often to check a given warehouse Table for new updates. This should usually +be configured to match the expected change frequency of the Table, although is can also be more frequently. +If the Table changes daily, it should be daily. If it changes hourly, it should be hourly. You can also specify specific days of the week, hours in the day, or even +minutes in an hour. + +#### 2. Change Window + +The **Change Window**: This defines the window of time that is used when determining whether a change has been made to a Table. +We can either check for change to the Table + +- _Since the freshness check was last evaluated_. For example, if the evaluation schedule is set to run every day at + 8am PST, we can check whether a change was made between the previous day at 8am and the following day at 8am. + +- _Within a specific amount of time of the freshness check being evaluated_ (A fixed interval). For example, if the evaluation schedule is set to run + every day at 8am PST, we can check whether a change was made in the _8 hours before_ the check is evaluated, which would mean + in the time between midnight (12:00am) and 8:00am PST. + +#### 3. Change Source + +The **Change Source**: This is the mechanism that DataHub Cloud should use to determine whether the Table has changed. The supported +Change Source types vary by the platform, but generally fall into these categories: + +- **Audit Log** (Default): A metadata API or Table that is exposed by the Data Warehouse which contains captures information about the + operations that have been performed to each Table. It is usually efficient to check, but some useful operations are not + fully supported across all major Warehouse platforms. Note that for Databricks, [this option](https://docs.databricks.com/en/delta/history.html) + is only available for tables stored in Delta format. + +- **Information Schema**: A system Table that is exposed by the Data Warehouse which contains live information about the Databases + and Tables stored inside the Data Warehouse. It is usually efficient to check, but lacks detailed information about the _type_ + of change that was last made to a specific table (e.g. the operation itself - INSERT, UPDATE, DELETE, number of impacted rows, etc). + Note that for Databricks, [this option](https://docs.databricks.com/en/delta/table-details.html) is only available for tables stored in Delta format. + +- **Last Modified Column**: A Date or Timestamp column that represents the last time that a specific _row_ was touched or updated. + Adding a Last Modified Column to each warehouse Table is a pattern is often used for existing use cases around change management. + If this change source is used, a query will be issued to the Table to search for rows that have been modified within a specific + window of time (based on the Change Window) + +- **High Watermark Column**: A column that contains a constantly-incrementing value - a date, a time, or another always-increasing number. + If this change source is used, a query will be issued to the Table to look for rows with a new "high watermark", e.g. a value that + is higher than the previously observed value, in order to determine whether the Table has been changed within a given period of time. + Note that this approach is only supported if the Change Window does not use a fixed interval. + +- **DataHub Operation**: A DataHub "Operation" aspect contains timeseries information used to describe changes made to an entity. Using this + option avoids contacting your data platform, and instead uses the DataHub Operation metadata to evaluate Freshness Assertions. + This relies on Operations being reported to DataHub, either via ingestion or via use of the DataHub APIs (see [Report Operation via API](#reporting-operations-via-api)). + Note if you have not configured an ingestion source through DataHub, then this may be the only option available. By default, any operation type found will be considered a valid change. Use the **Operation Types** dropdown when selecting this option to specify which operation types should be considered valid changes. You may choose from one of DataHub's standard Operation Types, or specify a "Custom" Operation Type by typing in the name of the Operation Type. + +- **File Metadata** (Databricks Only): A column that is exposed by Databricks for both Unity Catalog and Hive Metastore based tables + which includes information about the last time that a file for the table was changed. Read more about it [here](https://docs.databricks.com/en/ingestion/file-metadata-column.html). + + Using either of the column value approaches (**Last Modified Column** or **High Watermark Column**) to determine whether a Table has changed can be useful because it can be customized to determine whether specific types of changes have been made to a given Table. + And because this type of assertion does not involve system warehouse tables, they are easily portable across Data Warehouse and Data Lake providers. + +Freshness Assertions also have an off switch: they can be started or stopped at any time with the click of button. + +## Creating a Freshness Assertion + +### Prerequisites + +1. **Permissions**: To create or delete Freshness Assertions for a specific entity on DataHub, you'll need to be granted the + `Edit Assertions` and `Edit Monitors` privileges for the entity. This will be granted to Entity owners as part of the `Asset Owners - Metadata Policy` + by default. +2. (Optional) **Data Platform Connection**: In order to create a Freshness Assertion that queries the source data platform directly (instead of DataHub metadata), you'll need to have an **Ingestion Source** configured to your + Data Platform: Snowflake, BigQuery, or Redshift under the **Integrations** tab. + +Once these are in place, you're ready to create your Freshness Assertions! + +You can also apply Smart Freshness Assertions at scale using [Monitoring Rules](/docs/managed-datahub/observe/data-health-dashboard.md#monitoring-rules) on the Data Health page. + +### Steps + +1. Navigate to the Table that to monitor for freshness +2. Click the **Quality** tab + +

+ +

+ +3. Click **+ Create Assertion** + +

+ +

+ +4. Choose **Freshness** + +5. Configure the evaluation **schedule**. This is the frequency that the table will be checked for changes. This represents you + expectation about the frequency at which the table should be updated. +6. Configure the evaluation **period**. This defines the period of time that will be considered when looking for changes to the table. Choose between _Since the previous check_ to check whether the table has changed since the past evaluation, + or _In the past X hours_ to configure a fixed interval that is used when checking the table. + +_Check whether the table has changed between subsequent evaluations of the check_ + +

+ +

+ +_Check whether the table has changed in a specific window of time_ + +

+ +

+ +7. (Optional) Click **Advanced** to customize the evaluation **source**. This is the mechanism that will be used to evaluate + the check. Each Data Platform supports different options including Audit Log, Information Schema, Last Modified Column, High Watermark Column, and DataHub Operation. + +

+ +

+ +- **Audit Log**: Check the Data Platform operational audit log to determine whether the table changed within the evaluation period. This will filter out No-Ops (e.g. `INSERT 0`). However, the Audit Log can be delayed by several hours depending on the Data Platform. This is also a little more costly on the warehouse than Information Schema. +- **Information Schema**: Check the Data Platform system metadata tables to determine whether the table changed within the evaluation period. This is the optimal balance between cost and accuracy for most Data Platforms. +- **Last Modified Column**: Check for the presence of rows using a "Last Modified Time" column, which should reflect the time at which a given row was last changed in the table, to + determine whether the table changed within the evaluation period. This issues a query to the table, which can be more expensive than Information Schema. +- **High Watermark Column**: Monitor changes to a continuously-increasing "high watermark" column value to determine whether a table + has been changed. This option is particularly useful for tables that grow consistently with time, for example fact or event (e.g. click-stream) tables. It is not available + when using a fixed lookback period. This issues a query to the table, which can be more expensive than Information Schema. +- **DataHub Operation**: Use DataHub Operations to determine whether the table changed within the evaluation period. This is the cheapest option, but requires that Operations are reported to DataHub. By default, Ingestion will report Operations to DataHub, which can be and infrequent. You can report Operations via the DataHub APIs for more frequent and reliable data. + +8. Configure actions that should be taken when the Freshness Assertion passes or fails + +

+ +

+ +- **Raise incident**: Automatically raise a new DataHub `Freshness` Incident for the Table whenever the Freshness Assertion is failing. This + may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when + an incident is created due to an Assertion failure. + +- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Freshness Assertion. Note that + any other incidents will not be impacted. + +9. Click **Next** and add a description. + +10. Click **Save**. + +And that's it! DataHub will now begin to monitor your Freshness Assertion for the table. + +Once your assertion has run, you will begin to see Success or Failure status for the Table + +

+ +

+ +## Stopping a Freshness Assertion + +In order to temporarily stop the evaluation of the assertion: + +1. Navigate to the **Quality** tab of the Table with the assertion +2. Click **Freshness** to open the Freshness Assertion assertions +3. Click the "Stop" button for the assertion you wish to pause. + +

+ +

+ +To resume the assertion, simply click **Start**. + +

+ +

+ +## Anomaly Detection with Smart Assertions ⚡ + +As part of the **DataHub Cloud Observe** module, DataHub Cloud also provides **Smart Assertions** out of the box. These are +dynamic, AI-powered Freshness Assertions that you can use to monitor the freshness of important warehouse Tables, without +requiring any manual setup. The Smart Assertion's ML model will train based on the Table change history that is captured in the [`operation` aspect](../../api/tutorials/operations.md). Normally this is populated during ingestion run time. + +You can create smart assertions by simply selecting the `Detect with AI` option in the UI: + +

+ +

+ +## Creating Freshness Assertions via API + +Under the hood, DataHub Cloud implements Freshness Assertion Monitoring using two concepts: + +- **Assertion**: The specific expectation for freshness, e.g. "The table was changed int the past 7 hours" + or "The table is changed on a schedule of every day by 8am". This is the "what". +- **Monitor**: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific + mechanisms. This is the "how". + +Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the +`Edit Assertions` and `Edit Monitors` privileges for it. + +#### GraphQL + +In order to create or update a Freshness Assertion, you can use the `upsertDatasetFreshnessAssertionMonitor` mutation. + +##### Examples + +To create a Freshness Assertion Entity that checks whether a table has been updated in the past 8, and runs every 8 hours: + +```graphql +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + input: { + entityUrn: "" + schedule: { + type: FIXED_INTERVAL + fixedInterval: { unit: HOUR, multiple: 8 } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +To create an AI Smart Freshness Assertion: + +```graphql +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + input: { + entityUrn: "" + inferWithAI: true + evaluationSchedule: { timezone: "America/Los_Angeles", cron: "0 * * * *" } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can use same endpoint with assertion urn input to update an existing Freshness Assertion and corresponding Monitor: + +```graphql +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + schedule: { + type: FIXED_INTERVAL + fixedInterval: { unit: HOUR, multiple: 6 } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. + +### Reporting Operations via API + +DataHub Operations can be used to capture changes made to entities. This is useful for cases where the underlying data platform does not provide a mechanism +to capture changes, or where the data platform's mechanism is not reliable. In order to report an operation, you can use the `reportOperation` GraphQL mutation. + +##### Examples + +```graphql +mutation reportOperation { + reportOperation( + input: { + urn: "" + operationType: INSERT + sourceType: DATA_PLATFORM + timestampMillis: 1693252366489 + } + ) +} +``` + +Use the `timestampMillis` field to specify the time at which the operation occurred. If no value is provided, the current time will be used. + +### Tips + +:::info +**Authorization** + +Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows: + +``` +Authorization: Bearer +``` + +**Exploring GraphQL API** + +Also, remember that you can play with an interactive version of the DataHub Cloud GraphQL API at `https://your-account-id.acryl.io/api/graphiql` +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/schema-assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/schema-assertions.md new file mode 100644 index 00000000..ce8930de --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/schema-assertions.md @@ -0,0 +1,267 @@ +--- +description: This page provides an overview of working with DataHub Schema Assertions +title: Schema Assertions +slug: /managed-datahub/observe/schema-assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/schema-assertions.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Schema Assertions + + + +> The **Schema Assertions** feature is available as part of the **DataHub Cloud Observe** module of DataHub Cloud. +> If you are interested in learning more about **DataHub Cloud Observe** or trying it out, please [visit our website](https://datahub.com/products/data-observability/). + +## Introduction + +Can you remember a time when columns were unexpectedly added, removed, or altered for a key Table in your Data Warehouse? +Perhaps this caused downstream tables, views, dashboards, data pipelines, or AI models to break. + +There are many reasons why the structure of an important Table on Snowflake, Redshift, or BigQuery may schema change, breaking the expectations +of downstream consumers of the table. + +What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data +issues _before_ anyone else? With DataHub Cloud **Schema Assertions**, you can. + +DataHub Cloud allows users to define expectations about a table's columns and their data types, and will monitor and validate these expectations over +time, notifying you when a breaking change occurs. + +In this article, we'll cover the basics of monitoring Schema Assertions - what they are, how to configure them, and more - so that you and your team can +start building trust in your most important data assets. + +Let's get started! + +## Support + +Schema Assertions are currently supported for all data sources that provide a schema via the normal ingestion process. + +## What is a Schema Assertion? + +A **Schema Assertion** is a Data Quality rule used to monitor the columns in a particular table and their data types. +They allow you to define a set of "required" columns for the table along with their expected types, and then be notified +if anything changes via a failing assertion. + +This type of assertion can be particularly useful if you want to monitor the structure of a table which is outside of your +direct control, for example the result of an ETL process from an upstream application or tables provided by a 3rd party data vendor. It +allows you to get ahead of potentially breaking schema changes, by alerting you as soon as they occur, and before +they have a chance to negatively impact downstream assets. + +### Anatomy of a Schema Assertion + +At the most basic level, **Schema Assertions** consist of a few important parts: + +1. A **Condition Type** +2. A set of **Expected Columns** + +In this section, we'll give an overview of each. + +#### 1. Condition Type + +The **Condition Type** defines the conditions under which the Assertion will **fail**. More concretely, it determines +how the _expected_ columns should be compared to the _actual_ columns found in the schema to determine a passing or failing +state for the data quality check. + +The list of supported condition types: + +- **Contains**: The assertion will fail if the actual schema does not contain all expected columns and their types. +- **Exact Match**: The assertion will fail if the actual schema does not EXACTLY match the expected columns and their types. No + additional columns will be permitted. + +Schema Assertions will be evaluated whenever a change in the schema of the underlying table is detected. +They also have an off switch: they can be started or stopped at any time by pressing the start (play) or stop (pause) buttons. + +#### 2. Expected Columns + +The **Expected Columns** are a set of column **names** along with their high-level **data +types** that should be used to compare against the _actual_ columns found in the table. By default, the expected column +set will be derived from the current set of columns found in the table. This conveniently allows you to "freeze" or "lock" +the current schema of a table in just a few clicks. + +Each "expected column" is composed of a + +1. **Name**: The name of the column that should be present in the table. Nested columns are supported in a flattened + fashion by simply providing a dot-separated path to the nested column. For example, `user.id` would be a nested column `id`. + In the case of a complex array or map, each field in the elements of the array or map will be treated as dot-delimited columns. + Note that verifying the specific type of object in primitive arrays or maps is not currently supported. Note that the comparison performed + is currently not case-sensitive. + +2. **Type**: The high-level data type of the column in the table. This type intentionally "high level" to allow for normal column widening practices + without the risk of failing the assertion unnecessarily. For example a `varchar(64)` and a `varchar(256)` will both resolve to the same high-level + "STRING" type. The currently supported set of data types include the following: + + - String + - Number + - Boolean + - Date + - Timestamp + - Struct + - Array + - Map + - Union + - Bytes + - Enum + +## Creating a Schema Assertion + +### Prerequisites + +- **Permissions**: To create or delete Schema Assertions for a specific entity on DataHub, you'll need to be granted the + `Edit Assertions`, `Edit Monitors` privileges for the entity. This will be granted to Entity owners as part of the `Asset Owners - Metadata Policy` + by default. + +Once these are in place, you're ready to create your Schema Assertions! + +### Steps + +1. Navigate to the Table you want to monitor +2. Click the **Quality** tab + +

+ +

+ +3. Click **+ Create Assertion** + +

+ +

+ +4. Choose **Schema** + +5. Select the **condition type**. + +6. Define the **expected columns** that will be continually compared against the actual column set. This defaults to the current columns for the table. + +

+ +

+ +7. Configure actions that should be taken when the assertion passes or fails + +

+ +

+ +- **Raise incident**: Automatically raise a new DataHub Incident for the Table whenever the Custom SQL Assertion is failing. This + may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when + an incident is created due to an Assertion failure. + +- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Custom SQL Assertion. Note that + any other incidents will not be impacted. + +Then click **Next**. + +7. (Optional) Add a **description** for the assertion. This is a human-readable description of the assertion. If you do not provide one, a description will be generated for you. + +

+ +

+ +8. Click **Save**. + +And that's it! DataHub will now begin to monitor your Schema Assertion for the table. + +Once your assertion has run, you will begin to see Success or Failure status: + +

+ +

+ +## Stopping a Schema Assertion + +In order to temporarily stop the evaluation of the assertion: + +1. Navigate to the **Quality** tab of the Table with the assertion +2. Click **Schema** to open the Schema Assertion +3. Click the "Stop" button. + +

+ +

+ +To resume the assertion, simply click **Start**. + +

+ +

+ +## Creating Schema Assertions via API + +Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the +`Edit Assertions` and `Edit Monitors` privileges to create schema assertion via API. + +#### GraphQL + +In order to create a Schema Assertions, you can use the `upsertDatasetSchemaAssertionMonitor` mutation. + +##### Examples + +To create a Schema Assertion that checks for a the presence of a specific set of columns: + +```graphql +mutation upsertDatasetSchemaAssertionMonitor { + upsertDatasetSchemaAssertionMonitor( + input: { + entityUrn: "" + assertion: { + compatibility: SUPERSET # How the actual columns will be compared against the expected fields (provided next) + fields: [ + { path: "id", type: STRING } + { path: "count", type: NUMBER } + { path: "struct", type: STRUCT } + { path: "struct.nestedBooleanField", type: BOOLEAN } + ] + } + description: "" + mode: ACTIVE + } + ) +} +``` + +The supported compatibility types are `EXACT_MATCH` and `SUPERSET` (Contains). + +You can use same endpoint with assertion urn input to update an existing Schema Assertion, simply add the `assertionUrn` field: + +```graphql +mutation upsertDatasetSchemaAssertionMonitor { + upsertDatasetSchemaAssertionMonitor( + assertionUrn: "urn:li:assertion:existing-assertion-id" + input: { + entityUrn: "" + assertion: { + compatibility: EXACT_MATCH + fields: [ + { path: "id", type: STRING } + { path: "count", type: NUMBER } + { path: "struct", type: STRUCT } + { path: "struct.nestedBooleanField", type: BOOLEAN } + ] + } + description: "" + mode: ACTIVE + } + ) +} +``` + +You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. + +### Tips + +:::info +**Authorization** + +Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows: + +``` +Authorization: Bearer +``` + +**Exploring GraphQL API** + +Also, remember that you can play with an interactive version of the DataHub Cloud GraphQL API at `https://your-account-id.acryl.io/api/graphiql` +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/smart-assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/smart-assertions.md new file mode 100644 index 00000000..a04a9924 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/smart-assertions.md @@ -0,0 +1,95 @@ +--- +description: This page provides an overview of Smart Assertions (AI Anomaly Detection) +title: Smart Assertions (AI Anomaly Detection) ⚡ +slug: /managed-datahub/observe/smart-assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/smart-assertions.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Smart Assertions (AI Anomaly Detection) ⚡ + + + +## What are Smart Assertions? + +Smart Assertions are Anomaly Detection monitors that will train on the historical patterns of the data, and make predictions of what 'normal' looks like. They are powered by an incredibly sophisticated ML pipeline, enabling them to account for a large variety of trends in the data, including seasonality. + +

+ +

+ +## How do I create Smart Assertions? + +Today, you can create Smart Assertions for 4 types of assertions. To learn more about each one, click the respective links below: + +1. [Volume](./volume-assertions.md#anomaly-detection-with-smart-assertions-) +2. [Freshness](./freshness-assertions.md#anomaly-detection-with-smart-assertions-) +3. [Column Metrics](./column-assertions.md#anomaly-detection-with-smart-assertions-) +4. [Custom SQL](./custom-sql-assertions.md#anomaly-detection-with-smart-assertions-) + +You can also create Smart Assertions at scale using [Monitoring Rules](/docs/managed-datahub/observe/data-health-dashboard.md#monitoring-rules) on the Data Health page. Monitoring Rules let you define a search predicate (e.g. a domain, platform, or schema) and automatically apply Freshness, Volume, and Schema anomaly monitors to all matching datasets — including new datasets as they appear. + +## Time-Series Bucketing + +Smart Assertions can be configured with **time-series bucketing** to evaluate data quality at a day or week granularity. Instead of checking the entire table each time, the assertion partitions rows into time buckets using a timestamp column and evaluates each bucket independently. + +This is especially powerful for Smart Assertions because it enables the AI model to learn patterns like "Mondays always have higher volume" or "weekend null counts are typically lower", leading to more accurate anomaly detection. + +Time-series bucketing is supported for: + +- [Smart Volume Assertions](./volume-assertions.md#time-series-bucketing) +- [Smart Column Metric Assertions](./column-assertions.md#time-series-bucketing-for-column-metric-assertions) + +## Backfill Assertion History + +When you create a Smart Assertion with time-series bucketing, you can optionally **backfill historical data** so the AI model has enough context to make accurate predictions from day one. Without backfill, the model needs to accumulate data over days or weeks of scheduled evaluations before it can reliably detect anomalies. + +With backfill enabled, the system queries your warehouse for historical data and populates the assertion's metrics history immediately. This means you get meaningful anomaly detection thresholds right away, with full awareness of seasonality patterns. + +For full details on how backfill works, how to configure it, and how to retry failed backfills, see the dedicated [Backfill Assertion History](./assertion-backfill.md) page. + +
+ +## Improving Smart assertion quality + +You can improve predictions through two key levers: + +1. Tuning +2. Anomaly feedback + +### Tuning + +You can fix most Smart Assertions with 3 key actions - correct training data, adjust sensitivity, and increasing the lookback window. Each of these can be accessed via the **Tune Predictions** button on the Assertion Profile, or the **Settings tab**. + +
+ +**Exclusion Windows** +Set time windows to exclude from training data. This can be useful to exclude known maintenance windows or other periods of downtime, seasonal spikes e.g. holidays or any other windows that are not representative of normal data trends. + +**Sensitivity** +A higher sensitivity will have a tighter fit on the data. A lower sensitivity will allow for more data variation before an anomaly is flagged. + +**Training data lookback window** +This is the number of days our ML models will look back to gather training data to generate predictions. If this is too large, we may pick up on old data that is no longer part of the current trend. If it is too short, we may miss key seasonal patterns. You can leverage this alongside exclusion windows to improve the quality of predictions. + +### Anomaly Feedback + +#### False alarms + +When an anomaly is flagged by the smart assertion, you may hover over the result dot and select `Mark as Normal`. This will include this data point in the training set, and the model will adjust to ensure it does not flag such data points as anomalies again. + +**However**, there are some cases where anomalies are actually expected. For instance, if this is a sudden jump in data, **but it will be the new normal moving forward**, we recommend selecting the `Train as new Normal` option. This will add an exclusion window to all data prior to this run event, and the smart assertion will begin training on data points from this point forward. + +#### Missed alarms + +If an anomaly is not caught by our Smart Assertions, we recommend doing a few things: + +1. You can click `Mark as Anomaly` to flag this specific data point as an anomaly. This will exclude that data point from the training data. +2. Click **Tune Predictions** on the assertion, then exclude any “bad” historical periods from the training set (by adding an `Exclusion Window`). This is useful if older incidents or one-off events are polluting the model’s notion of “normal”. +3. Finally, consider increasing the sensitivity of the assertion in the **Settings tab** which will reduce the range of allowable values. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/volume-assertions.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/volume-assertions.md new file mode 100644 index 00000000..330147a2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/observe/volume-assertions.md @@ -0,0 +1,468 @@ +--- +description: This page provides an overview of working with DataHub Volume Assertions +title: Volume Assertions +slug: /managed-datahub/observe/volume-assertions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/observe/volume-assertions.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Volume Assertions + + + +> The **Volume Assertions** feature is available as part of the **DataHub Cloud Observe** module of DataHub Cloud. +> If you are interested in learning more about **DataHub Cloud Observe** or trying it out, please [visit our website](https://datahub.com/products/data-observability/). + +## Introduction + +Can you remember a time when the meaning of Data Warehouse Table that you depended on fundamentally changed, with little or no notice? +If the answer is yes, how did you find out? We'll take a guess - someone looking at an internal reporting dashboard or worse, a user using your your product, sounded an alarm when +a number looked a bit out of the ordinary. Perhaps your table initially tracked purchases made on your company's e-commerce web store, but suddenly began to include purchases made +through your company's new mobile app. + +There are many reasons why an important Table on Snowflake, Redshift, BigQuery, or Databricks may change in its meaning - application code bugs, new feature rollouts, +changes to key metric definitions, etc. Often times, these changes break important assumptions made about the data used in building key downstream data products +like reporting dashboards or data-driven product features. + +What if you could reduce the time to detect these incidents, so that the people responsible for the data were made aware of data +issues _before_ anyone else? With DataHub Cloud **Volume Assertions**, you can. + +DataHub Cloud allows users to define expectations about the normal volume, or size, of a particular warehouse Table, +and then monitor those expectations over time as the table grows and changes. + +In this article, we'll cover the basics of monitoring Volume Assertions - what they are, how to configure them, and more - so that you and your team can +start building trust in your most important data assets. + +Let's get started! + +## Support + +Volume Assertions are currently supported for: + +1. Snowflake +2. Redshift +3. BigQuery +4. Databricks +5. DataHub Dataset Profile (collected via ingestion) + +Note that an Ingestion Source _must_ be configured with the data platform of your choice in DataHub Cloud's **Ingestion** +tab. + +> Note that Volume Assertions are not yet supported if you are connecting to your warehouse +> using the DataHub CLI. + +## What is a Volume Assertion? + +A **Volume Assertion** is a configurable Data Quality rule used to monitor a Data Warehouse Table +for unexpected or sudden changes in "volume", or row count. Volume Assertions can be particularly useful when you have frequently-changing +Tables which have a relatively stable pattern of growth or decline. + +For example, imagine that we work for a company with a Snowflake Table that stores user clicks collected from our e-commerce website. +This table is updated with new data on a specific cadence: once per hour (In practice, daily or even weekly are also common). +In turn, there is a downstream Business Analytics Dashboard in Looker that shows important metrics like +the number of people clicking our "Daily Sale" banners, and this dashboard is generated from data stored in our "clicks" table. +It is important that our clicks Table is updated with the correct number of rows each hour, else it could mean +that our downstream metrics dashboard becomes incorrect. The risk of this situation is obvious: our organization +may make bad decisions based on incomplete information. + +In such cases, we can use a **Volume Assertion** that checks whether the Snowflake "clicks" Table is growing in an expected +way, and that there are no sudden increases or sudden decreases in the rows being added or removed from the table. +If too many rows are added or removed within an hour, we can notify key stakeholders and begin to root cause before the problem impacts stakeholders of the data. + +### Anatomy of a Volume Assertion + +At the most basic level, **Volume Assertions** consist of a few important parts: + +1. An **Evaluation Schedule** +2. A **Volume Condition** +3. A **Volume Source** + +In this section, we'll give an overview of each. + +#### 1. Evaluation Schedule + +The **Evaluation Schedule**: This defines how often to check a given warehouse Table for its volume. This should usually +be configured to match the expected change frequency of the Table, although it can also be less frequently depending +on the requirements. You can also specify specific days of the week, hours in the day, or even +minutes in an hour. + +#### 2. Volume Condition + +The **Volume Condition**: This defines the type of condition that we'd like to monitor, or when the Assertion +should result in failure. + +There are a 2 different categories of conditions: **Total** Volume and **Change** Volume. + +_Total_ volume conditions are those which are defined against the point-in-time total row count for a table. They allow you to specify conditions like: + +1. **Table has too many rows**: The table should always have less than 1000 rows +2. **Table has too few rows**: The table should always have more than 1000 rows +3. **Table row count is outside a range**: The table should always have between 1000 and 2000 rows. + +_Change_ volume conditions are those which are defined against the growth or decline rate of a table, measured between subsequent checks +of the table volume. They allow you to specify conditions like: + +1. **Table growth is too fast**: When the table volume is checked, it should have < 1000 more rows than it had during the previous check. +2. **Table growth is too slow**: When the table volume is checked, it should have > 1000 more rows than it had during the previous check. +3. **Table growth is outside a range**: When the table volume is checked, it should have between 1000 and 2000 more rows than it had during the previous check. + +For change volume conditions, both _absolute_ row count deltas and relative percentage deltas are supported for identifying +table that are following an abnormal pattern of growth. + +#### 3. Volume Source + +The **Volume Source**: This is the mechanism that DataHub Cloud should use to determine the table volume (row count). The supported +source types vary by the platform, but generally fall into these categories: + +- **Information Schema**: A system Table that is exposed by the Data Warehouse which contains live information about the Databases + and Tables stored inside the Data Warehouse, including their row count. It is usually efficient to check, but can in some cases be slightly delayed to update + once a change has been made to a table. This is the optimal balance between cost and accuracy for most Data Platforms. + +- **Query**: A `COUNT(*)` query is used to retrieve the latest row count for a table, with optional SQL filters applied (depending on platform). + This can be less efficient to check depending on the size of the table. This approach is more portable, as it does not involve + system warehouse tables, it is also easily portable across Data Warehouse and Data Lake providers. This issues a query to the table, which can be more expensive than Information Schema. + +- **DataHub Dataset Profile**: The DataHub Dataset Profile aspect is used to retrieve the latest row count information for a table. + Using this option avoids contacting your data platform, and instead uses the DataHub Dataset Profile metadata to evaluate Volume Assertions. + Note if you have not configured a managed ingestion source through DataHub, then this may be the only option available. This is the cheapest option, but requires that Dataset Profiles are reported to DataHub. By default, Ingestion will report Dataset Profiles to DataHub, which can be and infrequent. You can report Dataset Profiles via the DataHub APIs for more frequent and reliable data. + +Volume Assertions also have an off switch: they can be started or stopped at any time with the click of button. + +## Creating a Volume Assertion + +### Prerequisites + +1. **Permissions**: To create or delete Volume Assertions for a specific entity on DataHub, you'll need to be granted the + `Edit Assertions` and `Edit Monitors` privileges for the entity. This will be granted to Entity owners as part of the `Asset Owners - Metadata Policy` + by default. + +2. (Optional) **Data Platform Connection**: In order to create a Volume Assertion that queries the source data platform directly (instead of DataHub metadata), you'll need to have an **Ingestion Source** configured to your + Data Platform: Snowflake, BigQuery, or Redshift under the **Integrations** tab. + +Once these are in place, you're ready to create your Volume Assertions! + +You can also apply Smart Volume Assertions at scale using [Monitoring Rules](/docs/managed-datahub/observe/data-health-dashboard.md#monitoring-rules) on the Data Health page. + +### Steps + +1. Navigate to the Table that to monitor for volume +2. Click the **Quality** tab + +

+ +

+ +3. Click **+ Create Assertion** + +

+ +

+ +4. Choose **Volume** + +5. Configure the evaluation **schedule**. This is the frequency at which the assertion will be evaluated to produce a pass or fail result, and the times + when the table volume will be checked. + +6. Configure the evaluation **condition type**. This determines the cases in which the new assertion will fail when it is evaluated. + +

+ +

+ +7. (Optional) Click **Advanced** to customize the volume **source**. This is the mechanism that will be used to obtain the table + row count metric. Each Data Platform supports different options including Information Schema, Query, and DataHub Dataset Profile. + +

+ +

+ +- **Information Schema**: Check the Data Platform system metadata tables to determine the table row count. +- **Query**: Issue a `COUNT(*)` query to the table to determine the row count. +- **DataHub Dataset Profile**: Use the DataHub Dataset Profile metadata to determine the row count. + +8. Configure actions that should be taken when the Volume Assertion passes or fails + +

+ +

+ +- **Raise incident**: Automatically raise a new DataHub `Volume` Incident for the Table whenever the Volume Assertion is failing. This + may indicate that the Table is unfit for consumption. Configure Slack Notifications under **Settings** to be notified when + an incident is created due to an Assertion failure. + +- **Resolve incident**: Automatically resolved any incidents that were raised due to failures in this Volume Assertion. Note that + any other incidents will not be impacted. + +9. Click **Next** and provide a description. + +10. Click **Save**. + +And that's it! DataHub will now begin to monitor your Volume Assertion for the table. + +Once your assertion has run, you will begin to see Success or Failure status for the Table + +

+ +

+ +## Anomaly Detection with Smart Assertions ⚡ + +As part of the **DataHub Cloud Observe** module, DataHub Cloud also provides **Smart Assertions** out of the box. These are +dynamic, AI-powered Volume Assertions that you can use to monitor the volume of important warehouse Tables, without +requiring any manual setup. + +You can create smart assertions by simply selecting the `Detect with AI` option in the UI: + +

+ +

+ +## Time-Series Bucketing + +By default, volume assertions evaluate the **total row count** of a table at a point in time. With **time-series bucketing**, you can partition your data into time-based buckets (e.g., daily or weekly) and evaluate volume metrics within each bucket. This fundamentally changes what the assertion measures: + +| | Without Bucketing | With Bucketing | +| -------------------- | ------------------------------------------ | ----------------------------------------------------------- | +| **Total row count** | Total table row count | Rows matching the timestamp-in-bucket condition | +| Example | "Total row count should stay above 10,000" | "Rows added per day should be above 500" | +| **Row count change** | Row count change since previous evaluation | Row count compared to previous bucket | +| Example | "Row count should grow by at least 100" | "Difference in rows added per week should not exceed 5,000" | + +Time-series bucketing is useful when: + +- Your table has a timestamp column that represents when rows were created or updated +- You want to monitor data quality at a day or week granularity rather than the whole table +- You want to detect issues like "no data arrived today" or "this week's volume is abnormally low" + +### Bucketing Configuration + +A time-series bucketing strategy consists of: + +- **Timestamp column**: The date/time column used to partition rows into buckets (e.g., `created_at`, `event_date`). +- **Bucket interval**: The size of each time bucket. Currently supported intervals are **Daily** (1 DAY) and **Weekly** (1 WEEK). +- **Timezone**: The IANA timezone for bucket boundaries (e.g., `America/Los_Angeles`, `UTC`). This should match your timestamp column's timezone. Defaults to UTC. +- **Late arrival grace period** (optional): A buffer after the bucket end time before the bucket is considered complete. This accounts for late-arriving data. For example, a 2-day grace period on a daily bucket means the bucket for Monday won't be evaluated until Thursday at midnight (instead of Tuesday at midnight). + +:::note +When time-series bucketing is enabled, the assertion's **evaluation schedule is automatically computed** based on the bucket interval and grace period. You do not need to (and cannot) manually set a cron schedule for bucketed assertions. +::: + +### Limitations + +- Only **Query** source type supports bucketing. Information Schema and DataHub Dataset Profile sources cannot be bucketed. +- Only single-unit bucket intervals are supported (1 DAY or 1 WEEK, not 2 DAYs). +- Bucketing configuration (timestamp column, bucket interval, timezone) **cannot be changed after creation**. The late arrival grace period can be updated. +- If a bucket is missed due to downtime, it will not be retroactively evaluated. + +### Configuring Bucketing in the UI + +When creating a volume assertion: + +

+ +

+ +1. In the assertion builder, expand the **Time-Series Bucketing** section. +2. Toggle bucketing **on**. +3. Select the **timestamp column** from the dropdown (filtered to date/time fields). +4. Choose the **bucket size** (Daily or Weekly). +5. Select a **timezone** that matches your timestamp column's timezone. +6. (Optional) Set a **grace period** to account for late-arriving data. + +### Configuring Bucketing via the Python SDK + +```python +from datahub.sdk import DataHubClient +from datahub.metadata.urns import DatasetUrn + +client = DataHubClient(server="", token="") +dataset_urn = DatasetUrn.from_string( + "urn:li:dataset:(urn:li:dataPlatform:snowflake,database.schema.table,PROD)" +) + +# Volume assertion with daily bucketing +volume_assertion = client.assertions.sync_volume_assertion( + dataset_urn=dataset_urn, + display_name="Daily Row Count Check", + criteria_condition="ROW_COUNT_IS_GREATER_THAN_OR_EQUAL_TO", + criteria_parameters=100, + detection_mechanism="information_schema", + time_bucketing_strategy={ + "timestamp_field_path": "created_at", + "bucket_interval": {"unit": "DAY", "multiple": 1}, + "timezone": "America/Los_Angeles", + "late_arrival_grace_period": {"unit": "DAY", "multiple": 2}, + }, + tags=["automated", "volume", "daily"], + enabled=True, +) + +# Smart volume assertion with weekly bucketing and backfill +smart_volume = client.assertions.sync_smart_volume_assertion( + dataset_urn=dataset_urn, + display_name="Weekly Volume Anomaly Monitor", + detection_mechanism="information_schema", + sensitivity="medium", + time_bucketing_strategy={ + "timestamp_field_path": "event_date", + "bucket_interval": {"unit": "WEEK", "multiple": 1}, + "timezone": "UTC", + }, + backfill_config={"backfill_start_date_ms": 1704067200000}, + enabled=True, +) +``` + +See the [Assertions SDK tutorial](/docs/api/tutorials/assertions.md) for more examples. + +:::info +For smart assertions with bucketing enabled, you can also configure **historical backfill** to populate the assertion's metrics history. See [Backfill Assertion History](./assertion-backfill.md) for details. +::: + +## Stopping a Volume Assertion + +In order to temporarily stop the evaluation of the assertion: + +1. Navigate to the **Quality** tab of the Table with the assertion +2. Click **Volume** to open the Volume Assertion assertions +3. Click the "Stop" button for the assertion you wish to pause. + +

+ +

+ +To resume the assertion, simply click **Start**. + +

+ +

+ +## Creating Volume Assertions via API + +Under the hood, DataHub Cloud implements Volume Assertion Monitoring using two concepts: + +- **Assertion**: The specific expectation for volume, e.g. "The table was changed int the past 7 hours" + or "The table is changed on a schedule of every day by 8am". This is the "what". + +- **Monitor**: The process responsible for evaluating the Assertion on a given evaluation schedule and using specific + mechanisms. This is the "how". + +Note that to create or delete Assertions and Monitors for a specific entity on DataHub, you'll need the +`Edit Assertions` and `Edit Monitors` privileges for it. + +#### GraphQL + +In order to create or update a Volume Assertion, you can use the `upsertDatasetVolumeAssertionMonitor` mutation. + +##### Examples + +To create a Volume Assertion Entity that verifies that the row count for a table is between 10 and 20 rows, and runs every 8 hours: + +```graphql +mutation upsertDatasetVolumeAssertionMonitor { + upsertDatasetVolumeAssertionMonitor( + input: { + entityUrn: "" + type: ROW_COUNT_TOTAL + rowCountTotal: { + operator: BETWEEN + parameters: { + minValue: { value: "10", type: NUMBER } + maxValue: { value: "20", type: NUMBER } + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +To create an AI Smart Freshness Assertion that runs every 8 hours: + +```graphql +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + input: { + entityUrn: "" + inferWithAI: true + type: ROW_COUNT_TOTAL + # you can provide any value here as it will be overwritten continuously by the AI engine + rowCountTotal: { + operator: BETWEEN + parameters: { + minValue: { value: "0", type: NUMBER } + maxValue: { value: "0", type: NUMBER } + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +The supported volume assertion types are `ROW_COUNT_TOTAL` and `ROW_COUNT_CHANGE`. Other (e.g. incrementing segment) types are not yet supported. +The supported operator types are `GREATER_THAN`, `GREATER_THAN_OR_EQUAL_TO`, `LESS_THAN`, `LESS_THAN_OR_EQUAL_TO`, and `BETWEEN` (requires minValue, maxValue). +The supported parameter types are `NUMBER`. + +You can use same endpoint with assertion urn input to update an existing Volume Assertion and corresponding Monitor: + +```graphql +mutation upsertDatasetVolumeAssertionMonitor { + upsertDatasetVolumeAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: ROW_COUNT_TOTAL + rowCountTotal: { + operator: BETWEEN + parameters: { + minValue: { value: "10", type: NUMBER } + maxValue: { value: "20", type: NUMBER } + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + evaluationParameters: { sourceType: INFORMATION_SCHEMA } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. + +### Tips + +:::info +**Authorization** + +Remember to always provide a DataHub Personal Access Token when calling the GraphQL API. To do so, just add the 'Authorization' header as follows: + +``` +Authorization: Bearer +``` + +**Exploring GraphQL API** + +Also, remember that you can play with an interactive version of the DataHub Cloud GraphQL API at `https://your-account-id.acryl.io/api/graphiql` +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md new file mode 100644 index 00000000..5cdc99ef --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md @@ -0,0 +1,143 @@ +--- +description: >- + This guide will walk through the configuration required to start receiving + DataHub Cloud events via AWS EventBridge. +title: Setting up Events API on AWS EventBridge +slug: /managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/operator-guide/setting-up-events-api-on-aws-eventbridge.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Setting up Events API on AWS EventBridge + + + +## Entity Events API + +- See the Entity Events API Docs [here](docs/managed-datahub/datahub-api/entity-events-api.md) + +## Event Structure + +As are all AWS EventBridge events, the payload itself will be wrapped by a set of standard fields, outlined [here](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-events.html). The most notable include + +- **source:** A unique identifier for the source of the event. We tend to use \`acryl.events\` by default. +- **account**: The account in which the event originated. This will be the DataHub Cloud AWS Account ID provided by your DataHub Cloud customersuccess rep. +- **detail**: The place where the Entity Event payload will appear. + +#### Sample Event + +``` +{ + "version": "0", + "id": "6a7e8feb-b491-4cf7-a9f1-bf3703467718", + "detail-type": "entityChangeEvent", + "source": "acryl.events", + "account": "111122223333", + "time": "2017-12-22T18:43:48Z", + "region": "us-west-1", + "detail": { + "entityUrn": "urn:li:dataset:abc", + "entityType": "dataset", + "category": "TAG", + "operation": "ADD", + "modifier": "urn:li:tag:pii", + "parameters": { + "tagUrn": "urn:li:tag:pii" + } + } +} +``` + +#### Sample Pattern + +``` +{ + "source": ["acryl.events"], + "detail": { + "category": ["TAG"], + "parameters": { + "tagUrn": ["urn:li:tag:pii"] + } + } +} +``` + +_Sample Event Pattern Filtering any Add Tag Events on a PII Tag_ + +## Step 1: Create an Event Bus + +We recommend creating a dedicated event bus for DataHub Cloud. To do so, follow the steps below: + +1\. Navigate to the AWS console inside the account where you will deploy Event Bridge. + +2\. Search and navigate to the **EventBridge** page. + +3\. Navigate to the **Event Buses** tab. + +3\. Click **Create Event Bus.** + +4\. Give the new bus a name, e.g. **acryl-events.** + +5\. Define a **Resource Policy** + +When creating your new event bus, you need to create a Policy that allows the DataHub Cloud AWS account to publish messages to the bus. This involves granting the **PutEvents** privilege to the DataHub Cloud account via an account id. + +**Sample Policy** + +``` +{ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "allow_account_to_put_events", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::795586375822:root" + }, + "Action": "events:PutEvents", + "Resource": "" + }] +} +``` + +Notice that you'll need to populate the following fields on your own + +- **event-bus-arn**: This is the AWS ARN of your new event bus. + +## Step 2: Create a Routing Rule + +Once you've defined an event bus, you need to create a rule for routing incoming events to your destinations, for example an SQS topic, a Lambda function, a Log Group, etc. + +To do so, follow the below steps + +1\. Navigate to the **Rules** tab. + +2\. Click **Create Rule**. + +3\. Give the rule a name. This will usually depend on the target where you intend to route requests matching the rule. + +4\. In the **Event Bus** field, select the event bus created in **Step 1**. + +5\. Select the 'Rule with Event Pattern' option + +6\. Click **Next.** + +7\. For **Event Source**, choose **Other** + +8\. \***\* Optional: Define a Sample Event. You can use the Sample Event defined in the **Event Structure\*\* section above. + +9\. Define a matching Rule. This determines which DataHub events will be routed based on the current rule. You can use the Sample Rule defined in the **Event Structure** section above as a reference. + +10\. Define a Target: This defines where the events that match the rule should be routed. + +## Step 3: Configure DataHub Cloud to Send Events + +Once you've completed these steps, communicate the following information to your DataHub Cloud CustomerSuccess rep: + +- The ARN of the new Event Bus. +- The AWS region in which the Event Bus is located. + +This will enable DataHub Cloud to begin sending events to your EventBridge bus. + +\_\_ diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md new file mode 100644 index 00000000..266b7fef --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md @@ -0,0 +1,402 @@ +--- +title: Configuring Remote Executor +description: >- + Learn how to set up, deploy, and configure Remote Executors in your + environment +sidebar_label: Configuring Remote Executor +slug: /managed-datahub/operator-guide/setting-up-remote-ingestion-executor +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md +--- + +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Configuring Remote Executor + + + +## Overview + +This guide will walk you through the process of setting up Remote Executors in your environment, including: + +1. Understanding Remote Executors and Pools +2. Configuration prerequisites +3. Creating and managing Executor Pools +4. Deploying Remote Executors in your environment +5. Assigning Ingestion Sources to Pools + +## Understanding Remote Executors and Pools + +A Remote Executor Pool provides a way to organize and manage your Remote Executors in DataHub. Here's how they work: + +- A **Remote Executor Pool** (or **Pool**) is a logical grouping of one or more Remote Executors +- Each **Remote Executor** in a Pool automatically reports its status and health +- Each **Ingestion Source** can be assigned to a specific Pool +- One Pool can be designated as the **Default Pool** for new Ingestion Sources +- Multiple Pools can be created for different purposes (e.g., by region, department, or workload type) + +## Configuration Prerequisites + +Before deploying a Remote Executor, ensure you have the following: + +1. **DataHub Cloud** + + - A DataHub User with the **Manage Metadata Ingestion** Platform privilege + - A DataHub Remote Executor Access Token (generate from **Settings > Access Tokens > Generate new token > Remote Executor**) + - Your DataHub Cloud URL (e.g., `.acryl.io/gms`). **NOTE:** you MUST include the trailing `/gms` when configuring the executor. + +2. **Deployment Environment** + + - Access to your deployment platform (AWS ECS or Kubernetes) + - Necessary permissions to create resources + +3. **Network Connectivity** + + The Remote Executor requires **outbound** HTTPS (port 443) connectivity only — no inbound connectivity is needed. Ensure the following endpoints are reachable from your deployment environment: + + - `https://.acryl.io/*` — DataHub GMS API + - `https://sqs.*.amazonaws.com/*` — AWS SQS, used for remote execution task dispatch + - A Python package index (e.g., `https://pypi.org`) or an alternate internal mirror, to download pip packages required by ingestion sources + - A container registry hosting the DataHub Remote Executor image (e.g., AWS ECR or `docker.datahub.com`) + +4. **Registry Access** + - For AWS: Provide your AWS account ID to DataHub Cloud + - For Kubernetes: Work with DataHub team to set up access to the Remote Executor Docker Image Registry + +## Creating and Managing Executor Pools + +Complete the following steps to create a new Executor Pool from the DataHub Cloud UI: + +1. Navigate to the **Data Sources** section on the left sidebar +2. Click the **Executors** tab and click **Create** + +

+ +

+ +3. Configure Pool settings: + - **Pool Identifier**: A unique identifier for this Pool + - **Description**: Purpose or details about the Pool + - **Default Pool**: Optionally set as the Default Pool for new Ingestion Sources + +

+ +

+ +4. Click **Create** to provision the Pool + +## Deploying Remote Executors in Your Environment + +Once you have created an Executor Pool in DataHub Cloud, you are now ready to deploy an Executor within your environment. + +:::note +Work with DataHub team to receive deployment templates specific to your environment (Helm charts, CloudFormation, or Terraform) for deploying Remote Executors in this Pool. +::: + +### Deploy on Amazon ECS + +1. **AWS Account Configuration** + +To access the private DataHub Cloud ECR registry, you'll need to provide your AWS account ID to DataHub Cloud. You can securely share your account ID through: + +- Your DataHub Cloud representative +- A secure secret-sharing service like [One Time Secret](https://onetimesecret.com/) + +This step is required to grant your AWS account access to pull the Remote Executor container image. + +2. **Configure CloudFormation Template** + +The DataHub Team will provide a [Cloudformation Template](https://raw.githubusercontent.com/acryldata/datahub-cloudformation/master/remote-executor/datahub-executor.ecs.template.yaml) that you can run to provision an ECS cluster with a single remote ingestion task. It will also provision an AWS role for the task which grants the permissions necessary to read and delete from the private queue created for you, along with reading the secrets you've specified. At minimum, the template requires the following parameters: + +- Deployment Location (VPC and subnet) +- DataHub Personal Access Token +- DataHub Cloud URL (e.g., `.acryl.io/gms`) +- Executor Pool ID you set in the DataHub UI +- Optional: DataHub Cloud Remote Executor Version; defaults to latest + +Optional parameters: + +- Source Secrets: `SECRET_NAME=SECRET_ARN` (up to 10); separate multiple secrets by comma, e.g. `SECRET_NAME_1=SECRET_ARN_1,SECRET_NAME_2,SECRET_ARN_2`. +- Environment Variables: `ENV_VAR_NAME=ENV_VAR_VALUE` (up to 10); separate multiple variable by comma, e.g. `ENV_VAR_NAME_1=ENV_VAR_VALUE_1,ENV_VAR_NAME_2,ENV_VAR_VALUE_2`. + +:::note +Configuring Secrets enables you to manage ingestion sources from the DataHub UI without storing credentials inside DataHub. Once defined, secrets can be referenced by name inside of your DataHub Ingestion Source configurations using the usual convention: `${SECRET_NAME}`. +::: + +3. **Deploy Stack** + + ```bash + + # Using AWS CLI + + aws --region us-east-1 cloudformation create-stack \ + --stack-name datahub-remote-executor \ + --template-body file://datahub-executor.ecs.template.yaml \ + --capabilities CAPABILITY_AUTO_EXPAND CAPABILITY_NAMED_IAM \ + --parameters ParameterKey=ExecutorPoolId,ParameterValue="remote" \ + ParameterKey=VPCID,ParameterValue="" \ + ParameterKey=SubnetID,ParameterValue="" \ + ParameterKey=DataHubBaseUrl,ParameterValue="https://.acryl.io/gms" \ + ParameterKey=DataHubAccessToken,ParameterValue="" + ``` + + Or use the [CloudFormation Console](https://console.aws.amazon.com/cloudformation) + +4. **Configure Secrets (Optional)** + + ```bash + + # Create a secret in AWS Secrets Manager + + aws secretsmanager create-secret \ + --name my-source-secret \ + --secret-string '{"username":"user","password":"pass"}' + ``` + +### Update Amazon ECS Deployment + +To update your Remote Executor deployment (e.g., to deploy a new container version or modify configuration), you'll need to update your existing CloudFormation Stack. This process involves re-deploying the CloudFormation template with your updated parameters while preserving your existing resources. + +1. **Access CloudFormation** + + - Navigate to the AWS CloudFormation Console + - Locate and select your Remote Executor stack + - Click the **Update** button + +2. **Update Template** + - Select **Replace current template** + - Choose **Upload a template file** + - Download the latest DataHub Cloud Remote Executor [CloudFormation Template](https://raw.githubusercontent.com/acryldata/datahub-cloudformation/master/remote-executor/datahub-executor.ecs.template.yaml) + - Upload the template file + +

+ +

+ +3. **Configure Parameters** + + - Review and update parameters as needed: + - `ImageTag`: Specify a new version if upgrading + - `DatahubGmsURL`: Verify your DataHub URL is correct + - Other parameters will retain their previous values unless changed + - Click **Next** to proceed + +4. **Review and Deploy** + - Review the configuration changes + - Acknowledge any capabilities if prompted + - Click **Update stack** to begin the update process + +:::note +The update process will maintain your existing resources (e.g., secrets, IAM roles) while deploying the new configuration. Monitor the stack events to track the update progress. +::: + +### Deploy on Kubernetes + +The [datahub-executor-worker](https://executor-helm.acryl.io/index.yaml) Helm chart provides a streamlined way to deploy Remote Executors on any Kubernetes cluster, including Amazon EKS and Google GKE. + +1. **Registry Access Configuration** + +To access the private DataHub Cloud container registry, you'll need to work with your DataHub Cloud representative to set up the necessary permissions: + +For AWS EKS: Provide the IAM principal that will pull from the ECR repository + +- For Google Cloud: Provide the cluster's IAM service account +- For other platforms: Contact DataHub team for specific requirements + +2. **Configure Secrets** + +Create the required secrets in your Kubernetes cluster: + +```bash +# Create DataHub PAT secret (required) +# Generate token from Settings > Access Tokens in DataHub UI +kubectl create secret generic datahub-access-token-secret \ + --from-literal=datahub-access-token-secret-key= + +# Create source credentials (optional) +kubectl create secret generic datahub-secret-store \ + --from-literal=REDSHIFT_PASSWORD=password \ + --from-literal=SNOWFLAKE_PASSWORD=password +``` + +3. **Install Helm Chart** + +Add the DataHub Cloud Helm repository and install the chart: + +```bash +# Add Helm repository +helm repo add acryl https://executor-helm.acryl.io +helm repo update + +# Install the chart +helm install \ + --set global.datahub.executor.pool_id="remote" \ + --set global.datahub.gms.url="https://.acryl.io/gms" \ + acryl-executor-worker acryl/datahub-executor-worker +``` + +Required parameters: + +- `global.datahub.executor.pool_id`: Your Executor Pool ID +- `global.datahub.gms.url`: Your DataHub Cloud URL (must include `/gms`) + +4. **Configure Secret Mounting (Optional)** + +Starting from DataHub Cloud v0.3.8.2, you can manage secrets using Kubernetes Secret CRDs. This enables runtime secret updates without executor restarts. + +Create a Kubernetes secret: + +```yaml +# secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: datahub-secret-store +data: + REDSHIFT_PASSWORD: + SNOWFLAKE_PASSWORD: +``` + +Mount the secret in your `values.yaml`: + +```yaml +extraVolumes: + - name: datahub-secret-store + secret: + secretName: datahub-secret-store +extraVolumeMounts: + - mountPath: /mnt/secrets + name: datahub-secret-store +``` + +:::note +Secret Configuration: + +- Default mount path: `/mnt/secrets` (override with `DATAHUB_EXECUTOR_FILE_SECRET_BASEDIR`) +- Default file size limit: 1MB (override with `DATAHUB_EXECUTOR_FILE_SECRET_MAXLEN`) +- Reference secrets in ingestion recipes using `${SECRET_NAME}` syntax + +::: + +Example ingestion recipe using mounted secrets: + +```yaml +source: + type: redshift + config: + host_port: "" + username: connector_test + password: "${REDSHIFT_PASSWORD}" + # ... other configuration ... +``` + +For additional configuration options, refer to the [values.yaml](https://github.com/acryldata/datahub-executor-helm/blob/main/charts/datahub-executor-worker/values.yaml) file in the Helm chart repository. + +### Update Kubernetes Deployment + +To update your Kubernetes deployment (e.g., to deploy a new image version or modify configuration), you'll need to upgrade your existing Helm release. This process involves upgrading the Helm release with any new parameters while preserving your existing parameters. + +1. **Upgrade Helm release** + +```bash +# Update Helm repository +helm repo update acryl + +# Upgrade your existing Helm release +# See https://helm.sh/docs/helm/helm_upgrade/ for more options +helm upgrade \ + --reuse-values \ + --set ="" \ # if any new options need to be set + acryl-executor-worker acryl/datahub-executor-worker +``` + +For configuration options, refer to the [values.yaml](https://github.com/acryldata/datahub-executor-helm/blob/main/charts/datahub-executor-worker/values.yaml) file in the Helm chart repository. + +## Checking Remote Executor status + +Once you have successfully deployed the Executor in your environment, DataHub will automatically begin reporting Executor Status in the UI: + +

+ +

+ +## Assigning Ingestion Sources to an Executor Pool + +After you have created an Executor Pool and deployed the Executor within your environment, you are now ready to configure an Ingestion Source to run in that Pool. + +1. Navigate to **Manage Data Sources** in DataHub Cloud +2. Edit an existing Source or click **Create new source** +3. In the **Finish Up** step, expand the **Advanced** to select your desired **Executor Pool** + +

+ +

+ +:::note +New Ingestion Sources will automatically use your designated Default Pool if you have assigned one. You can override this assignment when creating or editing an Ingestion Source at any time. +::: + +5. Click **Save & Run**. The task should show as 'Running' if properly configured. + +

+ +

+ +## Advanced: Performance Settings and Task Weight-Based Queuing + +Executors use a weight-based queuing system to manage resource allocation efficiently: + +- **Default Behavior**: With 4 ingestion threads (default), each task gets a weight of 0.25, allowing up to 4 parallel tasks +- **Resource-Intensive Tasks**: Tasks can be assigned a higher weight (up to 1.0) to limit parallelism +- **Queue Management**: If the total weight of running tasks exceeds 1.0, new tasks are queued until capacity becomes available +- **Priority Tasks**: Setting a weight of 1.0 ensures exclusive resource access - the task will run alone until completion + +The following environment variables can be configured to manage memory-intensive ingestion tasks, prevent resource contention, and ensure stable execution of resource-demanding processes: + +- `DATAHUB_EXECUTOR_INGESTION_MAX_WORKERS` (default: 4) - Maximum concurrent Ingestion tasks +- `DATAHUB_EXECUTOR_MONITORS_MAX_WORKERS` (default: 10) - Maximum concurrent Observe monitoring tasks +- `EXECUTOR_TASK_MEMORY_LIMIT` - Memory limit per task in kilobytes, configured per Ingestion Source under **Extra Environment Variables**. This setting helps prevent the executor's master process from being OOM-killed and protects against memory-leaking ingestion tasks. Example configuration: + ```json + { "EXECUTOR_TASK_MEMORY_LIMIT": "128000000" } + ``` +- `EXECUTOR_TASK_WEIGHT` - Task weight for resource allocation, configured per Ingestion Source under **Extra Environment Variables**. By default, each task is assigned a weight of 1/MAX_THREADS (e.g., 0.25 with 4 threads). The total weight of concurrent tasks cannot exceed 1.0. Example configuration for a resource-intensive task: + ```json + { "EXECUTOR_TASK_WEIGHT": "1.0" } + ``` + +## Troubleshooting + +### Common Issues + +1. **Connection Failed** + + - Verify network connectivity + - Check DataHub URL configuration + - Validate Executor Pool ID + - Validate access token + +2. **Secret Access Failed** + + - Confirm secret ARNs/names + - Check permissions + - Verify secret format + +3. **Container Failed to Start** + - Check resource limits + - Verify registry access + - Review container logs + +### Frequently Asked Questions + +**Do AWS Secrets Manager secrets automatically update in the executor?** + +No. Secrets are wired into the executor container at deployment time. The ECS Task needs to be restarted when secrets change. + +**How can I verify successful deployment?** + +For ECS deployments, check AWS Console: + +1. Navigate to **ECS > Cluster > Stack Name > Services > Logs** +2. Look for the log line: `Starting datahub executor worker` + +This indicates successful connection to DataHub Cloud. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/next.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/next.md new file mode 100644 index 00000000..b54d0a1a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/next.md @@ -0,0 +1,42 @@ +--- +title: Next +slug: /managed-datahub/release-notes/next +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/next.md +--- +# Next + +:::info + + + +::: + +#### Release Availability Date + +TBD + +#### Recommended Versions + +- **CLI/SDK**: TBD +- **Remote Executor**: TBD +- **On-Prem Versions**: + - **Helm**: TBD + - **API Gateway**: TBD + - **Actions**: TBD + +## Release Changelog + +### Next + +New Features: + +- TODO + +Fixes: + +- TODO + +## Known Issues + +- TODO diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_69.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_69.md new file mode 100644 index 00000000..c6597b2d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_69.md @@ -0,0 +1,21 @@ +--- +title: v0.1.69 +slug: /managed-datahub/release-notes/v_0_1_69 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_1_69.md +--- +# v0.1.69 + +--- + +This is a scheduled release which contains all changes from OSS DataHub upto commit `10a31b1aa08138c616c0e44035f8f843bef13085`. In addition to all the features added in OSS DataHub below are DataHub Cloud specific release notes. + +## Release Availability Date + +06 Dec 2022 + +## Release Changlog + +--- + +- We now support >10k results in Metadata Test results diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_70.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_70.md new file mode 100644 index 00000000..26af94de --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_70.md @@ -0,0 +1,22 @@ +--- +title: v0.1.70 +slug: /managed-datahub/release-notes/v_0_1_70 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_1_70.md +--- +# v0.1.70 + +--- + +This is a scheduled release which contains all changes from OSS DataHub upto commit `70659711a841bcce4bb1e0350027704b3783f6a5`. In addition to all the features added in OSS DataHub below are DataHub Cloud specific release notes. + +## Release Availability Date + +30 Dec 2022 + +## Release Changlog + +--- + +- Improvements in Caching implementation to fix search consistency problems +- We have heard many organisations ask for metrics for the SaaS product. We have made good progress towards this goal which allows us to share Grafana dashboards. We will be testing it selectively. Expect more updates in coming month on this. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_72.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_72.md new file mode 100644 index 00000000..f62fb05a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_72.md @@ -0,0 +1,27 @@ +--- +title: v0.1.72 +slug: /managed-datahub/release-notes/v_0_1_72 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_1_72.md +--- +# v0.1.72 + +--- + +## Release Availability Date + +18 Jan 2023 + +## Release Changlog + +--- + +- Since `v0.1.70` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/43c566ee4ff2ee950a4f845c2fd8a1c690c1d607...afaee58ded40dc4cf39f94f1b4331ceb0a4d93eb have been pulled in +- add GZip compression to lineage cache +- Make browse paths upgrade non-blocking + +## Special Notes + +--- + +- If anyone faces issues with login please clear your cookies. Some security updates are part of this release. That may cause login issues until cookies are cleared. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_73.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_73.md new file mode 100644 index 00000000..170ab0ea --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_1_73.md @@ -0,0 +1,22 @@ +--- +title: v0.1.73 +slug: /managed-datahub/release-notes/v_0_1_73 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_1_73.md +--- +# v0.1.73 + +--- + +## Release Availability Date + +01 Fev 2023 + +## Release Changlog + +--- + +- Since `v0.1.72` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/afaee58ded40dc4cf39f94f1b4331ceb0a4d93eb...36afdec3946df2fb4166ac27a89b933ced87d00e have been pulled in +- Fixes related to Metadata tests to ensure correct results +- Adding Properties and Searchable Annotations to Usage + Storage Features +- Fixes delete references for single relationship aspects diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_0.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_0.md new file mode 100644 index 00000000..c09ca027 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_0.md @@ -0,0 +1,34 @@ +--- +title: v0.2.0 +slug: /managed-datahub/release-notes/v_0_2_0 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_2_0.md +--- +# v0.2.0 + +--- + +## Release Availability Date + +09 Feb 2023 + +## Update Downtime + +During release installation the Elasticsearch indices will be reindex to improve search capabilities. While the upgrade is in progress +DataHub will be set to a read-only mode. Once this operation is completed, the upgrade will proceed normally. Depending on index sizes and +infrastructure this process can take 5 minutes to hours however as a rough estimate 1 hour for every 2.3 million entities. + +## Release Changlog + +--- + +- Since `v0.1.73` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/36afdec3946df2fb4166ac27a89b933ced87d00e...v0.10.0 have been pulled in + - Improved documentation editor + - Filter lineage graphs based on time windows + - Improvements in Search + - Metadata Ingestion + - Redshift: You can now extract lineage information from unload queries + - PowerBI: Ingestion now maps Workspaces to DataHub Containers + - BigQuery: You can now extract lineage metadata from the Catalog + - Glue: Ingestion now uses table name as the human-readable name +- SSO Preferred Algorithm Setting diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_1.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_1.md new file mode 100644 index 00000000..3c16b268 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_1.md @@ -0,0 +1,24 @@ +--- +title: v0.2.1 +slug: /managed-datahub/release-notes/v_0_2_1 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_2_1.md +--- +# v0.2.1 + +--- + +## Release Availability Date + +23-Feb-2023 + +## Release Changlog + +--- + +- Since `v0.2.0` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/cf1e627e55431fc69d72918b2bcc3c5f3a1d5002...36037cf288eea12f1760dd0718255eeb1d7039c7 have been pulled in. +- Add first, last synched + last updated properties to metadata tests. +- Update link colors to pass accessibility. +- Extend tag and term proposals to other entity types besides datasets. This allows proposals to work on entities other than datasets. +- We are skipping running metadata tests in real-time processing as it was not scaling out and causing issues in ingestion +- Re-enabling hard-deletes which was temporarily disabled diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_10.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_10.md new file mode 100644 index 00000000..c6864b57 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_10.md @@ -0,0 +1,40 @@ +--- +title: v0.2.10 +slug: /managed-datahub/release-notes/v_0_2_10 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_2_10.md +--- +# v0.2.10 + +--- + +## Release Availability Date + +09-Aug-2023 + +## Recommended CLI/SDK + +- `v0.10.5.5` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.10.5.5 + +If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better. + +Special Notes + +- We have a new search and browse experience. We cannot enable it unless all of your CLI/SDK usages are upgraded. If you are using a custom source then you need to upgrade your source to produce `browsePathv2` aspects. Details are in [this doc](../../browseV2/browse-paths-v2.md). +- [Breaking change] For all sql-based sources that support profiling, you can no longer specify `profile_table_level_only` together with `include_field_xyz` config options to ingest certain column-level metrics. Instead, set `profile_table_level_only` to false and individually enable / disable desired field metrics. +- [Breaking change] The `bigquery-beta` and `snowflake-beta` source aliases have been dropped. Use `bigquery` and `snowflake` as the source type instead. +- [Behaviour change] Ingestion runs created with Pipeline.create will show up in the DataHub ingestion tab as CLI-based runs. To revert to the previous behavior of not showing these runs in DataHub, pass `no_default_report=True`. +- [Behaviour change] snowflake connector will use user's email attribute as is, as the urn. To revert to previous behavior disable `email_as_user_identifier` in recipe. + +## Release Changelog + +--- + +- Since `v0.2.9` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/1f0723fad109658a69bb1d4279100de8514f35d7...2b0952195b7895df0a2bf92b28e71aac18217781 have been pulled in. + +## Some notable features in this SaaS release + +- New search and Browse v2 experience. This can only be enabled if you upgrade all your CLI/SDK usage as per our recommendation provided above. +- We will be enabling these features selectively. If you are interested in trying it and providing feedback, please reach out to your DataHub Cloud CustomerSuccess representative. + - DataHub Cloud Observe Freshness Assertions available in private beta as shared [here](../observe/freshness-assertions.md). + - New notifications and Subscriptions feature available. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_11.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_11.md new file mode 100644 index 00000000..df8bd0ba --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/release-notes/v_0_2_11.md @@ -0,0 +1,82 @@ +--- +title: v0.2.11 +slug: /managed-datahub/release-notes/v_0_2_11 +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/release-notes/v_0_2_11.md +--- +# v0.2.11 + +--- + +## Release Availability Date + +14-Sep-2023 + +## Recommended CLI/SDK + +- `v0.11.0` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.11.0 +- [Deprecation] In LDAP ingestor, the manager_pagination_enabled changed to general pagination_enabled + +If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better. + +## Special Notes + +- Deployment process for this release is going to have a downtime when systme will be in a read only mode. A rough estimate 1 hour for every 2.3 million entities (includes soft-deleted entities). + +## Release Changelog + +--- + +- Since `v0.2.10` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/2b0952195b7895df0a2bf92b28e71aac18217781...75252a3d9f6a576904be5a0790d644b9ae2df6ac have been pulled in. +- Misc fixes & features + - Proposals + - Group names shown correctly for proposal Inbox + - Metadata tests + - Deprecate/Un-deprecate actions available in Metadata tests + - Last Observed (in underlying sql) available as a filter in metadata tests + - [Breaking change] Renamed `__lastUpdated` -> `__created` as a filter to correctly represent what it was. This was not surfaced in the UI. But if you were using it then this needs to be renamed. DataHub Cloud CustomerSuccess team will keep an eye out to pro-actively find and bring this up if you are affected by this. + - Robustness improvements to metadata test runs + - Copy urn for metadata tests to allow for easier filtering for iteration over metadata test results via our APIs. + - A lot more fixes to subscriptions, notifications and Observability (Beta). + - Some performance improvements to lineage queries + +## Some notable features in this SaaS release + +- We now enable you to create and delete pinned announcements on your DataHub homepage! If you have the “Manage Home Page Posts” platform privilege you’ll see a new section in settings called “Home Page Posts” where you can create and delete text posts and link posts that your users see on the home page. +- Improvements to search experience +
+
+ +### Steps to Install + +The following steps should be performed by a Slack Workspace Admin. + +1. Navigate to [https://api.slack.com/apps](https://api.slack.com/apps) +2. Click **Create App**, then select **'Generate Token'** +

+ +

+3. Select your workspace, then hit **'Generate'** +

+ +

+4. Now you will see two tokens available for you to copy, an _Access Token_ and a _Refresh Token_ +

+ +

+5. Navigate back to your DataHub Slack Integration setup page, and paste the tokens into their respective boxes, and click **'Connect'**. +

+ +

+6. You will be automatically re-directed to Slack to confirm DataHub Slack App's permissions and complete the installation process. + +:::note +You may need approval from a workspace admin to do this step. Learn about what to do in this scenario [below](#workspace-admin-approval-guide). +::: + +

+ +

+7. Congrats 🎉 Slack is set up! Now try it out by going to the **Platform Notifications** page +

+ +

+8. Enter your channel in, and click **'Send a test notification'** +

+ +

+ +Now proceed to [connect your Slack account](#connect-your-slack-account) so you can use Subscriptions & Notifications and Ask DataHub, or visit the [Slack App page](saas-slack-app.md) to learn more about DataHub's Slack capabilities. + +### DataHub Slack Bot Permissions + +The DataHub Slack bot requires a certain set of Slack scopes to function properly. + +
+View all required Slack bot scopes + +| Scope | Purpose | +| ----------------------- | ----------------------------------------------------------------------------------------------- | +| `commands` | Required for slash commands / shortcuts | +| `app_mentions:read` | Required to get @DataHub messages | +| `chat:write` | Required to send messages as @DataHub | +| `chat:write.public` | Required to post in public channels the bot hasn't joined | +| `chat:write.customize` | Allows using a custom icon so messages display the DataHub Cloud logo | +| `channels:history` | Required to read message history in public channels | +| `channels:read` | Required to see public channel details | +| `groups:history` | Required to read message history in private channels | +| `groups:read` | Required to see private channel details | +| `im:history` | Required to read direct message history | +| `im:read` | Required to see direct message details | +| `mpim:history` | Required to read group DM history | +| `mpim:read` | Required to see group DM details | +| `metadata.message:read` | Required to read message metadata | +| `team:read` | Required to get workspace ID and create links to user profiles | +| `channels:join` | Allows the bot to join a public channel when someone configures notifications to be sent to one | +| `links:read` | Required to unfurl links | +| `links:write` | Required to unfurl links | +| `users:read` | Required to resolve user IDs to names/emails | +| `users:read.email` | Required to enable user lookup by email address | +| `reactions:read` | Future-proofing | +| `reactions:write` | Future-proofing | + +
+ +### Workspace Admin Approval + +In some workspaces, you will find at step 6 above you will need approval from your workspace admin. In this case, you will want to: + +1. On step #6 above, continue by clicking the button to request their approval. +2. Once approved, you will get a notification from the **Slack bot** that your request has been approved. Follow the link it provides to complete the process. + **If you do not get the message:** visit [api.slack.com/apps](https://api.slack.com/apps), open your DataHub app and complete the installation from there. +3. Once it is done, you can visit `/settings/notifications`, and send a test notification to verify everything's working. +4. **Making Slack commands work:** when running Slack commands such as `/datahub search my dataset` you may get a `dispatch_failed` error. To resolve this, see [the steps here](/docs/managed-datahub/slack/saas-slack-troubleshoot#slack-bot-issues). + + + +## Connect Your Slack Account + +:::info +Starting in v0.3.17, all users are required to connect their DataHub account to Slack via OAuth in order to use **Subscriptions & Notifications** and **Ask DataHub**. This ensures that DataHub can securely link your DataHub user to your corresponding Slack account, and use your credentials when performing actions on DataHub. +::: + +Connecting your Slack account is a one-time setup. Once connected, DataHub will be able to securely identify you across both platforms. + +### Steps to Connect + +1. Navigate to **Settings > Notifications** in DataHub. You will see a **Connect to Slack** button under Slack Notifications. +

+ +

+ +2. Click the **Connect to Slack** button. You will be redirected to Slack's OAuth login page. + +3. Sign in to your Slack workspace if prompted, and authorize DataHub to access your Slack identity. + +4. After authorizing, you will be redirected back to DataHub. Your DataHub account is now linked to your Slack account. + +If you try to use Ask DataHub in Slack before connecting your account, the bot will prompt you to connect first: + +

+ +

+ +Once connected, you can use Subscriptions & Notifications and Ask DataHub. If you ever need to reconnect (e.g. you changed Slack workspaces), you can return to this page and click **Reconnect**. + +:::note Migrating from Manual Slack User ID Configuration +If you previously configured your Slack User ID manually (see [legacy instructions below](#how-to-find-user-id-in-slack-legacy)), your existing setup will continue to work until you connect your account using the new OAuth flow. Once you connect via OAuth, DataHub will use the OAuth-linked account going forward. Starting in v0.3.17, manual Slack User ID configuration for subscriptions & notifications is no longer available. +::: + +## Configure Subscriptions & Notifications + +We support sending notifications to + +- Slack Channel Name (e.g. `#troubleshoot`) +- Slack Channel ID (e.g. `C029A3M079U`) +- Specific Users (once you've connected your account) + +By default, the Slack app will be able to send notifications to public channels. If you want to send notifications to private channels or DMs, you will need to invite the Slack app to those channels. + +Learn more about how [subscriptions and notifications work](../subscription-and-notification.md), including what things you can be notified about and all the places you can receive notifications. + +### How to find Team ID and Channel ID in Slack + +:::note +We recommend just using the Slack channel name for simplicity (e.g. `#troubleshoot`). +::: + +**Via Slack App:** + +1. Go to the Slack channel for which you want to get a channel ID +2. Click the channel name at the top +

+ +

+3. At the bottom of the modal that pops up, you will see the Channel ID as well as a button to copy it +

+ +

+ +**Via Web:** + +1. Go to the Slack channel for which you want to get a channel ID +2. Check the URL e.g. for the troubleshoot channel in OSS DataHub Slack + ![](https://raw.githubusercontent.com/datahub-project/static-assets/main/imgs/integrations/slack/slack_channel_url.png) + +3. Notice `TUMKD5EGJ/C029A3M079U` in the URL + +- Team ID = `TUMKD5EGJ` from above +- Channel ID = `C029A3M079U` from above + +### How to find User ID in Slack (Legacy) + +:::caution Deprecated +Manual Slack User ID configuration is deprecated as of v0.3.17. Please use the [Connect Your Slack Account](#connect-your-slack-account) flow instead. If you previously configured your Slack User ID manually, it will continue to work until you connect via OAuth. +::: + +**Your User ID** + +1. Click your profile picture, then select **'Profile'** +

+ +

+2. Now hit the **'...'** and select **'Copy member ID'** +

+ +

+ +**Someone else's User ID** + +1. Click their profile picture in the Slack message +

+ +

+2. Now hit the **'...'** and select **'Copy member ID'** +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/slack/saas-slack-troubleshoot.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/slack/saas-slack-troubleshoot.md new file mode 100644 index 00000000..82c095c6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/slack/saas-slack-troubleshoot.md @@ -0,0 +1,111 @@ +--- +title: Troubleshoot Slack Issues +slug: /managed-datahub/slack/saas-slack-troubleshoot +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/slack/saas-slack-troubleshoot.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Troubleshoot Slack Issues + + + +This document provides troubleshooting guidance for the Slack integration. For more details on setting up the Slack integration, [click here](./saas-slack-setup.md). + +## Prerequisites - Testing the Slack Integration + +First and foremost, we recommend using the 'Send a test notification' feature to verify whether the issue is with the integration setup, slack's systems, or DataHub. The modal will provide a rich description of an error if there is one. +You can access this feature either by going to the Notifications page in your settings, or a subscription drawer. + +

+ +

+ +## Test notification failed with 'Re-Connect DataHub to Slack' + +There are several reasons why sending a test notification would fail. The description in the modal should give you insights as to what's broken and what you can do to resolve this issue. +If you're seeing a message that recommends the DataHub admin to re-connect Slack to DataHub, you may want to try the following options: + +### Refresh the existing app installation (Recommended) + +:::note +Whomever originally installed the Slack app will need to perform this. +If they are unable to do this, you may need to go down the 'Install a new app' path below. +::: + +1. Get your App Config tokens by following the first few steps outlined in the [installation guide](/docs/managed-datahub/slack/saas-slack-setup/#step-by-step-guide). If it's showing expired tokens, feel free to delete them and create a new set. +2. Paste them into their respective text inputs, and hit **'Re-connect'** +

+ +

+3. You will be re-directed to a page where you can finalize the app refresh. + +### Install a new app + +:::note +If you choose to install a new app, your team will have to re-add the new bot into any private channels the old one was previously in. +If you'd like support in getting a list of the private channels that are subscribed to Slack notifications on DataHub, please reach out to your customer success representative. +::: + +1. Get your App Config tokens by following the first few steps outlined in the [installation guide](/docs/managed-datahub/slack/saas-slack-setup/#step-by-step-guide). If it's showing expired tokens, feel free to delete them and create a new set. +2. Paste them into their respective text inputs, and hit **'create a new installation'** +

+ +

+3. You will be re-directed to a page where you can finalize the app installation. +4. Now to uninstall the old app, visit the **'Manage Apps'** page for your Slack workspace. +

+ +

+5. Find the previously installed DataHub Slack bot in the list of installed apps, and open it. +6. Open the 'App details' page +

+ +

+7. Then, switch to the Configuration tab +

+ +

+8. Finally, scroll to the bottom to find the remove button. +

+ +

+ +## Test notification works, but not receiving notifications + +There are a few reasons why you may not receive notifications when you'd expect. + +### Actors do not receive notifications for their actions + +If you've subscribed to an entity, and then performed an action (i.e., raised an incident or added a tag), you will not be notified about your own action. + +### There is an issue with DataHub's systems + +If sending a test notification works, and you've verified that none of the above cases apply, then you should contact your DataHub Customer Success rep to help troubleshoot and resolve the issue. + +## Other issues + +Below you'll find some tips to troubleshoot issues with your Slack bot. + +### Command failed with error "dispatch_failed" + +If you've installed the Slack bot, but your commands are failing with an error 'dispatch_failed', you can try a **Manual Installation Refresh**, as outlined below. + +### Manual Installation Refresh + +1. Open your DataHub cloud instance with the following url: `/settings/integrations/slack?display_all_configs=true`. +2. Switch to the **Bot Token** tab. +

+ +

+3. Refill in the Bot Token and Signing Secret, even if the values are already present. +4. Visit [api.slack.com/apps](https://api.slack.com/apps), and open your currently installed app. You will see fields like `App ID` and `Signing Secret` here: +

+ +

+5. You can get your `Bot Token` from the **OAuth & Permissions** tab in the side nav. +

+ +

+6. Paste the values in and hit **Update Configuration**. +7. Test the Slack command now and it should work. If it still fails, please reach out to your DataHub Cloud admin to troubleshoot further. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/subscription-and-notification.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/subscription-and-notification.md new file mode 100644 index 00000000..ce518699 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/subscription-and-notification.md @@ -0,0 +1,207 @@ +--- +title: Subscriptions & Notifications +slug: /managed-datahub/subscription-and-notification +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/subscription-and-notification.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Subscriptions & Notifications + + + +DataHub's Subscriptions and Notifications feature gives you real-time change alerts on data assets of your choice. +Currently, DataHub supports notifications on Slack, Microsoft Teams, and Email. Subscriptions can be created for both individual users and groups. + +## Prerequisites + +Email subscriptions & notifications are enabled by default. + +To install the Slack App, see: +👉 [Configure Slack for Notifications](slack/saas-slack-setup.md). + +To install the Microsoft Teams App, see: +👉 [Configure Microsoft Teams for Notifications](teams/saas-teams-setup.md). + +

+ +

+ +## Enabling Notifications + +### Personal Notifications + +To manage personal notification settings, go to Settings > "My Notifications". +From here, enable or disable each notification channel & provide your handle. + +### Group Notifications + +To manage group notifications, navigate to the group page, and select the **Notifications** tab. +Here, you can enable or disable each notification channel & provide relevant handles. + +If you want to create and manage group-level Subscriptions for your team, you will need [the following privileges](../../docs/authorization/roles.md#role-privileges): + +- Manage Group Notification Settings +- Manage Group Subscriptions + +**Admin-only:** And to manage other user's subscriptions: + +- Manage User Subscriptions + +## Using DataHub’s Subscriptions and Notifications Feature + +The first step to getting notified is identifying the assets you want to subscribe to. +DataHub’s [Lineage and Impact Analysis features](../../docs/act-on-metadata/impact-analysis.md#lineage-impact-analysis-setup-prerequisites-and-permissions) can help you identify upstream entities that could impact the assets you use and are responsible for. +You can use the Subscriptions & Notifications feature to sign up for updates for your entire team, or just for yourself. + +### Individually Subscribing to an Entity + +Select the **Subscribe Me** option in the Subscriptions dropdown menu. + +

+ +

+ +Pick the updates you want to be notified about, and connect your Slack account by using your Slack Member ID. + +

+ +

+ +:::note +You can find your Slack Member ID in your profile settings. + +

+ +

+::: + +### Subscribing Your Team/Group to Notifications + +The dropdown menu next to the Subscribe button lets you choose who the subscription is for. To create a group subscription, click on **Manage for Groups**. + +

+ +

+ +Next, customize the group's subscription by selecting the types of changes you want the group to be notified about. + +

+ +

+ +Connect to Slack. Currently, DataHub Cloud's Subscriptions and Notifications feature integrates only with Slack. Add your group’s Slack Channel ID to receive notifications on Slack. +(You can find your Channel ID in the About section of your channel on Slack.) + +

+ +

+ +### Managing Your Subscriptions + +You can enable, disable, or manage notifications at any time to ensure that you receive relevant updates. + +Simply use the Dropdown menu next to the Subscribe button to unsubscribe from the asset, or to manage/modify your subscription (say, to modify the changes you want to be updated about). + +You can also view and manage your subscriptions in your DataHub settings page. + +

+ +

+ +You can view and manage the group’s subscriptions on the group’s page on DataHub. + +

+ +

+ +### Subscribing to Assertions + +You can always subscribe to _all assertion status changes_ on a table using the steps outlined in the earlier sections. However, in some cases you may want to only be notified about specific assertions on a table. For instance, a table may contain several subsets of information, segmented by a category column - so there may be several different checks for each category. As a consumer, you may only care about the freshness check that runs on one specific category of this larger table. + +You can subscribe to individual assertions by clicking the bell button on the assertion itself - either in the list view: + +

+ 1 +

+ +Or on the assertion's profile page: + +

+ 2 +

+ +Note: if you are subscribed to all assertions at the dataset level, then you will not be able to **Unsubscribe** from an individual assertion. + +

+ 3 +

+ +You must first remove your dataset-level subscription: + +

+ 4 + 5 +

+ +Then select individual assertions you'd like to subscribe to: + +

+ 7 +

+ +## Programmatically Managing Subscriptions + +You can create and remove subscriptions programmatically using the [GraphQL APIs](/docs/api/graphql/overview.md) or the [Python Subscriptions SDK](/docs/api/tutorials/subscriptions.md). + +## FAQ + +
+ +What changes can I be notified about using this feature? + +You can subscribe to deprecations, Assertion status changes, Incident status changes, Schema changes, Ownership changes, Glossary Term changes, and Tag changes. +

+ +

+
+ +
+ +What if I no longer want to receive updates about a data asset? + +You can unsubscribe from any asset to stop receiving notifications about it. On the asset’s DataHub page, simply use the dropdown menu next to the Subscribe button to unsubscribe from the asset. + +

+ +

+
+ +
+ +What if I want to be notified about different changes? + +To modify your subscription, use the dropdown menu next to the Subscribe button to modify the changes you want to be notified about. +
+
+ +I want to configure multiple channels. How many Slack channels or emails can I configure to get notified? + +At the platform-level, you can configure one email and one Slack channel. + +At the user and group -levels, you can configure one default email and Slack channel as well as overwrite that email/channel when you +go to a specific asset to subscribe to. + +To configure multiple channels, as a prereq, ensure you have the appropriate privileges. And then: + +1. Create a datahub group for each channel you want notifications for. +2. Add yourself as a member to each of the groups. +3. Now, when you visit an asset and go to subscribe, you'll see the option "Manage Group Subscriptions". + +
+ +## Reference + +- [DataHub Blog - Simplifying Data Monitoring & Management with Subscriptions and Notifications with DataHub Cloud](https://www.acryldata.io/blog/simplifying-data-monitoring-and-management-with-subscriptions-and-notifications-with-acryl-datahub) +- Video Guide - Getting Started with Subscription & Notifications + diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-app.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-app.md new file mode 100644 index 00000000..0a4fe6ef --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-app.md @@ -0,0 +1,57 @@ +--- +title: Microsoft Teams App Features +slug: /managed-datahub/teams/saas-teams-app +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/teams/saas-teams-app.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Microsoft Teams App Features + + + +The DataHub Microsoft Teams App brings several of DataHub's key capabilities directly into your Teams experience. +The integration enables your team to: + +1. Ask DataHub about your data (by tagging @DataHub) +2. Get notified when Data Assets change +3. Manage Data Incidents + +The Teams App makes data discovery easier and more accessible, by making DataHub available where you work. +Learn more about [how to set up the Teams app](./saas-teams-setup.md). + +## Ask DataHub in Teams + +Simply mention @DataHub in any channel or chat to ask questions about your data. Some popular use cases include: + +- Search for data assets using natural language. +- Understand the impact of changes to data assets. +- Dig into specific asset context - their glossary terms, owners, and more. +- Write SQL queries to perform specific analysis. + +

+ Example Ask DataHub in Microsoft Teams. +

+ +## Get Notified + +To get personal notifications via Teams, you must first [connect your teams account](./saas-teams-setup.md#connecting-your-personal-teams-account). + +The DataHub Teams app enables you to be notified in specific Teams channels or direct messages when important changes to your Data Assets occur. +Notification preferences & subscriptions [can be configured](../subscription-and-notification.md) in the DataHub UI once the [Teams app is set up](./saas-teams-setup.md). + +

+ Example DataHub notifications in Microsoft Teams. +

+ +## Manage Data Incidents + +One of the most common use cases for the Teams app is incident management. +When an incident is raised, you will get notified about its impact, priority, and more. +You will also be able to directly resolve the incident, or dive deeper into DataHub, directly from Teams. + +To learn more about DataHub incident management, check out the [Incidents](/docs/incidents/incidents/) feature guide. + +

+ Example of incident notifications in Teams. +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-setup.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-setup.md new file mode 100644 index 00000000..32b85b4b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/teams/saas-teams-setup.md @@ -0,0 +1,79 @@ +--- +title: Microsoft Teams App Setup +slug: /managed-datahub/teams/saas-teams-setup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/teams/saas-teams-setup.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Microsoft Teams App Setup + + + +## Installing the DataHub Microsoft Teams App + +To enable the DataHub Teams app, you will need to install the DataHub Teams bot into your Teams workspace. +DataHub Teams app is currently in **Private Beta** - you will need to manually upload the Teams app package (a ZIP file) to your Teams workspace. + +The following steps should be performed by a Teams or Azure Administrator: + +### Creating the DataHub App in Teams + +1. Contact your DataHub Customer Success Rep for access to the Teams Private Beta. + +2. Open [Teams Admin UI](https://admin.teams.microsoft.com/) in your web browser, and in the sidebar click **Teams Apps** > **Manage Apps** +

Connected to Microsoft Teams.

+ +3. On the Apps page, click **Actions** > **Upload new app** +

Connected to Microsoft Teams.

+ +4. In the "Upload a custom app" window, click **Upload**, and then in the file browser, select the ZIP file you downloaded in Step #1. +

Connected to Microsoft Teams.

+ +5. Once the app is uploaded, log out of Teams Admin and User UIs, wait 5 minutes, and then log back into [Teams Web UI](https://teams.microsoft.com/v2/) in a new tab. +

Connected to Microsoft Teams.

+ +6. On the sidebar in the Teams Web UI, click **Apps** > **Built for your org** +

Connected to Microsoft Teams.

+ +7. On **Built for your org** page, locate **DataHub** application in the list, and click **Add**. +

Connected to Microsoft Teams.

+

Connected to Microsoft Teams.

+ +8. Confirm installation by clicking **Add** in the pop-up +

Connected to Microsoft Teams.

+ +9. In the pop-up, click **Open** to open a chat window with the DataHub Teams bot. +

Connected to Microsoft Teams.

+ +### Connecting the App to Your DataHub Instance + +10. **First, get your team URL:** On the sidebar, click **Chat**, locate your team name in the list, and click "..." next to open a dropdown manu. Then click **Copy Link** +

Connected to Microsoft Teams.

+ +11. **Then, link in DataHub:** In a separate web browser tab, open your DataHub UI, go to **Settings** > **Integrations** and click **Teams** +

Connected to Microsoft Teams.

+ +12. Paste the URL obtained is Step #10 into the Teams URL field, and click **Connect to Teams** +

Connected to Microsoft Teams.

+ +13. When prompted, complete the Teams authentication flow. After completion, you should be redirected back to DataHub. + +Your Teams App is now ready to use! Add the DataHub bot to any channel in your Teams workspace, and start asking questions about your data. + +## Connecting Your Personal Teams Account + +To receive personal notifications via teams (e.g. for asset subscriptions), you'll need to connect your Teams account to your DataHub user profile. +This can be done by navigating to **Settings** > **My Notifications** and enable Teams notifications by clicking the toggle +switch on the right hand side. + +

+ Connect to Microsoft Teams. +

+ +Next, click **Connect to Teams** to be redirected to teams, where you can bind your account. +Upon successfully linking accounts, you'll be redirected to DataHub. + +

+ Connected to Microsoft Teams. +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/upgrade_core_to_cloud.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/upgrade_core_to_cloud.md new file mode 100644 index 00000000..2fbd322b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/upgrade_core_to_cloud.md @@ -0,0 +1,268 @@ +--- +title: Upgrading from DataHub Core to DataHub Cloud +slug: /managed-datahub/upgrade_core_to_cloud +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/upgrade_core_to_cloud.md +--- +# Upgrading from DataHub Core to DataHub Cloud + +Looking to upgrade to **DataHub Cloud**, but don't have an account yet? Start [here](https://datahub.com/demo/). + +Once you have a **DataHub Cloud** instance, you can seamlessly transfer all metadata from your self-hosted **DataHub Core** instance +to **DataHub Cloud** using the DataHub CLI. In this guide, we'll show you how. + +## Prerequisites + +Before starting the upgrade process: + +1. **DataHub Cloud Account**: Ensure you have an active DataHub Cloud instance with an API token +2. **Database Access**: You'll need read access to your DataHub Core MySQL or PostgreSQL database +3. **DataHub CLI**: Install the DataHub CLI with `pip install acryl-datahub` +4. **Network Connectivity**: Ensure your upgrade environment can access both your source database and DataHub Cloud +5. **Database Index**: Verify that the `createdon` column is indexed in your source database (should by for newer versions by default) + +## Moving From Core To Cloud + +DataHub supports lifting and shifting your instance from DataHub Core to DataHub Cloud, if you'd like to retain the information already present in your DataHub Core instance. To transfer your instance cleanly, you can follow the steps below. + +### Transferring Core Data + +You can easily copy core metadata from DataHub Core to DataHub Cloud using a simple CLI command. + +By default, we'll transfer: + +- Data assets (datasets, dashboards, charts, etc.) +- Users and groups +- Lineage +- Descriptions +- Ownership +- Domains and data products +- Tags and glossary terms + +The following is NOT automatically transferred: + +- Ingestion Sources +- Ingestion Source Runs +- Ingestion Secrets +- Platform Settings + +Due to the different encryption scheme employed on DataHub Cloud. This method also excludes time-series metadata such as dataset profiles, column statistics, and assertion run history. + +#### Step 1: Create Your Upgrade Recipe + +Create a file named `upgrade_recipe.yml` with the following configuration: + +```yaml +pipeline_name: datahub_cloud_upgrade +source: + type: datahub + config: + # Disable version history to transfer only current state + include_all_versions: false + + # Configure your source database connection + database_connection: + # For MySQL + scheme: "mysql+pymysql" + # For PostgreSQL, use: "postgresql+psycopg2" + + host_port: "your-database-host:3306" # MySQL default port + username: "your-datahub-username" + password: "your-datahub-password" + database: "datahub" # Default database name + + # Disable stateful ingestion for one-time transfer. + # If you intend to incrementally sync over time, should enable this! + stateful_ingestion: + enabled: false + +# Preserve system metadata during transfer +flags: + set_system_metadata: true + +# Configure DataHub Cloud as destination +sink: + type: datahub-rest + config: + server: "https://your-instance.acryl.io" + token: "your-datahub-cloud-api-token" +``` + +#### Step 2: Run the Upgrade + +Execute the upgrade using the DataHub CLI: + +```bash +datahub ingest -c upgrade_recipe.yml +``` + +The upgrade will display progress as it transfers your metadata. Depending on the size of your catalog, this process can take anywhere from minutes to hours. + +#### Step 3: Verify the Upgrade + +After completion: + +1. Log into your DataHub Cloud instance +2. Navigate to the Browse page to verify your assets +3. Check a few key datasets to ensure documentation, owners, and tags transferred correctly +4. Verify lineage relationships are intact + +### Transferring Time-Series Metadata + +For a complete transfer including recent time-series metadata, you'll need to provide additional configurations to connect to your **Kafka** cluster. This will transfer: + +- Dataset and column profiling history +- Dataset update history (inserts, updates, deletes) +- Dataset query statistics (query counts) +- Assertion run results + +**Important**: The amount of historical data available depends on your Kafka retention policy (typically 30-90 days). + +#### Extended Recipe Configuration + +```yaml +pipeline_name: datahub_upgrade_with_timeseries +source: + type: datahub + config: + include_all_versions: false + + # Database connection (same as quickstart) + database_connection: + scheme: "mysql+pymysql" + host_port: "your-database-host:3306" + username: "your-datahub-username" + password: "your-datahub-password" + database: "datahub" + + # Kafka configuration for time-series data + kafka_connection: + bootstrap: "your-kafka-broker:9092" + schema_registry_url: "http://your-schema-registry:8081" + consumer_config: + # Optional: Add security configuration if needed + # security.protocol: "SASL_SSL" + # sasl.mechanism: "PLAIN" + # sasl.username: "your-username" + # sasl.password: "your-password" + + # Topic containing time-series data (change if doesn't match default name) + kafka_topic_name: "MetadataChangeLog_Timeseries_v1" + + # Disable stateful ingestion for one-time transfer. + # If you intend to incrementally sync over time, should enable this! + stateful_ingestion: + enabled: false + +flags: + set_system_metadata: true + +sink: + type: datahub-rest + config: + server: "https://your-instance.acryl.io" + token: "your-datahub-cloud-api-token" +``` + +### Advanced Configuration: Transferring Specific Aspects + +You can override the default aspects which are excluded from transfer during upgrade using the `exclude_aspects` configuration. This is useful if you want to be more restrictive, opting to exclude certain information from being transferred to +your Cloud instance. + +**Be careful! Some aspects**, particularly those containing encrypted secrets, will NOT transfer to DataHub Cloud due to differences in encryption schemes. + +```yaml +source: + type: datahub + config: + # ... other config ... + + # Exclude specific aspects from transfer + exclude_aspects: + - dataHubIngestionSourceInfo + - datahubIngestionCheckpoint + - dataHubExecutionRequestInput + - dataHubIngestionSourceKey + - dataHubExecutionRequestResult + - globalSettingsInfo + - datahubIngestionRunSummary + - dataHubExecutionRequestSignal + - globalSettingsKey + - testResults + - dataHubExecutionRequestKey + - dataHubSecretValue + - dataHubSecretKey + # Add any other aspects you want to exclude +``` + +To learn about all aspects in DataHub, check out the [DataHub metadata model documentation](/docs/metadata-modeling/metadata-model/). + +## Best Practices + +### Performance Optimization + +1. **Batch Size**: For large transfers, adjust the batch configuration: + + ```yaml + source: + type: datahub + config: + database_query_batch_size: 10000 # Adjust based on your system + commit_state_interval: 1000 # Records before progress is saved to DataHub + ``` + +2. **Destination Settings**: For optimal performance on DataHub Cloud: + + - Ensuring batch async ingestion is enabled by setting `mode: ASYNC_BATCH` in the sink section of your recipe (enabled by default) + - Increase thread count if needed in sink settings by adjusting the `max_threads` parameter in the sink section of your recipe + +Check out the [sink docs](/docs/metadata-ingestion/sink_docs/datahub#config-details) to learn about other configuration parameters you may want to use during the upgrade process. + +4. **Stateful Ingestion**: For very large instances, use stateful ingestion: + + ```yaml + stateful_ingestion: + enabled: true + ignore_old_state: false # Set to true to restart from beginning! + ``` + + This enables you to upgrade incrementally over time, only syncing changes once the initial upgrade has completed. + +### Troubleshooting + +Common issues and solutions: + +- **Authentication Errors**: Verify your API token has write permissions +- **Network Timeouts**: Check firewall rules and consider adjusting `query_timeout` +- **Memory Issues**: Reduce `database_query_batch_size` for large transfers +- **Slow Performance**: Ensure the `createdon` index exists on your source database +- **Parse Errors**: Set `commit_with_parse_errors: true` to continue despite errors + +### Error Handling + +By default, the upgrade job will stop committing checkpoints if errors occur, allowing you to re-run and catch missed data. + +However, in some cases it's not possible to transfer data to DataHub Cloud, particularly if you've forked and extended the DataHub +metadata model. + +To continue making progress, ignoring errors: + +```yaml +source: + type: datahub + config: + commit_with_parse_errors: true # Continue even with parse errors +``` + +## Post-Upgrade Steps + +1. **Configure Data Sources**: Configure your ingestion sources on DataHub Cloud +2. **Configure SSO**: Set up authentication for your team +3. **Update API Clients**: Update any client applications to point to your DataHub Cloud instance +4. **Review Policies**: Recreate any custom policies and roles as needed + +## Additional Resources + +For more detailed configuration options, refer to the [DataHub source documentation](https://datahubproject.io/docs/generated/ingestion/sources/datahub). + +Need help? Contact DataHub Cloud support or visit our [community Slack](https://datahubproject.io/slack). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/welcome-acryl.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/welcome-acryl.md new file mode 100644 index 00000000..59d60019 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/welcome-acryl.md @@ -0,0 +1,63 @@ +--- +title: Getting Started with DataHub Cloud +slug: /managed-datahub/welcome-acryl +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/welcome-acryl.md +--- +# Getting Started with DataHub Cloud + +Welcome to the DataHub Cloud! We at DataHub are on a mission to make data reliable by bringing clarity to the who, what, when, & how of your data ecosystem. We're thrilled to be on this journey with you; and cannot wait to see what we build together! + +Close communication is not only welcomed, but highly encouraged. For all questions, concerns, & feedback, please reach out to us directly at help@acryl.io. + +## Prerequisites + +Before you go further, you'll need to have a DataHub instance provisioned. The DataHub integrations team will provide you the following once it has been deployed: + +1. The URL for your DataHub Cloud instance (https://your-domain-name.acryl.io) +2. Admin account credentials for logging into the DataHub UI + +Once you have these, you're ready to go. + +:::info +If you wish to have a private connection to your DataHub instance, DataHub Cloud supports [AWS PrivateLink](https://aws.amazon.com/privatelink/) to complete this connection to your existing AWS account. Please see more details [here](integrations/aws-privatelink.md). +::: + +### Logging In + +DataHub Cloud currently supports the following means to log into a DataHub instance: + +1. **Admin account**: When your subscriptions starts someone will share with you a invite link to create your admin account. If that has not happened please reach out at help@acryl.io through your official email ID / slack connect setup with our team and our team will share with you the invite url. +2. **OIDC**: DataHub Cloud also supports OIDC integration with the Identity Provider of your choice (Okta, Google, etc). To set this up, DataHub integrations team will require the following: +3. _Client ID_ - A unique identifier for your application with the identity provider +4. _Client Secret_ - A shared secret to use for exchange between you and your identity provider. To send this over securely, we recommend using [onetimesecret.com](https://onetimesecret.com/) to create a link. +5. _Discovery URL_ - A URL where the OIDC API of your identity provider can be discovered. This should suffixed by `.well-known/openid-configuration`. Sometimes, identity providers will not explicitly include this URL in their setup guides, though this endpoint will exist as per the OIDC specification. For more info see [here](http://openid.net/specs/openid-connect-discovery-1_0.html). + +The callback URL to register in your Identity Provider will be + +``` +https://your-acryl-domain.acryl.io/callback/oidc +``` + +_Note that we do not yet support LDAP or SAML authentication. Please let us know if either of these integrations would be useful for your organization._ + +## Getting Started + +DataHub Cloud is first and foremost a metadata Search & Discovery product. As such, the two most important parts of the experience are + +1. Ingesting metadata +2. Discovering metadata + +### Ingesting Metadata + +DataHub Cloud employs a push-based metadata ingestion model. In practice, this means running an DataHub-provided agent inside your organization's infrastructure, and pushing that data out to your DataHub instance in the cloud. One benefit of this approach is that metadata can be aggregated across any number of distributed sources, regardless of form or location. + +This approach comes with another benefit: security. By managing your own instance of the agent, you can keep the secrets and credentials within your walled garden. Skip uploading secrets & keys into a third-party cloud tool. + +To push metadata into DataHub, DataHub Cloud provides an ingestion framework written in Python. Typically, push jobs are run on a schedule at an interval of your choosing. For our step-by-step guide on ingestion, click [here](../../metadata-ingestion/cli-ingestion.md). + +### Discovering Metadata + +There are 2 primary ways to find metadata: search and browse. Both can be accessed via the DataHub home page. + +By default, we provide rich search capabilities across your ingested metadata. This includes the ability to search by tags, descriptions, column names, column descriptions, and more using the global search bar found on the home page. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/workflows/access-workflows.md b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/workflows/access-workflows.md new file mode 100644 index 00000000..2432a843 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/managed-datahub/workflows/access-workflows.md @@ -0,0 +1,430 @@ +--- +title: Data Access Workflows +slug: /managed-datahub/workflows/access-workflows +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/managed-datahub/workflows/access-workflows.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Data Access Workflows + + + + + + + +> **Note**: Access Workflows is currently in **Private Beta**. To enable this feature, please reach out to the DataHub team. + +Data Access Workflows enable organizations to create centralized approval processes for all data access requests, ensuring compliance while streamlining the request and review experience. + +## Key Capabilities + +**Create Approval Workflows**: Design access request workflows with custom fields, entry points, and multi-step approval processes. Define which types of assets users can request access to and configure routing rules for reviewers. + +**Keep Everyone in the Loop**: Stay informed with notifications via email or Slack when you have requests to review, or when a request you created is approved or denied. Monitor all your open requests directly in your Task Center. + +**Realtime, Event-Oriented Integration**: Provision access in real-time by tuning into events emitted when access requests are created or reviewed via the DataHub Actions Framework. + +## Typical Use Cases + +- **Centralized Access Requests**: Enable access requests across all data systems for tables and dashboards, or higher-level groups like domains, data products, and databases - all through a single interface +- **Audit Log**: View history of all access requests in one place to audit access grants over time and maintain compliance records +- **Dynamic Approval Routing**: Automatically route requests to appropriate reviewers based on asset ownership or organizational structure +- **Data Access Governance**: Implement controlled access to sensitive datasets with proper approval chains and organizational policies + +## Typical Users + +- **Data Platform Administrators**: Configure and manage workflow definitions +- **Data Stewards**: Review and approve access requests for their domains +- **Data Consumers**: Submit access requests for datasets they need +- **Integration Engineers**: Set up automated provisioning based on workflow events + +## Access Workflows Setup, Prerequisites, and Permissions + + + +### Prerequisites + +- Access Workflows feature must be enabled by the DataHub team (Private Beta) +- DataHub instance with GraphQL API access +- Administrative privileges to create workflow definitions +- Configured notification channels (Email/Slack) for workflow notifications - enabled by default + +### Required Permissions + +- **Create Workflows**: Platform Admin or user with `Manage Workflows` privilege +- **Submit Requests**: Any authenticated user (subject to workflow visibility rules) +- **Review Requests**: Users assigned as reviewers in workflow definitions +- **Configure Notifications**: Individual users can configure their own notification preferences + + + +## Creating an Access Workflow + +For each Access Workflow, you can configure the following: + +- **Form Fields**: Define what data needs to be collected from the user within the access request, with full control over which fields are required versus optional to match your organization's needs + +- **Form Entry Points**: Specify exactly where workflows should be activated, allowing you to display workflows on the home page or specific asset profiles. + +- **Approval Chain**: Designate specific individuals or groups who have the authority to review and finalize approvals. This can be dynamically assigned to a selected asset's owners, domain owners, or data product owners. + +Currently, workflow creation is must be done via the GraphQL API using the `upsertActionWorkflow` mutation. + +Here's a Python example that uses the DataHub Python client to create a basic dataset approval workflow: + +
+Create Access Workflow In Python + +```python +from datahub.ingestion.graph.client import DatahubClientConfig, DataHubGraph + +# Initialize DataHub client +config = DatahubClientConfig( + server="http://your-datahub-instance", + token="YOUR_ACCESS_TOKEN" +) +graph = DataHubGraph(config) + +# GraphQL mutation for creating an approval workflow +CREATE_WORKFLOW_MUTATION = """ +mutation upsertActionWorkflow($input: UpsertActionWorkflowInput!) { + upsertActionWorkflow(input: $input) { + urn + } +} +""" + +workflow_definition = { + "name": "Dataset Access Request", + "description": "Request access to sensitive datasets", + "category": "ACCESS", + "trigger": { + "type": "FORM_SUBMITTED", + "form": { + "entityTypes": ["DATASET"], # Limit to dataset entities, but can apply to many types. + "entrypoints": [ + { + "type": "HOME", # Display on Home Page + "label": "Request Dataset Access" # Home Page CTA + }, + { + "type": "ENTITY_PROFILE", # Display on Entity Profile Page + "label": "Request Access" # Entity Profile Page CTA + } + ], + "fields": [ + { + "id": "business_justification", + "name": "Business Justification", + "description": "Please explain why you need access to this dataset", + "valueType": "RICH_TEXT", + "cardinality": "SINGLE", + "required": True + }, + { + "id": "access_duration", + "name": "Access Duration", + "description": "How long do you need access?", + "valueType": "STRING", + "allowedValues": [ + {"stringValue": "30_DAYS"}, + {"stringValue": "90_DAYS"}, + {"stringValue": "PERMANENT"} + ], + "cardinality": "SINGLE", + "required": False + }, + # Create a conditionally visible field. Only visible based on previous field answer. + { + "id": "permanent_access_justification", + "name": "Permanent Access Justification", + "description": "Since you've requested permanent access, please provide additional justification for why this is necessary", + "valueType": "RICH_TEXT", + "cardinality": "SINGLE", + "required": True, + "condition": { + "type": "SINGLE_FIELD_VALUE", + "singleFieldValueCondition": { + "field": "access_duration", + "values": ["PERMANENT"], + "condition": "EQUAL", + "negated": False + } + } + } + ] + } + }, + "steps": [ + { + "id": "data_steward_review", + "type": "APPROVAL", + "description": "Data steward review and approval", + "actors": { + "userUrns": ["urn:li:corpuser:data.steward"], + "groupUrns": [], + "roleUrns": [], + "dynamicAssignment": { + "type": "ENTITY_OWNERS" + } + } + } + ] +} + +# Workflow definition +workflow_input = { + "input": workflow_definition +} + +# Execute the mutation +try: + result = graph.execute_graphql( + query=CREATE_WORKFLOW_MUTATION, + variables=workflow_input + ) + print(f"Workflow created successfully: {result['upsertActionWorkflow']['urn']}") + print(f"Workflow name: {result['upsertActionWorkflow']['name']}") +except Exception as e: + print(f"Error creating workflow: {e}") +``` + +
+ +### Workflow Concepts + +**Entry Points**: Define where users can initiate the workflow + +- `HOME`: Workflow appears on the home page to all users +- `ENTITY_PROFILE`: Workflow appears on entity detail pages + +**Field Types**: Supported form field types + +- `STRING`: Single-line text input +- `RICH_TEXT`: Multi-line rich text input with formatting +- `URN`: Entity reference (user, group, dataset, etc.) with configurable entity types +- `DATE`: Date/time value represented as epoch timestamp in milliseconds +- `NUMBER`: Numeric input (integer or float) + +**Field Cardinality**: Controls whether fields accept single or multiple values + +- `SINGLE`: Field accepts only one value +- `MULTIPLE`: Field accepts multiple values + +**Assignee Resolution**: Configure who reviews workflow requests + +- **Static Assignment**: Assign specific users, groups, and roles to review + - `userUrns`: Specific users assigned to review + - `groupUrns`: Specific groups assigned to review + - `roleUrns`: Specific DataHub roles assigned to review +- **Dynamic Assignment**: Automatically resolve reviewers based on entity context + - `ENTITY_OWNERS`: Assign to the owners of the requested entity + - `ENTITY_DOMAIN_OWNERS`: Assign to the owners of the entity's domain + - `ENTITY_DATA_PRODUCT_OWNERS`: Assign to the owners of the entity's data product + - Optional `ownershipTypes`: Filter by specific ownership types (e.g., Technical Owner, Business Owner) + +**Categories**: + +- `ACCESS`: Access-related workflows +- `CUSTOM`: Custom workflows with user-defined `customCategory` string. For example, to model data creation requests. + +## Submitting Access Workflow Request + +Once a workflow is created, users will be able to submit a Workflow Request form to trigger the review process. Depending on the entry point specified for the workflow, users will be able to start the workflow from either the + +1. Home Page +2. Entity Profile + +To create an approval workflow request, users must simply provide responses for all required fields. + +Once completed, it can be submitted by clicking "Submit". + +

+ +

+ +Once a request is submitted, your open requests will be visible from within **Tasks** > **Requests** > **My Requests**. + +### Reviewing Approval Workflow Requests + +Users assigned as reviewers can manage requests through the **Task Center**: + +### Accessing the Task Center + +1. From the Navigation sidebar, click "Tasks" +2. Click on the "Requests" tab + +### Reviewing a Request + +1. **View Request Details**: Click "View Details" on any pending request to see: + - Requestor information + - Selected entity + - Submitted form responses + +

+ +

+ +2. **Make a Decision**: For each request, you can: + + - **Accept**: Grant the access request + - **Reject**: Deny the request with optional comments + +3. **Add Comments**: Provide context for your decision to help requestors understand the outcome + +

+ +

+ +## Getting Notified About Access Workflows + +To stay informed about workflow activities, you can configure notifications: + +1. Navigate to **Settings** > **My Notifications** +2. Find the **Workflow Notifications** section +3. Enable notifications in the following cases: + - **Pending Reviews**: Get notified when you have requests to review + - **Request Updates**: Get notified when your submitted requests are approved or denied + +## Access Provisioning: Reacting to Approval Workflow Events + +Integrate with the [DataHub Actions Framework](/docs/actions) to automate access provisioning based on workflow events. + +### Event Types + +DataHub emits events for key workflow lifecycle moments: + +1. **Approval Workflow Request Create Event**: When a workflow request form is submitted +2. **Approval Workflow Request Step Complete**: When an intermediate review step is completed +3. **Approval Workflow Request Complete**: When the workflow request is fully approved or rejected + +The format of each JSON event can be found by visiting the + +### Setting Up Event Integration + +Create a custom [DataHub Actions](/docs/actions) listener for workflow events to trigger custom access provisioning. + +To get started by printing these events out: + +``` +name: "access-workflow-provisioner-action" +datahub: + server: "your-datahub-server" + token: "your-access-token" +source: + type: "datahub-cloud" +# Add filter to filter down to just access request lifecycle events. +filter: + event_type: "EntityChangeEvent_v1" + event: + entityType: "actionRequest" + category: "LIFECYCLE" + operation: "COMPLETED" # OR CREATE OR MODIFY + parameters: + actionRequestType: "WORKFLOW_FORM_REQUEST" +action: + type: "hello_world" + config: {} +``` + +In reality, this action would likely respond by making the changes required to provision access for the requesting user. + +For example, by: + +- Adding the user to a specific LDAP group +- Assigning a particular Snowflake role for the user +- Generating an AWS IAM policy granting access to an S3 bucket + +For full documentation on building a custom action, check out [Developing an Action](/docs/actions/guides/developing-an-action). + +### Event Schema Reference + +For event schemas & examples, see [Entity Change Events](/docs/actions/events/entity-change-event). + +## Additional Resources + + + +### GraphQL + +- **upsertActionWorkflow** - Create or update workflow definitions +- **createActionWorkflowFormRequest** - Submit workflow requests +- **reviewActionWorkflowFormRequest** - Review workflow requests +- **deleteActionWorkflow** - Delete workflow definitions + +To see the full schema types and documentation, visit GraphiQL at https://your-datahub-instance.acryl.io/api/graphiql and view definitions under the `Mutation` type. + +## FAQ and Troubleshooting + + + +**How do I enable Approval Workflows for my organization?** + +Approval Workflows is currently in Private Beta. Contact the DataHub team to request access to this feature for your instance. + +**Can I create workflows that don't require entity context?** + +Yes, by omitting the `entityTypes` field in your workflow definition, you can create general workflows that don't require a specific entity context. + +**How do I configure dynamic assignee resolution?** + +Configure dynamic assignment in your workflow step's `actors` section using the `dynamicAssignment` field. Available types include: + +- `ENTITY_OWNERS`: Route to owners of the requested entity +- `ENTITY_DOMAIN_OWNERS`: Route to owners of the entity's domain +- `ENTITY_DATA_PRODUCT_OWNERS`: Route to owners of the entity's data product + +You can optionally filter by specific ownership types using the `ownershipTypeUrns` field. + +**What field types are supported in workflow forms?** + +Access Workflows support the following field types: + +- `STRING`: Single-line text input +- `RICH_TEXT`: Multi-line rich text with formatting +- `URN`: Entity references (configurable by entity type) +- `DATE`: Date/time values (epoch milliseconds) +- `NUMBER`: Numeric inputs (integer or float) + +Each field can be configured for single or multiple values using the `cardinality` property. + +**Can I create custom workflow categories?** + +Yes, use the `CUSTOM` category and provide a `customCategory` string to group related workflows outside of the standard `ACCESS` category. + +**What happens if a reviewer is unavailable?** + +You can configure multiple reviewers in a single step to increase the chance that a user can review at each step. + +**Can I modify a workflow after it's been created?** + +Yes, use the `upsertActionWorkflow` mutation with the existing workflow URN to update the definition. Note that changes only affect new requests, not existing ones. + +**How do I handle requests that require multiple approval steps?** + +Define multiple steps in your workflow configuration. Each step can have different assignees and approval requirements. + +**Can users cancel their own requests?** + +Currently, request cancellation is managed through the review process. Users can contact their reviewers to withdraw requests if needed. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/metadata-ingestion-security.md b/docs-archive/versioned_docs/version-1.5.0/docs/metadata-ingestion-security.md new file mode 100644 index 00000000..151d3927 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/metadata-ingestion-security.md @@ -0,0 +1,124 @@ +--- +title: Ingestion Security Comparison +slug: /metadata-ingestion-security +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/metadata-ingestion-security.md +--- +# Ingestion Security Comparison + +DataHub supports three ways to ingest metadata. They differ primarily in where credentials are stored and what network access is required. + +## Quick Comparison + +| | **Credentials** | **Runs From** | **Network** | **Firewall/IP Allowlist Adjustments** | +| ------------------- | -------------------------------------------- | ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| **UI Ingestion** | Encrypted in DataHub | DataHub's infrastructure | DataHub infrastructure connects to your data sources | Required for sources behind firewalls or with IP allowlists | +| **CLI Ingestion** | Local files/env vars | Wherever you execute it (personal machine, CI/CD, scheduler like Airflow/Cron, etc.) | CLI connects to your data sources, then sends metadata to DataHub | Depends on where CLI runs and if those machines already have connectivity | +| **Remote Executor** | Your infrastructure (AWS Secrets, K8s, etc.) | Deployed in your infrastructure (K8s, ECS, etc.) | Executor connects to your data sources, then sends metadata to DataHub (outbound only) | Depends on where executor runs and if those machines already have connectivity | + +### Network Flow Diagram + +```mermaid +graph LR + UI[UI Ingestion
runs in DataHub infrastructure] + CLI[CLI Ingestion
runs where you execute it] + RE[Remote Executor
runs in your infrastructure] + DH[DataHub] + SRC[Data Sources] + + UI -.->|managed by DataHub| DH + DH -->|connects to| SRC + CLI -->|sends metadata to| DH + CLI -->|extracts from| SRC + RE -->|sends metadata to| DH + RE -->|extracts from| SRC +``` + +## Where Credentials Live + +### UI Ingestion + +Credentials are encrypted with an encryption key and stored in the DataHub database. DataHub manages the encryption and uses these credentials when connecting to your data sources. + +### CLI Ingestion + +Credentials are stored in your infrastructure via recipe files. **Best practice: Always use environment variables** rather than hardcoding credentials in recipe files. You can also integrate with local secret managers. + +### Remote Executor + +Integrates with enterprise secret management systems in your infrastructure, such as: + +- AWS Secrets Manager +- Kubernetes Secrets +- External Secrets Operator +- HashiCorp Vault +- Other secret management solutions + +## Network Patterns + +### UI Ingestion + +DataHub's infrastructure connects directly to your data sources. This requires configuring your sources to allow DataHub access, which is source-dependent: + +- **Cloud sources** (Snowflake, BigQuery, etc.): May require adding DataHub IPs to allowlists +- **On-premise sources**: May require VPN tunnels or firewall rules to allow DataHub to reach them + +### CLI Ingestion + +The CLI runs wherever you execute it (personal machine, CI/CD, cloud instance, scheduler like Airflow/Cron). It first connects to your data sources to extract metadata, then sends that metadata to DataHub. Network requirements depend entirely on where the CLI runs and whether that machine already has connectivity to both your sources and DataHub. + +### Remote Executor + +Deployed as software in your infrastructure (Kubernetes, ECS, etc.) with access to both your data sources and DataHub. Like CLI, it connects to sources first, then sends metadata to DataHub. + +**Key advantage**: Only makes outbound connections. You don't need to open inbound firewall ports or configure VPN access for external systems—the executor software runs entirely within your network perimeter. + +## When to Use What + +Most organizations use a mix of all three approaches based on their specific needs. Here are common patterns: + +### UI Ingestion - Best For: + +- **Cloud-hosted data sources**: Snowflake, BigQuery, Redshift, Tableau, Looker, PowerBI, etc. +- **When you want simplicity**: Built-in scheduling, no infrastructure to manage, easiest to scale +- **Getting started quickly**: Minimal setup required + +**Advantages**: Simplest for both scheduling and scale. DataHub handles all infrastructure and orchestration. + +### CLI Ingestion - Best For: + +- **Sources requiring local files**: dbt projects, custom SQL queries, local transformations +- **CI/CD pipelines**: Integrate metadata ingestion into your build/deploy process +- **Development and testing**: Quick iteration and testing of ingestion recipes +- **Custom orchestration**: When you need fine-grained control over scheduling (Airflow, Cron, etc.) + +**Scheduling**: Requires external scheduler (Airflow, Cron, Kubernetes CronJob, etc.) + +### Remote Executor - Best For: + +- **Databases inside your network**: On-premise databases, internal data warehouses +- **Strict security requirements**: When credentials cannot leave your infrastructure +- **Sources behind firewalls**: When you cannot or prefer not to configure external access +- **Enterprise secret management**: When you need integration with existing secret management systems + +**Advantages**: Runs in your infrastructure with your security controls, only requires outbound connectivity, integrates with your existing secret management. + +### Choosing the Right Approach + +The choice often depends on: + +- **Source location**: Cloud-hosted vs on-premise +- **Security requirements**: Where credentials can be stored +- **Network topology**: Firewall and connectivity constraints +- **Operational preferences**: Managed service vs self-hosted +- **Scale**: How many sources and how frequently you ingest + +**Note**: These are guidelines, not strict rules. The best choice varies by organization and even by individual data source within an organization. + +## More Info + +- [UI Ingestion Guide](ui-ingestion.md) +- [CLI Installation](cli.md#installation) +- [Remote Executor Overview](managed-datahub/remote-executor/about.md) +- [Scheduling CLI Ingestion with Airflow](https://datahubproject.io/docs/metadata-ingestion/schedule_docs/airflow) +- [Personal Access Tokens](authentication/personal-access-tokens.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/metadata-standards.md b/docs-archive/versioned_docs/version-1.5.0/docs/metadata-standards.md new file mode 100644 index 00000000..8c8d0cc7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/metadata-standards.md @@ -0,0 +1,69 @@ +--- +title: Metadata Standards +slug: /metadata-standards +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/metadata-standards.md +--- +# Metadata Standards + +The data and AI tooling and infrastructure stack is constantly evolving and adding new concepts (from datasets to dashboards, to models and training runs). DataHub’s goal is to harmonize this complexity and make it understandable for humans and machines, while not sacrificing fidelity. As a result, [over 10 years of iteration](https://www.linkedin.com/blog/engineering/data-management/datahub-popular-metadata-architectures-explained), the DataHub project has evolved into a comprehensive living metadata model that serves as a de-facto standard for metadata in the data and AI stack. + +While other standards exist - like the Iceberg Catalog API and OpenLineage - DataHub implements their interfaces so you can continue using familiar tools while maintaining a single source of truth. + +

+xkcd comic on standards +

+ +So instead of “yet another standard to understand and maintain” like the all too familiar example above - we reduce complexity in the stack. + +## The DataHub Metadata Standard + +Modern data and AI teams need metadata that's comprehensive, flexible, and accessible. DataHub's [open metadata standard](./modeling/metadata-model.md) delivers this through: + +- A complete model that encompasses business and technical metadata +- A schema-first approach to modeling that enables easy extensibility +- Open-source standards and APIs that prevent vendor lock-in + +Metadata is the shared backbone for data discovery, governance, and observability - so it’s important to get these interfaces right. DataHub draws on over 10 years of experience building metadata-driven systems at LinkedIn, and so our [third-generation platform](https://www.linkedin.com/blog/engineering/data-management/datahub-popular-metadata-architectures-explained) has been able to learn from previous modeling mistakes. + +### Aside: Business vs Technical Catalogs + +Historically, the data catalog world has been split in two: business catalogs for human-facing data discovery and documentation, and technical catalogs / meta-stores meant to be machine-facing [operational catalogs for query / compute engines](https://www.youtube.com/watch?v=yPqSR18BzO4). + +DataHub aims to bring these two worlds a bit closer together. Often companies end up operating both business and technical catalogs side by side - and the overlap between them causes messy bidirectional synchronization problems and unnecessary repetition in authentication and authorization. In many of these cases, there should just be a unified tool that does both. + +Being a unified catalog requires a unified metadata standard and a unified product experience. It needs an understandable user interface that uses progressive disclosure to enable power users without overwhelming casual users, and powerful APIs and SDKs to support tools and automations. To our knowledge, DataHub is the only fully open source platform that does this. + +## Interop with other standards + +Given that DataHub is the only open source unified metadata platform, our metadata standard is naturally a superset of most other metadata standards. + +As such, our general approach is to embrace and implement their interfaces, but automatically translate them to the more comprehensive DataHub metadata model for storage. + +We currently implement the Iceberg REST Catalog API and the OpenLineage API. + +### Apache Iceberg REST Catalog + +The Iceberg REST Catalog API is the standard interface for managing table metadata in Apache Iceberg, a popular table format for data lakes. It provides a REST interface for creating and managing Iceberg tables, including their schema, partitioning, and other metadata. + +**How DataHub implements it:** + +- DataHub serves as a drop-in Iceberg REST Catalog implementation. +- Once configured, you can use Iceberg exactly as you normally would. +- Tables created via Iceberg automatically appear in DataHub with full metadata. +- All table operations are available through both Iceberg and DataHub's interfaces. + +Learn more about the integration in the [Iceberg REST catalog docs](./iceberg-catalog.md). + +### OpenLineage + +OpenLineage is an evolving standard for collecting lineage metadata from data pipelines. It defines a common specification for capturing run-time information about data processing jobs, including inputs, outputs, and job context. + +**How DataHub implements it:** + +- DataHub natively accepts OpenLineage events through its API. +- Events are automatically translated into DataHub's richer metadata model. +- All lineage information is immediately available in DataHub's UI and APIs. +- While we support OpenLineage, our native integrations offer enhanced capabilities. For instance, the DataHub Airflow plugin uses schema-aware SQL parsing to generate more accurate column-level lineage. + +Learn more about the integration in the [OpenLineage integration docs](./lineage/openlineage.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/modeling/extending-the-metadata-model.md b/docs-archive/versioned_docs/version-1.5.0/docs/modeling/extending-the-metadata-model.md new file mode 100644 index 00000000..10e02036 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/modeling/extending-the-metadata-model.md @@ -0,0 +1,806 @@ +--- +slug: /metadata-modeling/extending-the-metadata-model +title: Extending the Metadata Model +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/modeling/extending-the-metadata-model.md +--- + +# Extending the Metadata Model + +You can extend the metadata model by either creating a new Entity or extending an existing one. Unsure if you need to +create a new entity or add an aspect to an existing entity? Read [metadata-model](./metadata-model.md) to understand +these two concepts prior to making changes. + +## To fork or not to fork? + +An important question that will arise once you've decided to extend the metadata model is whether you need to fork the main repo or not. Use the diagram below to understand how to make this decision. + +

+ +

+ +The green lines represent pathways that will lead to lesser friction for you to maintain your code long term. The red lines represent higher risk of conflicts in the future. We are working hard to move the majority of model extension use-cases to no-code / low-code pathways to ensure that you can extend the core metadata model without having to maintain a custom fork of DataHub. + +We will refer to the two options as the **open-source fork** and **custom repository** approaches in the rest of the document below. + +## This Guide + +This guide will outline what the experience of adding a new Entity should look like through a real example of adding the +Dashboard Entity. If you want to extend an existing Entity, you can skip directly to [Step 3](#step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity). + +At a high level, an entity is made up of: + +1. A Key Aspect: Uniquely identifies an instance of an entity, +2. A list of specified Aspects, groups of related attributes that are attached to an entity. + +## Defining an Entity + +Now we'll walk through the steps required to create, ingest, and view your extensions to the metadata model. We will use +the existing "Dashboard" entity for purposes of illustration. + +### Step 1: Define the Entity Key Aspect + +A key represents the fields that uniquely identify the entity. For those familiar with DataHub’s legacy architecture, +these fields were previously part of the Urn Java Class that was defined for each entity. + +This struct will be used to generate a serialized string key, represented by an Urn. Each field in the key struct will +be converted into a single part of the Urn's tuple, in the order they are defined. + +Let’s define a Key aspect for our new Dashboard entity. + +``` +namespace com.linkedin.metadata.key + +/** + * Key for a Dashboard + */ +@Aspect = { + "name": "dashboardKey", +} +record DashboardKey { + /** + * The name of the dashboard tool such as looker, redash etc. + */ + @Searchable = { + ... + } + dashboardTool: string + + /** + * Unique id for the dashboard. This id should be globally unique for a dashboarding tool even when there are multiple deployments of it. As an example, dashboard URL could be used here for Looker such as 'looker.linkedin.com/dashboards/1234' + */ + dashboardId: string +} + +``` + +The Urn representation of the Key shown above would be: + +``` +urn:li:dashboard:(,) +``` + +Because they are aspects, keys need to be annotated with an @Aspect annotation, This instructs DataHub that this struct +can be a part of. + +The key can also be annotated with the two index annotations: @Relationship and @Searchable. This instructs DataHub +infra to use the fields in the key to create relationships and index fields for search. See [Step 3](#step-3-define-custom-aspects-or-attach-existing-aspects-to-your-entity) for more details on +the annotation model. + +**Constraints**: Note that each field in a Key Aspect MUST be of String or Enum type. + +### Step 2: Create the new entity with its key aspect + +Define the entity within an `entity-registry.yml` file. Depending on your approach, the location of this file may vary. More on that in steps [4](#step-4-choose-a-place-to-store-your-model-extension) and [5](#step-5-attaching-your-non-key-aspects-to-the-entity). + +Example: + +```yaml +- name: dashboard + doc: A container of related data assets. + keyAspect: dashboardKey +``` + +- name: The entity name/type, this will be present as a part of the Urn. +- doc: A brief description of the entity. +- keyAspect: The name of the Key Aspect defined in step 1. This name must match the value in the PDL annotation. + +# + +### Step 3: Define custom aspects or attach existing aspects to your entity + +Some aspects, like Ownership and GlobalTags, are reusable across entities. They can be included in an entity’s set of +aspects freely. To include attributes that are not included in an existing Aspect, a new Aspect must be created. + +Let’s look at the DashboardInfo aspect as an example of what goes into a new aspect. + +``` +namespace com.linkedin.dashboard + +import com.linkedin.common.AccessLevel +import com.linkedin.common.ChangeAuditStamps +import com.linkedin.common.ChartUrn +import com.linkedin.common.Time +import com.linkedin.common.Url +import com.linkedin.common.CustomProperties +import com.linkedin.common.ExternalReference + +/** + * Information about a dashboard + */ +@Aspect = { + "name": "dashboardInfo" +} +record DashboardInfo includes CustomProperties, ExternalReference { + + /** + * Title of the dashboard + */ + @Searchable = { + "fieldType": "TEXT_WITH_PARTIAL_MATCHING", + "queryByDefault": true, + "enableAutocomplete": true, + "boostScore": 10.0 + } + title: string + + /** + * Detailed description about the dashboard + */ + @Searchable = { + "fieldType": "TEXT", + "queryByDefault": true, + "hasValuesFieldName": "hasDescription" + } + description: string + + /** + * Charts in a dashboard + */ + @Relationship = { + "/*": { + "name": "Contains", + "entityTypes": [ "chart" ] + } + } + charts: array[ChartUrn] = [ ] + + /** + * Captures information about who created/last modified/deleted this dashboard and when + */ + lastModified: ChangeAuditStamps + + /** + * URL for the dashboard. This could be used as an external link on DataHub to allow users access/view the dashboard + */ + dashboardUrl: optional Url + + /** + * Access level for the dashboard + */ + @Searchable = { + "fieldType": "KEYWORD", + "addToFilters": true + } + access: optional AccessLevel + + /** + * The time when this dashboard last refreshed + */ + lastRefreshed: optional Time +} +``` + +The Aspect has four key components: its properties, the @Aspect annotation, the @Searchable annotation and the +@Relationship annotation. Let’s break down each of these: + +- **Aspect properties**: The record’s properties can be declared as a field on the record, or by including another + record in the Aspect’s definition (`record DashboardInfo includes CustomProperties, ExternalReference {`). Properties + can be defined as PDL primitives, enums, records, or collections ( + see [pdl schema documentation](https://linkedin.github.io/rest.li/pdl_schema)) + references to other entities, of type Urn or optionally `Urn` +- **@Aspect annotation**: Declares record is an Aspect and includes it when serializing an entity. Unlike the following + two annotations, @Aspect is applied to the entire record, rather than a specific field. Note, you can mark an aspect + as a timeseries aspect. Check out this [doc](metadata-model.md#timeseries-aspects) for details. +- **@Searchable annotation**: This annotation can be applied to any primitive field or a map field to indicate that it + should be indexed in Elasticsearch and can be searched on. For a complete guide on using the search annotation, see + the annotation docs further down in this document. +- **@Relationship annotation**: These annotations create edges between the Entity’s Urn and the destination of the + annotated field when the entities are ingested. @Relationship annotations must be applied to fields of type Urn. In + the case of DashboardInfo, the `charts` field is an Array of Urns. The @Relationship annotation cannot be applied + directly to an array of Urns. That’s why you see the use of an Annotation override (`"/*":`) to apply the @Relationship + annotation to the Urn directly. Read more about overrides in the annotation docs further down on this page. +- **@UrnValidation**: This annotation can enforce constraints on Urn fields, including entity type restrictions and existence. + +After you create your Aspect, you need to attach to all the entities that it applies to. + +**Constraints**: Note that all aspects MUST be of type Record. + +### Step 4: Choose a place to store your model extension + +At the beginning of this document, we walked you through a flow-chart that should help you decide whether you need to maintain a fork of the open source DataHub repo for your model extensions, or whether you can just use a model extension repository that can stay independent of the DataHub repo. Depending on what path you took, the place you store your aspect model files (the .pdl files) and the entity-registry files (the yaml file called `entity-registry.yaml` or `entity-registry.yml`) will vary. + +- Open source Fork: Aspect files go under [`metadata-models`](https://github.com/datahub-project/datahub/blob/master/metadata-models) module in the main repo, entity registry goes into [`metadata-models/src/main/resources/entity-registry.yml`](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/resources/entity-registry.yml). Read on for more details in [Step 5](#step-5-attaching-your-non-key-aspects-to-the-entity). +- Custom repository: Read the [metadata-models-custom](../../metadata-models-custom/README.md) documentation to learn how to store and version your aspect models and registry. + +### Step 5: Attaching your non-key Aspect(s) to the Entity + +Attaching non-key aspects to an entity can be done simply by adding them to the entity registry yaml file. The location of this file differs based on whether you are following the oss-fork path or the custom-repository path. + +Here is an minimal example of adding our new `DashboardInfo` aspect to the `Dashboard` entity. + +```yaml +entities: + - name: dashboard + - keyAspect: dashBoardKey + aspects: + # the name of the aspect must be the same as that on the @Aspect annotation on the class + - dashboardInfo +``` + +Previously, you were required to add all aspects for the entity into an Aspect union. You will see examples of this pattern throughout the code-base (e.g. `DatasetAspect`, `DashboardAspect` etc.). This is no longer required. + +### Step 6 (Oss-Fork approach): Re-build DataHub to have access to your new or updated entity + +If you opted for the open-source fork approach, where you are editing models in the `metadata-models` repository of DataHub, you will need to re-build the DataHub metadata service using the steps below. If you are following the custom model repository approach, you just need to build your custom model repository and deploy it to a running metadata service instance to read and write metadata using your new model extensions. + +Read on to understand how to re-build DataHub for the oss-fork option. + +**_NOTE_**: If you have updated any existing types or see an `Incompatible changes` warning when building, you will need to run +`./gradlew :metadata-service:restli-servlet-impl:build -Prest.model.compatibility=ignore` +before running `build`. + +Then, run `./gradlew build` from the repository root to rebuild DataHub with access to your new entity. + +Then, re-deploy metadata-service (gms), and mae-consumer and mce-consumer (optionally if you are running them unbundled). See [docker development](../../docker/README.md) for details on how +to deploy during development. This will allow DataHub to read and write your new entity or extensions to existing entities, along with serving search and graph queries for that entity type. + +### (Optional) Step 7: Use custom models with the Python SDK + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +If you're purely using the custom models locally, you can use a local development-mode install of the DataHub CLI. + +Install the DataHub CLI locally by following the [developer instructions](../../metadata-ingestion/developing.md). +The `./gradlew build` command already generated the avro schemas for your local ingestion cli tool to use. +After following the developing guide, you should be able to emit your new event using the local DataHub CLI. + + + + +If you want to use your custom models beyond your local machine without forking DataHub, then you can generate a custom model package that can be installed from other places. + +This package should be installed alongside the base `acryl-datahub` package, and its metadata models will take precedence over the default ones. + +```bash +$ cd metadata-ingestion +$ ../gradlew customPackageGenerate -Ppackage_name=my-company-datahub-models -Ppackage_version="0.0.1" + +Successfully built my-company-datahub-models-0.0.1.tar.gz and acryl_datahub_cloud-0.0.1-py3-none-any.whl + +Generated package at custom-package/my-company-datahub-models +This package should be installed alongside the main acryl-datahub package. + +Install the custom package locally with `pip install custom-package/my-company-datahub-models` +To enable others to use it, share the file at custom-package/my-company-datahub-models/dist/.whl and have them install it with `pip install .whl` +Alternatively, publish it to PyPI with `twine upload custom-package/my-company-datahub-models/dist/*` +``` + +This will generate some Python build artifacts, which you can distribute within your team or publish to PyPI. +The command output contains additional details and exact CLI commands you can use. + +Once this package is installed, you can use the DataHub CLI as normal, and it will use your custom models. +You'll also be able to import those models, with IDE support, by changing your imports. + +```diff +- from datahub.metadata.schema_classes import DatasetPropertiesClass ++ from my_company_datahub_models.metadata.schema_classes import DatasetPropertiesClass +``` + + + + +### (Optional) Step 8: Extend the DataHub frontend to view your entity in GraphQL & React + +If you are extending an entity with additional aspects, and you can use the auto-render specifications to automatically render these aspects to your satisfaction, you do not need to write any custom code. + +However, if you want to write specific code to render your model extensions, or if you introduced a whole new entity and want to give it its own page, you will need to write custom React and Grapqhl code to view and mutate your entity in GraphQL or React. For +instructions on how to start extending the GraphQL graph, see [graphql docs](../../datahub-graphql-core/README.md). Once you’ve done that, you can follow the guide [here](../../datahub-web-react/README.md) to add your entity into the React UI. + +## Metadata Annotations + +There are four core annotations that DataHub recognizes: + +#### @Entity + +**Legacy** +This annotation is applied to each Entity Snapshot record, such as DashboardSnapshot.pdl. Each one that is included in +the root Snapshot.pdl model must have this annotation. + +It takes the following parameters: + +- **name**: string - A common name used to identify the entity. Must be unique among all entities DataHub is aware of. + +##### Example + +```aidl +@Entity = { + // name used when referring to the entity in APIs. + String name; +} +``` + +#### @Aspect + +This annotation is applied to each Aspect record, such as DashboardInfo.pdl. Each aspect that is included in an entity’s +set of aspects in the `entity-registry.yml` must have this annotation. + +It takes the following parameters: + +- **name**: string - A common name used to identify the Aspect. Must be unique among all aspects DataHub is aware of. +- **type**: string (optional) - set to "timeseries" to mark this aspect as timeseries. Check out + this [doc](metadata-model.md#timeseries-aspects) for details. +- **autoRender**: boolean (optional) - defaults to false. When set to true, the aspect will automatically be displayed + on entity pages in a tab using a default renderer. **_This is currently only supported for Charts, Dashboards, DataFlows, DataJobs, Datasets, Domains, and GlossaryTerms_**. +- **renderSpec**: RenderSpec (optional) - config for autoRender aspects that controls how they are displayed. **_This is currently only supported for Charts, Dashboards, DataFlows, DataJobs, Datasets, Domains, and GlossaryTerms_**. Contains three fields: + - **displayType**: One of `tabular`, `properties`. Tabular should be used for a list of data elements, properties for a single data bag. + - **displayName**: How the aspect should be referred to in the UI. Determines the name of the tab on the entity page. + - **key**: For `tabular` aspects only. Specifies the key in which the array to render may be found. + +##### Example + +```aidl +@Aspect = { + // name used when referring to the aspect in APIs. + String name; +} +``` + +#### @Searchable + +This annotation is applied to fields inside an Aspect. It instructs DataHub to index the field so it can be retrieved +via the search APIs. + +:::note If you are adding @Searchable to a field that already has data, you'll want to restore indices [via api](/docs/api/restli/restore-indices/) or [via upgrade step](https://github.com/datahub-project/datahub/blob/master/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/RestoreGlossaryIndices.java) to have it be populated with existing data. + +It takes the following parameters: + +- **fieldType**: string - The settings for how each field is indexed is defined by the field type. In general this defines how the field is indexed in the Elasticsearch document. **Note**: With the new search tier system, `fieldType` primarily determines the field's storage format and individual query capabilities. Fulltext search capabilities are now primarily handled by the common `_search.tier_{tier}` fields that consolidate fields from multiple aspects based on their tier assignments. + + **Available field types:** + + 1. _KEYWORD_ - Short text fields that only support exact matches, often used only for filtering. **Default length limit**: 100 characters (tier fields), 255 characters (regular fields). + + 2. _TEXT_ - Text fields delimited by spaces/slashes/periods. Default field type for string variables. **Default length limit**: 100 characters (tier fields), 255 characters (regular fields). + + 3. _BOOLEAN_ - Boolean fields used for filtering. + + 4. _COUNT_ - Count fields used for filtering. + + 5. _DATETIME_ - Datetime fields used to represent timestamps. + + 6. _OBJECT_ - Each property in an object will become an extra column in Elasticsearch and can be referenced as + `field.property` in queries. **Default limits**: Maximum 1000 object keys, maximum 4096 characters per value. You should be careful to not use it on objects with many properties as it can cause a mapping explosion in Elasticsearch. + + 7. _DOUBLE_ - Double precision numeric fields used for filtering and calculations. + + 8. _MAP_ARRAY_ - Array fields that are stored as maps in Elasticsearch. **Default limits**: Maximum 1000 array elements, maximum 4096 characters per value. + + **⚠️ Deprecated field types (avoid using in new code):** + + 10. ~~_TEXT_PARTIAL_~~ - **DEPRECATED**: Text fields with partial matching support. This field type is expensive and should not be applied to fields with long values. Use TEXT instead. + + 11. ~~_WORD_GRAM_~~ - **DEPRECATED**: Text fields with word gram support. This field type is expensive and should not be applied to fields with long values. Use TEXT instead. + + 12. ~~_BROWSE_PATH_~~ - **DEPRECATED**: Field type for browse paths. Browse paths are handled by name, use `browsePathV2` field name. There can only be one for a given entity. + + 13. ~~_URN_~~ - **DEPRECATED**: Urn fields where each sub-component is indexed. Use KEYWORD instead. + + 14. ~~_URN_PARTIAL_~~ - **DEPRECATED**: Urn fields with partial matching support. Use KEYWORD instead. + +**⚠️ Important Length Limitations:** + +- **Tier Fields**: Fields with `searchTier` are automatically limited to **100 characters** to optimize search performance +- **Regular Fields**: Fields without `searchTier` are limited to **255 characters** for Elasticsearch compatibility +- **Object Fields**: Maximum **1000 object keys** and **4096 characters per value** to prevent mapping explosion +- **Array Fields**: Maximum **1000 array elements** and **4096 characters per value** +- **Field Names**: Maximum **255 characters** for Elasticsearch field name compatibility + +**Configuration Overrides:** + +- **Environment Variables**: Some limits can be configured via environment variables: + - `SEARCH_DOCUMENT_MAX_VALUE_LENGTH`: Override default 4096 character limit for object/array values + - `SEARCH_DOCUMENT_MAX_ARRAY_LENGTH`: Override default 1000 element limit for arrays + - `SEARCH_DOCUMENT_MAX_OBJECT_KEYS`: Override default 1000 key limit for objects +- **Special Fields**: Some system fields have different limits: + - **URN fields**: Automatically set to **512 characters** (`ignore_above: 512`) + - **Tier fields**: Hard-coded to **100 characters** for performance optimization + +**Note**: The `ignore_above` settings are automatically applied by the system. While some limits can be configured via environment variables, the tier field limits (100 characters) and regular field limits (255 characters) are hard-coded and cannot be overridden through annotations or configuration. + +**Important**: The ability to have longer keyword fields is limited to system-level configurations and special field types. Regular user-defined fields will always be subject to the default limits for performance and compatibility reasons. + +- **fieldName**: string (optional) - The name of the field in search index document. Defaults to the field name where + the annotation resides. + +- **queryByDefault**: boolean (optional) - **⚠️ DEPRECATED**: Whether we should match the field for the default search query. True by + default for text and urn fields. **Use `searchTier` instead for better search organization and performance.** + +- **enableAutocomplete**: boolean (optional) - **⚠️ DEPRECATED**: Whether we should use the field for autocomplete. Defaults to false. **Use `searchTier: 1` based on the fact that an autocomplete field would be very important for search relevance.** + +- **addToFilters**: boolean (optional) - Whether or not to add field to filters. Defaults to false + +- **addHasValuesToFilters**: boolean (optional) - Whether or not to add the "has values" to filters. Defaults to true + +- **filterNameOverride**: string (optional) - Display name for the filter in the UI + +- **hasValuesFilterNameOverride**: string (optional) - Display name for the "has values" filter in the UI + +- **boostScore**: double (optional) - **⚠️ DEPRECATED**: Boost multiplier to the match score. Matches on fields with higher boost score + ranks higher. **Use `searchLabel` instead for more sophisticated ranking control.** + +- **hasValuesFieldName**: string (optional) - If set, add an index field of the given name that checks whether the field + exists + +- **numValuesFieldName**: string (optional) - If set, add an index field of the given name that checks the number of + elements + +- **weightsPerFieldValue**: map[object, double] (optional) - **⚠️ DEPRECATED**: Weights to apply to score for a given value. **Use `searchLabel` with `@SearchScore` annotations instead for value-based scoring.** + +- **fieldNameAliases**: array[string] (optional) - Aliases for this field that can be used for sorting and other operations. These aliases are created with the aspect name prefix (e.g., `metadata.aliasName`) and provide alternative names for accessing the same field data. Useful for creating multiple access paths to the same field. + +- **includeSystemModifiedAt**: boolean (optional) - **⚠️ DEPRECATED**: Whether to include a system-modified timestamp field for this searchable field. **This will be handled programmatically for all aspects in future versions.** + +- **systemModifiedAtFieldName**: string (optional) - **⚠️ DEPRECATED**: Custom name for the system-modified timestamp field. **This will be handled programmatically for all aspects in future versions.** + +- **includeQueryEmptyAggregation**: boolean (optional) - Whether to create a missing field aggregation when querying the corresponding field. Only affects query time, not mapping. Useful for analytics and reporting. + +- **searchTier**: integer (optional) - Search tier for the field (integer value >= 1). Creates a copy*to field that copies the field value to `\_search.tier*{tier}`. Fields with searchTier are automatically set to `index: false`unless`searchIndexed` is true. **Note**: searchTier can only be used with KEYWORD or TEXT field types. + +- **searchLabel**: string (optional) - Unified label for search operations. Creates a copy\*to field that copies the field value to `\_search.{label}` (without prefixes). Replaces the previous `sortLabel` and `boostLabel` annotations. Fields with searchLabel are automatically set to `index: false`. + +- **searchIndexed**: boolean (optional) - When combined with `searchTier`, determines whether the field is indexed outside of `_search` for direct access. The field will be indexed using its actual field type (KEYWORD or TEXT), not forced to KEYWORD. **Note**: searchIndexed can only be true when searchTier is specified and can only be used with KEYWORD or TEXT field types. Defaults to false. + +- **entityFieldName**: string (optional) - If set, this field will be copied to `_search.{entityFieldName}` and the root alias will point there. This allows multiple aspects to consolidate into a single entity-level field. + +- **eagerGlobalOrdinals**: boolean (optional) - Whether to set `eager_global_ordinals` to true for this field. This improves aggregation performance for frequently aggregated keyword fields by pre-building ordinals at index time. **Note**: eagerGlobalOrdinals can only be true for KEYWORD, URN, or URN_PARTIAL field types. Defaults to false. + +**⚠️ Note on deprecated parameters:** Some parameters like `queryByDefault`, `enableAutocomplete`, `boostScore`, and `weightsPerFieldValue` are still functional when using search version 2 but will be replaced by newer features in future versions. Consider using the new tier-based and label-based annotations for more advanced search functionality. + +**Migration from deprecated parameters:** + +- **`queryByDefault`** → Use `searchTier: 1` through `searchTier: 4` to include fields in default search queries +- **`enableAutocomplete`** → Use `searchTier: 1` +- **`boostScore`** → Use `searchLabel` for more sophisticated ranking control +- **`weightsPerFieldValue`** → Use `searchLabel` for value-based scoring control + +##### Example + +Let’s take a look at a real world example using the `title` field of `DashboardInfo.pdl`: + +```aidl +record DashboardInfo { + /** + * Title of the dashboard + */ + @Searchable = { + "fieldType": "KEYWORD", + "searchTier": 1, + "entityFieldName": "name" + } + title: string + .... +} +``` + +This annotation is saying that we want to index the title field in Elasticsearch. `searchTier: 1` ensures this field is included in default search queries with high relevancy. `entityFieldName: "name"` consolidates this field into the entity-level `_search.name` field, allowing other aspects to contribute to the same consolidated field. + +**Advanced Example with New Features:** + +```aidl +record DashboardInfo { + /** + * Priority level for the dashboard + */ + @Searchable = { + "fieldType": "COUNT", + "searchLabel": "priority", + "addToFilters": true + } + priority: int + + /** + * Status of the dashboard + */ + @Searchable = { + "fieldType": "KEYWORD", + "addToFilters": true, + "filterNameOverride": "Dashboard Status", + "eagerGlobalOrdinals": true + } + status: string + + /** + * Owner URN for the dashboard + */ + @Searchable = { + "fieldType": "URN", + "addToFilters": true, + "eagerGlobalOrdinals": true, + "searchLabel": "owner" + } + owner: string +} +``` + +This example demonstrates several new features: + +- **Priority field**: `fieldType: "COUNT"` with `searchLabel: "priority"` creates a numeric field that copies to `_search.priority` for proper numeric sorting operations, and `addToFilters: true` makes it available as a filter +- **Status field**: `addToFilters: true` makes it available as a filter, `filterNameOverride` provides a custom display name "Dashboard Status", and `eagerGlobalOrdinals: true` optimizes aggregation performance for this frequently filtered field +- **Owner field**: `fieldType: "URN"` with `eagerGlobalOrdinals: true` optimizes aggregation performance for owner-based filtering, and `searchLabel: "owner"` copies the field to `_search.owner` for ranking operations + +Now, when DataHub ingests Dashboards, it will index the priority and status fields in Elasticsearch. The priority field will be available for sorting operations, and both fields will be available as filters in the UI. + +Note, when @Searchable annotation is applied to a map, it will convert it into a list with "key.toString() +=value.toString()" as elements. This allows us to index map fields, while not increasing the number of columns indexed. +This way, the keys can be queried by `aMapField:key1=value1`. + +You can change this behavior by specifying the fieldType as OBJECT in the @Searchable annotation. It will put each key +into a column in Elasticsearch instead of an array of serialized kay-value pairs. This way the query would look more +like `aMapField.key1:value1`. As this method will increase the number of columns with each unique key - large maps can +cause a mapping explosion in Elasticsearch. You should _not_ use the object fieldType if you expect your maps to get +large. + +#### @SearchScore ⚠️ DEPRECATED + +**⚠️ DEPRECATED**: This annotation is deprecated and should not be used in new code. Use `searchLabel` with the new search tier system instead for ranking functionality. + +#### Search Tier and Label System + +The new search tier and label system provides a powerful way to organize search fields and create specialized search experiences: + +**Search Tiers (`searchTier`):** + +- Fields with `searchTier` are automatically copied to `_search.tier_{tier}` fields +- This creates a fundamental change in search architecture: **fulltext search capabilities are now determined by the tier, not the individual field type** +- All fields assigned to the same tier (e.g., `_search.tier_1`) are consolidated into a single searchable field, regardless of their individual `fieldType` +- This allows you to create tiered search experiences where different fields contribute to different search priorities +- Use `searchIndexed: true` if you need direct access to the field for filtering/sorting while maintaining tier functionality + +**Search Labels (`searchLabel`):** + +- Fields with `searchLabel` are copied to `_search.{label}` fields (without prefixes) +- Replaces the previous `sortLabel` and `boostLabel` annotations for a unified approach +- Useful for creating specialized search, sorting, and ranking operations across multiple aspects +- Automatically sets `index: false` to optimize storage + +**Entity Field Consolidation (`entityFieldName`):** + +- Allows multiple aspects to consolidate into a single entity-level field +- Useful for creating unified search experiences across different aspect types +- Fields are copied to `_search.{entityFieldName}` with root-level aliases + +**Benefits of the New System:** + +1. **Organized Search Fields**: All search-related fields are grouped under `_search.*` +2. **Efficient Indexing**: Original fields are not indexed (index: false) but copied to search fields +3. **Easy Access**: Aliases provide convenient access to fields at the root level +4. **Flexible Querying**: Search queries can target specific tier, sort, or ranking fields +5. **Performance**: Optimized storage and query patterns for complex search scenarios + +**Architectural Impact:** + +- **Before**: Each field's `fieldType` determined its individual search capabilities and analyzers +- **After**: The `searchTier` determines fulltext search capabilities, while `fieldType` primarily affects storage format and individual field queries +- **Search Consolidation**: Fields from different aspects with the same tier are automatically consolidated into unified search fields +- **Simplified Search Logic**: Search queries can target entire tiers rather than individual fields, making complex search scenarios more manageable + +#### Migration Guide for Deprecated Features + +If you're currently using deprecated field types or parameters, here's how to migrate to the new system: + +**Field Type Migrations:** + +| Deprecated | Recommended Replacement | Notes | +| -------------- | ----------------------- | ------------------------------------------------------------ | +| `TEXT_PARTIAL` | `TEXT` | Use TEXT with appropriate analyzers for partial matching | +| `WORD_GRAM` | `TEXT` | Use TEXT with word delimited analyzers for word-based search | +| `BROWSE_PATH` | `BROWSE_PATH_V2` | Use BROWSE_PATH_V2 for improved path hierarchy support | +| `URN` | `TEXT` | Use TEXT with URN analyzers for component-based search | +| `URN_PARTIAL` | `TEXT` | Use TEXT with URN analyzers and partial matching | + +**Parameter Migrations:** + +| Deprecated Pattern | New Pattern | Benefits | +| ----------------------------------------- | --------------- | ------------------------------------------------------------------------- | +| `queryByDefault: true` | `searchTier: 1` | More explicit control over search behavior and better performance | +| `enableAutocomplete: true` | `searchTier: 1` | Better performance and organization | +| `includeSystemModifiedAt: true` | **Automatic** | System modification tracking is now handled automatically for all aspects | +| `systemModifiedAtFieldName: "customName"` | **Automatic** | System modification field names are now standardized automatically | + +**Example Migration:** + +```aidl +// Old deprecated approach +@Searchable = { + "fieldType": "TEXT_PARTIAL", + "queryByDefault": true, + "enableAutocomplete": true, + "boostScore": 10.0 +} +title: string + +// New recommended approach +@Searchable = { + "fieldType": "TEXT", + "searchTier": 1, + "entityFieldName": "name" +} +title: string +``` + +**Tier Consolidation Example:** + +```aidl +// Multiple aspects can now contribute to the same search tier +record DatasetInfo { + @Searchable = { + "fieldType": "KEYWORD", + "searchTier": 1, + "entityFieldName": "name" + } + name: string + + @Searchable = { + "fieldType": "TEXT", + "searchTier": 1, + "entityFieldName": "description" + } + description: string +} + +record ChartInfo { + @Searchable = { + "fieldType": "KEYWORD", + "searchTier": 1, + "entityFieldName": "name" + } + chartName: string +} +``` + +In this example, all three fields (`name`, `description`, `chartName`) are automatically consolidated into `_search.tier_1`. A single search query against `_search.tier_1:*` will search across all these fields simultaneously, regardless of their individual aspect and field locations. The `fieldType` now primarily determines how each field is stored and accessed individually, while the tier determines how it participates in fulltext search. + +**Benefits of Migration:** + +- Better search performance through optimized indexing +- More organized search field structure +- Enhanced query capabilities with tier-based targeting +- Future-proof annotations that won't be deprecated +- Improved Elasticsearch mapping efficiency + +#### @Relationship + +This annotation is applied to fields inside an Aspect. This annotation creates edges between an Entity's Urn and the +destination of the annotated field when the Entity is ingested. @Relationship annotations must be applied to fields of +type Urn. + +It takes the following parameters: + +- **name**: string - A name used to identify the Relationship type. +- **entityTypes**: array[string] (Optional) - A list of entity types that are valid values for the foreign-key + relationship field. + +##### Example + +Let's take a look at a real world example to see how this annotation is used. The `Owner.pdl` struct is referenced by +the `Ownership.pdl` aspect. `Owned.pdl` contains a relationship to a CorpUser or CorpGroup: + +``` +namespace com.linkedin.common + +/** + * Ownership information + */ +record Owner { + + /** + * Owner URN, e.g. urn:li:corpuser:ldap, urn:li:corpGroup:group_name, and urn:li:multiProduct:mp_name + */ + @Relationship = { + "name": "OwnedBy", + "entityTypes": [ "corpUser", "corpGroup" ] + } + owner: Urn + + ... +} +``` + +This annotation says that when we ingest an Entity with an Ownership Aspect, DataHub will create an OwnedBy relationship +between that entity and the CorpUser or CorpGroup who owns it. This will be queryable using the Relationships resource +in both the forward and inverse directions. + +#### @UrnValidation + +This annotation can be applied to Urn fields inside an aspect. The annotation can optionally perform one or more of the following: + +- Enforce that the URN exists +- Enforce stricter URN validation +- Restrict the URN to specific entity types + +##### Example + +Using this example from StructuredPropertyDefinition, we are enforcing that the valueType URN must exist, +it must follow stricter Urn encoding logic, and it can only be of entity type `dataType`. + +``` + @UrnValidation = { + "exist": true, + "strict": true, + "entityTypes": [ "dataType" ], + } + valueType: Urn +``` + +#### Annotating Collections & Annotation Overrides + +You will not always be able to apply annotations to a primitive field directly. This may be because the field is wrapped +in an Array, or because the field is part of a shared struct that many entities reference. In these cases, you need to +use annotation overrides. An override is done by specifying a fieldPath to the target field inside the annotation, like +so: + +``` + /** + * Charts in a dashboard + */ + @Relationship = { + "/*": { + "name": "Contains", + "entityTypes": [ "chart" ] + } + } + charts: array[ChartUrn] = [ ] +``` + +This override applies the relationship annotation to each element in the Array, rather than the array itself. This +allows a unique Relationship to be created for between the Dashboard and each of its charts. + +Another example can be seen in the case of tags. In this case, TagAssociation.pdl has a @Searchable annotation: + +``` + @Searchable = { + "fieldName": "tags", + "fieldType": "URN_WITH_PARTIAL_MATCHING", + "queryByDefault": true, + "hasValuesFieldName": "hasTags" + } + tag: TagUrn +``` + +At the same time, SchemaField overrides that annotation to allow for searching for tags applied to schema fields +specifically. To do this, it overrides the Searchable annotation applied to the `tag` field of `TagAssociation` and +replaces it with its own- this has a different boostScore and a different fieldName. + +``` + /** + * Tags associated with the field + */ + @Searchable = { + "/tags/*/tag": { + "fieldName": "fieldTags", + "fieldType": "URN_WITH_PARTIAL_MATCHING", + "queryByDefault": true, + "boostScore": 0.5 + } + } + globalTags: optional GlobalTags +``` + +As a result, you can issue a query specifically for tags on Schema Fields via `fieldTags:` or tags directly +applied to an entity via `tags:`. Since both have `queryByDefault` set to true, you can also search for +entities with either of these properties just by searching for the tag name. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/modeling/metadata-model.md b/docs-archive/versioned_docs/version-1.5.0/docs/modeling/metadata-model.md new file mode 100644 index 00000000..133cdefd --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/modeling/metadata-model.md @@ -0,0 +1,630 @@ +--- +title: The Metadata Model +sidebar_label: The Metadata Model +slug: /metadata-modeling/metadata-model +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/modeling/metadata-model.md +--- + +# How does DataHub model metadata? + +DataHub takes a schema-first approach to modeling metadata. We use the open-source Pegasus schema language ([PDL](https://linkedin.github.io/rest.li/pdl_schema)) extended with a custom set of annotations to model metadata. The DataHub storage, serving, indexing and ingestion layer operates directly on top of the metadata model and supports strong types all the way from the client to the storage layer. + +Conceptually, metadata is modeled using the following abstractions + +- **Entities**: An entity is the primary node in the metadata graph. For example, an instance of a Dataset or a CorpUser is an Entity. An entity is made up of a type, e.g. 'dataset', a unique identifier (e.g. an 'urn') and groups of metadata attributes (e.g. documents) which we call aspects. + +- **Aspects**: An aspect is a collection of attributes that describes a particular facet of an entity. They are the smallest atomic unit of write in DataHub. That is, multiple aspects associated with the same Entity can be updated independently. For example, DatasetProperties contains a collection of attributes that describes a Dataset. Aspects can be shared across entities, for example "Ownership" is an aspect that is re-used across all the Entities that have owners. Common aspects include + + - [ownership](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl): Captures the users and groups who own an Entity. + - [globalTags](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/GlobalTags.pdl): Captures references to the Tags associated with an Entity. + - [glossaryTerms](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTerms.pdl): Captures references to the Glossary Terms associated with an Entity. + - [institutionalMemory](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/InstitutionalMemory.pdl): Captures internal company Documents associated with an Entity (e.g. links!) + - [status](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Status.pdl): Captures the "deletion" status of an Entity, i.e. whether it should be soft-deleted. + - [subTypes](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/SubTypes.pdl): Captures one or more "sub types" of a more generic Entity type. An example can be a "Looker Explore" Dataset, a "View" Dataset. Specific sub types can imply that certain additional aspects are present for a given Entity. + +- **Relationships**: A relationship represents a named edge between 2 entities. They are declared via foreign key attributes within Aspects along with a custom annotation (@Relationship). Relationships permit edges to be traversed bi-directionally. For example, a Chart may refer to a CorpUser as its owner via a relationship named "OwnedBy". This edge would be walkable starting from the Chart _or_ the CorpUser instance. + +- **Identifiers (Keys & Urns)**: A key is a special type of aspect that contains the fields that uniquely identify an individual Entity. Key aspects can be serialized into _Urns_, which represent a stringified form of the key fields used for primary-key lookup. Moreover, _Urns_ can be converted back into key aspect structs, making key aspects a type of "virtual" aspect. Key aspects provide a mechanism for clients to easily read fields comprising the primary key, which are usually generally useful like Dataset names, platform names etc. Urns provide a friendly handle by which Entities can be queried without requiring a fully materialized struct. + +Here is an example graph consisting of 3 types of entity (CorpUser, Chart, Dashboard), 2 types of relationship (OwnedBy, Contains), and 3 types of metadata aspect (Ownership, ChartInfo, and DashboardInfo). + +

+ +

+ +## The Core Entities + +DataHub's "core" Entity types model the Data Assets that comprise the Modern Data Stack. They include + +1. **[Data Platform](docs/generated/metamodel/entities/dataPlatform.md)**: A type of Data "Platform". That is, an external system that is involved in processing, storing, or visualizing Data Assets. Examples include MySQL, Snowflake, Redshift, and S3. +2. **[Dataset](docs/generated/metamodel/entities/dataset.md)**: A collection of data. Tables, Views, Streams, Document Collections, and Files are all modeled as "Datasets" on DataHub. Datasets can have tags, owners, links, glossary terms, and descriptions attached to them. They can also have specific sub-types, such as "View", "Collection", "Stream", "Explore", and more. Examples include Postgres Tables, MongoDB Collections, or S3 files. +3. **[Chart](docs/generated/metamodel/entities/chart.md)**: A single data vizualization derived from a Dataset. A single Chart can be a part of multiple Dashboards. Charts can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include a Superset or Looker Chart. +4. **[Dashboard](docs/generated/metamodel/entities/dashboard.md)**: A collection of Charts for visualization. Dashboards can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include a Superset or Mode Dashboard. +5. **[Data Job](docs/generated/metamodel/entities/dataJob.md)** (Task): An executable job that processes data assets, where "processing" implies consuming data, producing data, or both. Data Jobs can have tags, owners, links, glossary terms, and descriptions attached to them. They must belong to a single Data Flow. Examples include an Airflow Task. +6. **[Data Flow](docs/generated/metamodel/entities/dataFlow.md)** (Pipeline): An executable collection of Data Jobs with dependencies among them, or a DAG. Data Jobs can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include an Airflow DAG. + +See the **Metadata Modeling/Entities** section on the left to explore the entire model. + +## The Entity Registry + +Where are Entities and their aspects defined in DataHub? Where does the Metadata Model "live"? The Metadata Model is stitched together by means +of an **Entity Registry**, a catalog of Entities that comprise the Metadata Graph along with the aspects associated with each. Put +simply, this is where the "schema" of the model is defined. + +Traditionally, the Entity Registry was constructed using [Snapshot](https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot) models, which are schemas that explicitly tie +an Entity to the Aspects associated with it. An example is [DatasetSnapshot](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/DatasetSnapshot.pdl), which defines the core `Dataset` Entity. +The Aspects of the Dataset entity are captured via a union field inside a special "Aspect" schema. An example is +[DatasetAspect](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/DatasetAspect.pdl). +This file associates dataset-specific aspects (like [DatasetProperties](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProperties.pdl)) and common aspects (like [Ownership](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl), +[InstitutionalMemory](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/InstitutionalMemory.pdl), +and [Status](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Status.pdl)) +to the Dataset Entity. This approach to defining Entities will soon be deprecated in favor of a new approach. + +As of January 2022, DataHub has deprecated support for Snapshot models as a means of adding new entities. Instead, +the Entity Registry is defined inside a YAML configuration file called [entity-registry.yml](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/resources/entity-registry.yml), +which is provided to DataHub's Metadata Service at start up. This file declares Entities and Aspects by referring to their [names](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl#L7). +At boot time, DataHub validates the structure of the registry file and ensures that it can find PDL schemas associated with +each aspect name provided by configuration (via the [@Aspect](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl#L6) annotation). + +By moving to this format, evolving the Metadata Model becomes much easier. Adding Entities & Aspects becomes a matter of adding a +to the YAML configuration, instead of creating new Snapshot / Aspect files. + +## Exploring DataHub's Metadata Model + +To explore the current DataHub metadata model, you can inspect this high-level picture that shows the different entities and edges between them showing the relationships between them. + +

+ +

+ +To navigate the aspect model for specific entities and explore relationships using the `foreign-key` concept, you can view them in our demo environment or navigate the auto-generated docs in the **Metadata Modeling/Entities** section on the left. + +For example, here are helpful links to the most popular entities in DataHub's metadata model: + +- [Dataset](docs/generated/metamodel/entities/dataset.md): [Profile](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,Dataset,PROD)/Schema`?is_lineage_mode=false) [Documentation](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,Dataset,PROD)/Documentation`?is_lineage_mode=false) +- [Dashboard](docs/generated/metamodel/entities/dashboard.md): [Profile](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,Dashboard,PROD)/Schema`?is_lineage_mode=false) [Documentation](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,Dashboard,PROD)/Documentation`?is_lineage_mode=false) +- [User (a.k.a CorpUser)](docs/generated/metamodel/entities/corpuser.md): [Profile](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,Corpuser,PROD)/Schema`?is_lineage_mode=false) [Documentation](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,Corpuser,PROD)/Documentation`?is_lineage_mode=false) +- [Pipeline (a.k.a DataFlow)](docs/generated/metamodel/entities/dataFlow.md): [Profile](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,DataFlow,PROD)/Schema`?is_lineage_mode=false) [Documentation](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,DataFlow,PROD)/Documentation`?is_lineage_mode=false) +- [Feature Table (a.k.a. MLFeatureTable)](docs/generated/metamodel/entities/mlFeatureTable.md): [Profile](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Schema`?is_lineage_mode=false) [Documentation](https://demo.datahub.com/dataset/`urn:li:dataset:(urn:li:dataPlatform:datahub,MlFeatureTable,PROD)/Documentation`?is_lineage_mode=false) +- For the full list of entities in the metadata model, browse them [here](https://demo.datahub.com/browse/dataset/prod/datahub/entities) or use the **Metadata Modeling/Entities** section on the left. + +### Generating documentation for the Metadata Model + +- This website: Metadata model documentation for this website is generated using `./gradlew :docs-website:yarnBuild`, which delegates the model doc generation to the `modelDocGen` task in the `metadata-ingestion` module. +- Uploading documentation to a running DataHub Instance: The metadata model documentation can be generated and uploaded into a running DataHub instance using the command `./gradlew :metadata-ingestion:modelDocUpload`. **_NOTE_**: This will upload the model documentation to the DataHub instance running at the environment variable `$DATAHUB_SERVER` (http://localhost:8080 by default) + +## Querying the Metadata Graph + +DataHub’s modeling language allows you to optimize metadata persistence to align with query patterns. + +There are three supported ways to query the metadata graph: by primary key lookup, a search query, and via relationship traversal. + +> New to [PDL](https://linkedin.github.io/rest.li/pdl_schema) files? Don't fret. They are just a way to define a JSON document "schema" for Aspects in DataHub. All Data ingested to DataHub's Metadata Service is validated against a PDL schema, with each @Aspect corresponding to a single schema. Structurally, PDL is quite similar to [Protobuf](https://developers.google.com/protocol-buffers) and conveniently maps to JSON. + +### Querying an Entity + +#### Fetching Latest Entity Aspects (Snapshot) + +Querying an Entity by primary key means using the "entities" endpoint, passing in the +urn of the entity to retrieve. + +For example, to fetch a Chart entity, we can use the following `curl`: + +``` +curl --location --request GET 'http://localhost:8080/entities/urn%3Ali%3Achart%3Acustomers +``` + +This request will return a set of versioned aspects, each at the latest version. + +As you'll notice, we perform the lookup using the url-encoded _Urn_ associated with an entity. +The response would be an "Entity" record containing the Entity Snapshot (which in turn contains the latest aspects associated with the Entity). + +#### Fetching Versioned Aspects + +DataHub also supports fetching individual pieces of metadata about an Entity, which we call aspects. To do so, +you'll provide both an Entity's primary key (urn) along with the aspect name and version that you'd like to retrieve. + +For example, to fetch the latest version of a Dataset's SchemaMetadata aspect, you would issue the following query: + +``` +curl 'http://localhost:8080/aspects/urn%3Ali%3Adataset%3A(urn%3Ali%3AdataPlatform%3Afoo%2Cbar%2CPROD)?aspect=schemaMetadata&version=0' + +{ + "version":0, + "aspect":{ + "com.linkedin.schema.SchemaMetadata":{ + "created":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "platformSchema":{ + "com.linkedin.schema.KafkaSchema":{ + "documentSchema":"{\"type\":\"record\",\"name\":\"MetadataChangeEvent\",\"namespace\":\"com.linkedin.mxe\",\"doc\":\"Kafka event for proposing a metadata change for an entity.\",\"fields\":[{\"name\":\"auditHeader\",\"type\":{\"type\":\"record\",\"name\":\"KafkaAuditHeader\",\"namespace\":\"com.linkedin.avro2pegasus.events\",\"doc\":\"Header\"}}]}" + } + }, + "lastModified":{ + "actor":"urn:li:corpuser:fbar", + "time":0 + }, + "schemaName":"FooEvent", + "fields":[ + { + "fieldPath":"foo", + "description":"Bar", + "type":{ + "type":{ + "com.linkedin.schema.StringType":{ + + } + } + }, + "nativeDataType":"string" + } + ], + "version":0, + "hash":"", + "platform":"urn:li:dataPlatform:foo" + } + } +} +``` + +#### Fetching Timeseries Aspects + +DataHub supports an API for fetching a group of Timeseries aspects about an Entity. For example, you may want to use this API +to fetch recent profiling runs & statistics about a Dataset. To do so, you can issue a "get" request against the `/aspects` endpoint. + +For example, to fetch dataset profiles (ie. stats) for a Dataset, you would issue the following query: + +``` +curl -X POST 'http://localhost:8080/aspects?action=getTimeseriesAspectValues' \ +--data '{ + "urn": "urn:li:dataset:(urn:li:dataPlatform:redshift,global_dev.larxynx_carcinoma_data_2020,PROD)", + "entity": "dataset", + "aspect": "datasetProfile", + "startTimeMillis": 1625122800000, + "endTimeMillis": 1627455600000 +}' + +{ + "value":{ + "limit":2000, + "aspectName":"datasetProfile", + "endTimeMillis":1627455600000, + "startTimeMillis":1625122800000, + "entityName":"dataset", + "values":[ + { + "aspect":{ + "value":"{\"timestampMillis\":1626912000000,\"fieldProfiles\":[{\"uniqueProportion\":1.0,\"sampleValues\":[\"123MMKK12\",\"13KDFMKML\",\"123NNJJJL\"],\"fieldPath\":\"id\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":3742},{\"uniqueProportion\":1.0,\"min\":\"1524406400000\",\"max\":\"1624406400000\",\"sampleValues\":[\"1640023230002\",\"1640343012207\",\"16303412330117\"],\"mean\":\"1555406400000\",\"fieldPath\":\"date\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":3742},{\"uniqueProportion\":0.037,\"min\":\"21\",\"median\":\"68\",\"max\":\"92\",\"sampleValues\":[\"45\",\"65\",\"81\"],\"mean\":\"65\",\"distinctValueFrequencies\":[{\"value\":\"12\",\"frequency\":103},{\"value\":\"54\",\"frequency\":12}],\"fieldPath\":\"patient_age\",\"nullCount\":0,\"nullProportion\":0.0,\"uniqueCount\":79},{\"uniqueProportion\":0.00820873786407767,\"sampleValues\":[\"male\",\"female\"],\"fieldPath\":\"patient_gender\",\"nullCount\":120,\"nullProportion\":0.03,\"uniqueCount\":2}],\"rowCount\":3742,\"columnCount\":4}", + "contentType":"application/json" + } + }, + ] + } +} +``` + +You'll notice that the aspect itself is serialized as escaped JSON. This is part of a shift toward a more generic set of READ / WRITE APIs +that permit serialization of aspects in different ways. By default, the content type will be JSON, and the aspect can be deserialized into a normal JSON object +in the language of your choice. Note that this will soon become the de-facto way to both write and read individual aspects. + +### Search Query + +A search query allows you to search for entities matching an arbitrary string. + +For example, to search for entities matching the term "customers", we can use the following CURL: + +``` +curl --location --request POST 'http://localhost:8080/entities?action=search' \ +--header 'X-RestLi-Protocol-Version: 2.0.0' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "input": "\"customers\"", + "entity": "chart", + "start": 0, + "count": 10 +}' +``` + +The notable parameters are `input` and `entity`. `input` specifies the query we are issuing and `entity` specifies the Entity Type we want to search over. This is the common name of the Entity as defined in the @Entity definition. The response contains a list of Urns, that can be used to fetch the full entity. + +### Relationship Query + +A relationship query allows you to find Entity connected to a particular source Entity via an edge of a particular type. + +For example, to find the owners of a particular Chart, we can use the following CURL: + +``` +curl --location --request GET --header 'X-RestLi-Protocol-Version: 2.0.0' 'http://localhost:8080/relationships?direction=OUTGOING&urn=urn%3Ali%3Achart%3Acustomers&types=List(OwnedBy)' +``` + +The notable parameters are `direction`, `urn` and `types`. The response contains _Urns_ associated with all entities connected +to the primary entity (`urn:li:chart:customer)` by an relationship named "OwnedBy". That is, it permits fetching the owners of a given +chart. + +### Special Aspects + +There are a few special aspects worth mentioning: + +1. Key aspects: Contain the properties that uniquely identify an Entity. +2. Browse Paths aspect: Represents a hierarchical path associated with an Entity. + +#### Key aspects + +As introduced above, Key aspects are structs / records that contain the fields that uniquely identify an Entity. There are +some constraints about the fields that can be present in Key aspects: + +- All fields must be of STRING or ENUM type +- All fields must be REQUIRED + +Keys can be created from and turned into _Urns_, which represent the stringified version of the Key record. +The algorithm used to do the conversion is straightforward: the fields of the Key aspect are substituted into a +string template based on their index (order of definition) using the following template: + +```aidl +// Case 1: # key fields == 1 +urn:li::key-field-1 + +// Case 2: # key fields > 1 +urn:li::(key-field-1, key-field-2, ... key-field-n) +``` + +By convention, key aspects are defined under [metadata-models/src/main/pegasus/com/linkedin/metadata/key](https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/metadata/key). + +##### Example + +A CorpUser can be uniquely identified by a "username", which should typically correspond to an LDAP name. + +Thus, it's Key Aspect is defined as the following: + +```aidl +namespace com.linkedin.metadata.key + +/** + * Key for a CorpUser + */ +@Aspect = { + "name": "corpUserKey" +} +record CorpUserKey { + /** + * The name of the AD/LDAP user. + */ + username: string +} +``` + +and it's Entity Snapshot model is defined as + +```aidl +/** + * A metadata snapshot for a specific CorpUser entity. + */ +@Entity = { + "name": "corpuser", + "keyAspect": "corpUserKey" +} +record CorpUserSnapshot { + + /** + * URN for the entity the metadata snapshot is associated with. + */ + urn: CorpuserUrn + + /** + * The list of metadata aspects associated with the CorpUser. Depending on the use case, this can either be all, or a selection, of supported aspects. + */ + aspects: array[CorpUserAspect] +} +``` + +Using a combination of the information provided by these models, we are able to generate the Urn corresponding to a CorpUser as + +``` +urn:li:corpuser: +``` + +Imagine we have a CorpUser Entity with the username "johnsmith". In this world, the JSON version of the Key Aspect associated with the Entity would be + +```aidl +{ + "username": "johnsmith" +} +``` + +and its corresponding Urn would be + +```aidl +urn:li:corpuser:johnsmith +``` + +#### BrowsePaths aspect + +The BrowsePaths aspect allows you to define a custom "browse path" for an Entity. A browse path is a way to hierarchically organize +entities. They manifest within the "Explore" features on the UI, allowing users to navigate through trees of related entities of a given type. + +To support browsing a particular entity, add the "browsePaths" aspect to the entity in your `entity-registry.yml` file. + +```aidl +/// entity-registry.yml +entities: + - name: dataset + doc: Datasets represent logical or physical data assets stored or represented in various data platforms. Tables, Views, Streams are all instances of datasets. + keyAspect: datasetKey + aspects: + ... + - browsePaths +``` + +By declaring this aspect, you can produce custom browse paths as well as query for browse paths manually using a CURL like the following: + +```aidl +curl --location --request POST 'http://localhost:8080/entities?action=browse' \ +--header 'X-RestLi-Protocol-Version: 2.0.0' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "path": "/my/custom/browse/path", + "entity": "dataset", + "start": 0, + "limit": 10 +}' +``` + +Please note you must provide: + +- The "/"-delimited root path for which to fetch results. +- An entity "type" using its common name ("dataset" in the example above). + +### Types of Aspect + +There are 2 "types" of Metadata Aspects. Both are modeled using PDL schemas, and both can be ingested in the same way. +However, they differ in what they represent and how they are handled by DataHub's Metadata Service. + +#### 1. Versioned Aspects + +Versioned Aspects each have a **numeric version** associated with them. When a field in an aspect changes, a new +version is automatically created and stored within DataHub's backend. In practice, all versioned aspects are stored inside a relational database +that can be backed up and restored. Versioned aspects power much of the UI experience you're used to, including Ownership, Descriptions, +Tags, Glossary Terms, and more. Examples include Ownership, Global Tags, and Glossary Terms. + +#### 2. Timeseries Aspects + +Timeseries Aspects each have a **timestamp** associated with them. They are useful for representing +time-ordered events about an Entity. For example, the results of profiling a Dataset, or a set of Data Quality checks that +run every day. It is important to note that Timeseries aspects are NOT persisted inside the relational store, and are instead +persisted only in the search index (e.g. elasticsearch) and the message queue (Kafka). This makes restoring timeseries aspects +in a disaster scenario a bit more challenge. Timeseries aspects can be queried by time range, which is what makes them most different from Versioned Aspects. +A timeseries aspect can be identified by the "timeseries" [type](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl#L10) in its [@Aspect](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl#L8) annotation. +Examples include [DatasetProfile](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl) & [DatasetUsageStatistics](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetUsageStatistics.pdl). + +Timeseries aspects are aspects that have a timestampMillis field, and are meant for aspects that continuously change on a +timely basis e.g. data profiles, usage statistics, etc. + +Each timeseries aspect must be declared "type": "timeseries" and must +include [TimeseriesAspectBase](https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/timeseries/TimeseriesAspectBase.pdl) +, which contains a timestampMillis field. + +Timeseries aspect can also have fields annotated with @Searchable and @Relationship. + +Please refer +to [DatasetProfile](https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetProfile.pdl) +to see an example of a timeseries aspect. + +Because timeseries aspects are updated on a frequent basis, ingests of these aspects go straight to elastic search ( +instead of being stored in local DB). + +You can retrieve timeseries aspects using the "aspects?action=getTimeseriesAspectValues" end point. + +##### Aggregatable Timeseries aspects + +Being able to perform SQL like _group by + aggregate_ operations on the timeseries aspects is a very natural use-case for +this kind of data (dataset profiles, usage statistics etc.). This section describes how to define, ingest and perform an +aggregation query against a timeseries aspect. + +###### Defining a new aggregatable Timeseries aspect. + +The _@TimeseriesField_ and the _@TimeseriesFieldCollection_ are two new annotations that can be attached to a field of +a _Timeseries aspect_ that allows it to be part of an aggregatable query. The kinds of aggregations allowed on these +annotated fields depends on the type of the field, as well as the kind of aggregation, as +described [here](#performing-an-aggregation-on-a-timeseries-aspect). + +- `@TimeseriesField = {}` - this annotation can be used with any type of non-collection type field of the aspect such as + primitive types and records (see the fields _stat_, _strStat_ and _strArray_ fields + of [TestEntityProfile.pdl](https://github.com/datahub-project/datahub/blob/master/test-models/src/main/pegasus/com/datahub/test/TestEntityProfile.pdl)). + +- The `@TimeseriesFieldCollection {"key":""}` annotation allows for + aggregation support on the items of a collection type (supported only for the array type collections for now), where the + value of `"key"` is the name of the field in the collection item type that will be used to specify the group-by clause ( + see _userCounts_ and _fieldCounts_ fields of [DatasetUsageStatistics.pdl](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/dataset/DatasetUsageStatistics.pdl)). + +In addition to defining the new aspect with appropriate Timeseries annotations, +the [entity-registry.yml](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/resources/entity-registry.yml) +file needs to be updated as well. Just add the new aspect name under the list of aspects against the appropriate entity as shown below, such as `datasetUsageStatistics` for the aspect DatasetUsageStatistics. + +```yaml +entities: + - name: dataset + keyAspect: datasetKey + aspects: + - datasetProfile + - datasetUsageStatistics +``` + +###### Ingesting a Timeseries aspect + +The timeseries aspects can be ingested via the GMS REST endpoint `/aspects?action=ingestProposal` or via the python API. + +Example1: Via GMS REST API using curl. + +```shell +curl --location --request POST 'http://localhost:8080/aspects?action=ingestProposal' \ +--header 'X-RestLi-Protocol-Version: 2.0.0' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "proposal" : { + "entityType": "dataset", + "entityUrn" : "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "changeType" : "UPSERT", + "aspectName" : "datasetUsageStatistics", + "aspect" : { + "value" : "{ \"timestampMillis\":1629840771000,\"uniqueUserCount\" : 10, \"totalSqlQueries\": 20, \"fieldCounts\": [ {\"fieldPath\": \"col1\", \"count\": 20}, {\"fieldPath\" : \"col2\", \"count\": 5} ]}", + "contentType": "application/json" + } + } +}' +``` + +Example2: Via Python API to Kafka(or REST) + +```python +from datahub.metadata.schema_classes import ( + ChangeTypeClass, + DatasetFieldUsageCountsClass, + DatasetUsageStatisticsClass, +) +from datahub.emitter.kafka_emitter import DatahubKafkaEmitter +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + +usageStats = DatasetUsageStatisticsClass( + timestampMillis=1629840771000, + uniqueUserCount=10, + totalSqlQueries=20, + fieldCounts=[ + DatasetFieldUsageCountsClass( + fieldPath="col1", + count=10 + ) + ] + ) + +mcpw = MetadataChangeProposalWrapper( + entityType="dataset", + aspectName="datasetUsageStatistics", + changeType=ChangeTypeClass.UPSERT, + entityUrn="urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + aspect=usageStats, +) + +# Instantiate appropriate emitter (kafka_emitter/rest_emitter) +# my_emitter = DatahubKafkaEmitter("""""") +my_emitter = DatahubRestEmitter("http://localhost:8080") +my_emitter.emit(mcpw) +``` + +###### Performing an aggregation on a Timeseries aspect + +Aggreations on timeseries aspects can be performed by the GMS REST API for `/analytics?action=getTimeseriesStats` which +accepts the following params. + +- `entityName` - The name of the entity the aspect is associated with. +- `aspectName` - The name of the aspect. +- `filter` - Any pre-filtering criteria before grouping and aggregations are performed. +- `metrics` - A list of aggregation specification. The `fieldPath` member of an aggregation specification refers to the + field name against which the aggregation needs to be performed, and the `aggregationType` specifies the kind of aggregation. +- `buckets` - A list of grouping bucket specifications. Each grouping bucket has a `key` field that refers to the field + to use for grouping. The `type` field specifies the kind of grouping bucket. + +We support three kinds of aggregations that can be specified in an aggregation query on the Timeseries annotated fields. +The values that `aggregationType` can take are: + +- `LATEST`: The latest value of the field in each bucket. Supported for any type of field. +- `SUM`: The cumulative sum of the field in each bucket. Supported only for integral types. +- `CARDINALITY`: The number of unique values or the cardinality of the set in each bucket. Supported for string and + record types. + +We support two types of grouping for defining the buckets to perform aggregations against: + +- `DATE_GROUPING_BUCKET`: Allows for creating time-based buckets such as by second, minute, hour, day, week, month, + quarter, year etc. Should be used in conjunction with a timestamp field whose value is in milliseconds since _epoch_. + The `timeWindowSize` param specifies the date histogram bucket width. +- `STRING_GROUPING_BUCKET`: Allows for creating buckets grouped by the unique values of a field. Should always be used in + conjunction with a string type field. + +The API returns a generic SQL like table as the `table` member of the output that contains the results of +the `group-by/aggregate` query, in addition to echoing the input params. + +- `columnNames`: the names of the table columns. The group-by `key` names appear in the same order as they are specified + in the request. Aggregation specifications follow the grouping fields in the same order as specified in the request, + and will be named `_`. +- `columnTypes`: the data types of the columns. +- `rows`: the data values, each row corresponding to the respective bucket(s). + +Example: Latest unique user count for each day. + +```shell +# QUERY +curl --location --request POST 'http://localhost:8080/analytics?action=getTimeseriesStats' \ +--header 'X-RestLi-Protocol-Version: 2.0.0' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "entityName": "dataset", + "aspectName": "datasetUsageStatistics", + "filter": { + "criteria": [] + }, + "metrics": [ + { + "fieldPath": "uniqueUserCount", + "aggregationType": "LATEST" + } + ], + "buckets": [ + { + "key": "timestampMillis", + "type": "DATE_GROUPING_BUCKET", + "timeWindowSize": { + "multiple": 1, + "unit": "DAY" + } + } + ] +}' + +# SAMPLE RESPOSNE +{ + "value": { + "filter": { + "criteria": [] + }, + "aspectName": "datasetUsageStatistics", + "entityName": "dataset", + "groupingBuckets": [ + { + "type": "DATE_GROUPING_BUCKET", + "timeWindowSize": { + "multiple": 1, + "unit": "DAY" + }, + "key": "timestampMillis" + } + ], + "aggregationSpecs": [ + { + "fieldPath": "uniqueUserCount", + "aggregationType": "LATEST" + } + ], + "table": { + "columnNames": [ + "timestampMillis", + "latest_uniqueUserCount" + ], + "rows": [ + [ + "1631491200000", + "1" + ] + ], + "columnTypes": [ + "long", + "int" + ] + } + } +} +``` + +For more examples on the complex types of group-by/aggregations, refer to the tests in the group `getAggregatedStats` of [TimeseriesAspectServiceTestBase.java](https://github.com/datahub-project/datahub/blob/master/metadata-io/src/test/java/com/linkedin/metadata/timeseries/search/TimeseriesAspectServiceTestBase.java). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/ownership/ownership-types.md b/docs-archive/versioned_docs/version-1.5.0/docs/ownership/ownership-types.md new file mode 100644 index 00000000..a60bc41c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/ownership/ownership-types.md @@ -0,0 +1,194 @@ +--- +title: Custom Ownership Types +slug: /ownership/ownership-types +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/ownership/ownership-types.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Custom Ownership Types + + + +**🤝 Version compatibility** + +> DataHub Core: **0.10.4** | DataHub Cloud: **0.2.8** + +## What are Custom Ownership Types? + +Custom Ownership Types are an improvement on the way to establish ownership relationships between users and the data assets they manage within DataHub. + +## Why Custom Ownership Types? + +DataHub brings a pre-defined opinion on ownership relationships. We are aware that it may not always precisely match what you may need. +With this feature you can modify it to better match the terminology used by stakeholders. + +## Benefits of Custom Ownership Types + +Custom ownership types allow users to bring in their organization's ownership nomenclature straight into DataHub. +This allows stakeholders to discover what relationships an owner of an entity has using the language already in-use at organizations. + +## How Can You Use Custom Ownership Types? + +Custom Ownership types have been implemented as a net-new entity in DataHub's Metadata Model meaning all entity-related APIs can be used for them. +Additionally, they can be managed through DataHub's Admin UI and then used for ownership across the system in the same way pre-existing ownership types are. + +## Custom Ownership Types Setup, Prerequisites, and Permissions + +What you need to create and add ownership types: + +- **Manage Ownership Types** metadata privilege to create/delete/update Ownership Types at the platform level. These can be granted by a [Platform Policy](./../authorization/policies.md#platform-policies). +- **Edit Owners** metadata privilege to add or remove an owner with an associated custom ownership type for a given entity. + +You can create this privileges by creating a new [Metadata Policy](./../authorization/policies.md#metadata-policies). + +## Using Custom Ownership Types + +Custom Ownership Types can be managed using the UI, via a graphQL command or ingesting an MCP which can be managed using software engineering (GitOps) practices. + +### Managing Custom Ownership Types (UI) + +To manage a Custom Ownership type, first navigate to the DataHub Admin page: + +

+ +

+ +

+ +Then navigate to the `Ownership Types` tab under the `Management` section. + +To create a new type simply click '+ Create new Ownership Type'. + +This will open a new modal where you can configure your Ownership Type. + +Inside the form, you can choose a name for your Ownership Type. You can also add descriptions for your ownership types to help other users more easily understand their meaning. + +Don't worry, this can be changed later. + +

+ +

+ +Once you've chosen a name and a description, click 'Save' to create the new Ownership Type. + +You can also edit and delete types in this UI by click on the ellipsis in the management view for the type you wish to change/delete. + +### Managing Custom Ownership Types (CLI) + +Just like all other DataHub metadata entities DataHub ships with a JSON-based custom ownership type spec, for defining and managing Custom Ownership Types as code. + +Here is an example of a custom ownership type named "Architect": + +```json +# Inlined from /metadata-ingestion/examples/ownership/ownership_type.json +[ + { + "auditHeader":null, + "entityType":"ownershipType", + "entityUrn": "urn:li:ownershipType:architect", + "changeType":"UPSERT", + "aspectName":"ownershipTypeInfo", + "aspect":{ + "value":"{\"name\": \"Architect\", \"description\": \"Technical person responsible for the asset\", \"created\": {\"time\": 1674291843000, \"actor\": \"urn:li:corpuser:jdoe\", \"impersonator\": null},\n\"lastModified\": {\"time\": 1674291843000, \"actor\": \"urn:li:corpuser:jdoe\", \"impersonator\": null}}", + "contentType":"application/json" + }, + "systemMetadata":null + } +] +``` + +To upload this file to DataHub, use the `datahub` cli via the `ingest` group of commands using the file-based recipe: + +```yaml +# see https://docs.datahub.com/docs/generated/ingestion/sources/file for complete documentation +source: + type: "file" + config: + # path to json file + path: "metadata-ingestion/examples/ownership/ownership_type.json" + +# see https://docs.datahub.com/docs/metadata-ingestion/sink_docs/datahub for complete documentation +sink: + type: "datahub-rest" + config: + server: "http://localhost:9002/api/gms" +``` + +Finally running + +```shell +datahub ingest -c recipe.yaml +``` + +For any update you wish to do, simply update the json file and re-ingest via the cli. + +To delete the ownership type, simply run a [delete command](../how/delete-metadata.md#soft-delete-the-default) for the urn of the ownership type in question, in this case `urn:li:ownershipType:architect`. + +### Managing Custom Ownership Types (GraphQL) + +You can also create/update/delete custom ownership types using DataHub's built-in [`GraphiQL` editor](../api/graphql/how-to-set-up-graphql.md#graphql-explorer-graphiql): + +```json +mutation { + createOwnershipType( + input: { + name: "Architect" + description: "Technical person responsible for the asset" + } + ) { + urn + type + info { + name + description + } + } +} +``` + +If you see the following response, the operation was successful: + +```json +{ + "data": { + "createOwnershipType": { + "urn": "urn:li:ownershipType:ccf9aa80-e3f3-4620-93a1-8d4a2ceaf5de", + "type": "CUSTOM_OWNERSHIP_TYPE", + "status": null, + "info": { + "name": "Architect", + "description": "Technical person responsible for the asset", + "created": null, + "lastModified": null + } + } + }, + "extensions": {} +} +``` + +There are also `updateOwnershipType`, `deleteOwnershipType` and `listOwnershipTypes` endpoints for CRUD operations. + +Feel free to read our [GraphQL reference documentation](../api/graphql/overview.md) on these endpoints. + +### Assigning a Custom Ownership Type to an Entity (UI) + +You can assign an owner with a custom ownership type to an entity either using the Entity's page as the starting point. + +On an Entity's profile page, use the right sidebar to locate the Owners section. + +

+ +

+ +Click 'Add Owners', select the owner you want and then search for the Custom Ownership Type you'd like to add this asset to. When you're done, click 'Add'. + +

+ +

+ +To remove ownership from an asset, click the 'x' icon on the Owner label. + +> Notice: Adding or removing an Owner to an asset requires the `Edit Owners` Metadata Privilege, which can be granted +> by a [Policy](./../authorization/policies.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/platform-instances.md b/docs-archive/versioned_docs/version-1.5.0/docs/platform-instances.md new file mode 100644 index 00000000..9f9ba405 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/platform-instances.md @@ -0,0 +1,224 @@ +--- +title: Working With Platform Instances +slug: /platform-instances +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/platform-instances.md +--- +# Working With Platform Instances + +DataHub's metadata model for Datasets supports a three-part key currently: + +- Data Platform (e.g. `urn:li:dataPlatform:mysql)` +- Name (e.g. db.schema.name) +- Env or Fabric (e.g. DEV, PROD, etc.) + +This naming scheme unfortunately does not allow for easy representation of the multiplicity of platforms (or technologies) that might be deployed at an organization within the same environment or fabric. For example, an organization might have multiple Redshift instances in Production and would want to see all the data assets located in those instances inside the DataHub metadata repository. + +**Note**: While platform instances provide one solution to this problem it comes with trade-offs with respect to immutability. DataHub also offers alternative approaches for organizing and managing multiple platform instances. See the [Alternative Approaches](#alternative-approaches) section below for more information. + +As part of the `v0.8.24+` releases, we are unlocking the first phase of supporting Platform Instances in the metadata model. This is done via two main additions: + +- The `dataPlatformInstance` aspect that has been added to Datasets which allows datasets to be associated to an instance of a platform +- Enhancements to all ingestion sources that allow them to attach a platform instance to the recipe that changes the generated urns to go from `urn:li:dataset:(urn:li:dataPlatform:,,ENV)` format to `urn:li:dataset:(urn:li:dataPlatform:,<instance.name>,ENV)` format. Sources that produce lineage to datasets in other platforms (e.g. Looker, Superset etc) also have specific configuration additions that allow the recipe author to specify the mapping between a platform and the instance name that it should be mapped to. + +## ⚠️ Critical: URN Immutability + +**DataHub URNs are immutable identifiers that must remain unchanged once assigned to an entity.** This immutability is fundamental to maintaining data integrity, lineage tracking, and consistent references throughout the system. Once a URN is created, it should never be modified, even if the underlying data asset's attributes change. + +### The URN Immutability Challenge + +Many organizations face a critical challenge: **URNs serve dual purposes** - they are both internal system identifiers AND visible user-facing identifiers in the DataHub UI. This creates a conflict when organizational taxonomy changes (domains, products, systems) because: + +1. **Orphaned Assets**: When URNs change, all metadata added outside of ingestion (descriptions, tags, lineage, ownership) associated with the old asset +2. **Integration Disruption**: Downstream applications and integrations that rely on specific URNs break +3. **User Confusion**: URNs visible in the UI become outdated and misleading +4. **Operational Overhead**: Teams must migrate all references to new URNs + +### Solution: Separate Technical Identifiers from Business Context + +When establishing platform instance naming conventions, it is crucial to choose names that are: + +- **Intrinsic to the data**: Based on stable, inherent properties of the data asset +- **Not subject to change**: Avoid names that might change due to organizational restructuring, technology migrations, or operational changes +- **Consistent across all ingestion sources**: The same platform instance name must be used consistently across all recipes to ensure URN alignment + +

+ +

+ +## Naming Platform Instances + +When configuring a platform instance, choose an instance name that is understandable and will be stable for the foreseeable future. e.g. `core_warehouse` or `finance_redshift` are allowed names, as are pure guids like `a37dc708-c512-4fe4-9829-401cd60ed789`. Remember that whatever instance name you choose, you will need to specify it in more than one recipe to ensure that the identifiers produced by different sources will line up. + +### Best Practices for Platform Instance Naming + +To ensure URN immutability and long-term stability, platform instance names should be **technical identifiers** that are intrinsic to the infrastructure, not business concepts. Use DataHub's built-in features for domains, ownership, and business context. + +**✅ Good Examples:** + +- **Infrastructure identifiers**: `us-east-1-cluster-1`, `eu-west-2-cluster-2` +- **Technical naming**: `primary-redshift`, `secondary-mysql`, `analytics-snowflake` +- **GUIDs/UUIDs**: `a37dc708-c512-4fe4-9829-401cd60ed789` +- **Infrastructure codes**: `rds-prod-001`, `redshift-analytics-01` + +**❌ Avoid These Patterns:** + +- **Organizational taxonomy**: `company.domain.product.system` (domains, products, systems change) +- **Business domain names**: `customer_data_warehouse`, `finance_redshift` (use DataHub domains instead) +- **Ownership references**: `john_warehouse`, `sarah_analytics` (use DataHub ownership features) +- **Version numbers**: `redshift_v2`, `mysql_8_0` (use DataHub's versioning capabilities) +- **Temporary indicators**: `temp_warehouse`, `migration_db` +- **Technology migration names**: `legacy_mysql`, `old_redshift` (use DataHub tags instead) + +**Key Principles:** + +1. **Technical focus**: Use infrastructure-level identifiers, not business concepts +2. **Stability**: Choose names that reflect permanent technical characteristics +3. **Consistency**: Use the same naming pattern across all platform instances +4. **Uniqueness**: Ensure each platform instance has a unique identifier +5. **Separation of concerns**: Use DataHub's domain and ownership features for business context + +**Note**: Business context like domains, ownership, data classification, and technology migration status should be managed through DataHub's dedicated features (domains, ownership, tags, etc.) rather than embedded in the platform instance name. Environment information is best handled by tags instead of fabric type which allows for promotion over time, and versioning should use DataHub's versioning capabilities. + +## Enabling Platform Instances + +Read the Ingestion source specific guides for how to enable platform instances in each of them. +The general pattern is to add an additional optional configuration parameter called `platform_instance`. + +e.g. here is how you would configure a recipe to ingest a mysql instance that you want to call `primary-mysql` + +```yaml +source: + type: mysql + config: + # Coordinates + host_port: localhost:3306 + platform_instance: primary-mysql + database: dbname + + # Credentials + username: root + password: example + +sink: + # sink configs +``` + +## Alternative Solutions to URN Immutability Challenges + +Instead of changing URNs when organizational taxonomy evolves, DataHub provides several alternative approaches that maintain URN immutability while enabling flexible business context management: + +### Recommended Approach: Separate Technical from Business Context + +The most effective solution is to design your platform instance naming to be **technically stable** while using DataHub's metadata features for **business context**: + +1. **Use Stable Technical Identifiers**: Design platform instance names that won't change + + - ✅ `us-east-1-cluster-001`, `anomalo-prod-01`, `primary-redshift` + - ❌ `company.domain.product.system` (changes when taxonomy evolves) + +2. **Leverage DataHub's Business Context Features**: + - **Data Products**: Group related assets for business purposes + - **Tags and Custom Properties**: Add flexible metadata that can be updated + - **Glossary Terms**: Associate business concepts with technical assets + - **Domains**: Use DataHub domains for business domain classification + +## Detailed Alternative Approaches + +DataHub offers several organizational concepts that can complement or serve as alternatives to platform instances: + +### Data Products + +**Data Products** group related data assets for business purposes, following data mesh principles: + +- **Domain-Oriented**: Owned by specific business teams +- **Cohesive Units**: Related assets (tables, dashboards, pipelines) managed together +- **Business Context**: Focus on business value and consumer needs +- **Cross-Platform**: Can span multiple platform instances + +**Example Data Product**: + +``` +Customer Analytics Data Product +├── Tables from Redshift Cluster 1 +├── Tables from Snowflake Analytics +├── Dashboards from Looker +└── Pipelines from Airflow +``` + +### Additional Metadata Management Approaches + +DataHub offers several other ways to handle organizational context without changing URNs: + +#### Tags and Labels + +- **Purpose**: Add flexible metadata context without changing URNs +- **Use Cases**: + - Tag datasets with organizational context (domain, product, system) + - Add environment-specific labels + - Mark migration status or legacy systems +- **Benefits**: Flexible, searchable, and can be updated without URN changes +- **Example**: Tag a dataset with `domain.voice`, `product.billing`, `system.anomalo` + +#### Custom Properties + +- **Purpose**: Add structured metadata to entities +- **Use Cases**: + - Store organizational taxonomy as structured data + - Add infrastructure-specific metadata + - Track business context that changes over time +- **Benefits**: Structured data that can be queried and filtered +- **Example**: Add custom property `org_domain: "voice"` that can be updated when domain changes + +#### Glossary Terms and Business Context + +- **Purpose**: Associate business meaning with technical assets +- **Use Cases**: + - Link datasets to business concepts + - Associate platform instances with business domains + - Create business-friendly groupings +- **Benefits**: Bridges technical and business perspectives +- **Example**: Associate datasets with glossary term "Customer Billing" that can be renamed without affecting URNs + +#### Search and Discovery Features + +- **Purpose**: Find and organize assets without changing URNs +- **Use Cases**: + - Search by organizational tags + - Filter by custom properties + - Use saved searches for common organizational queries +- **Benefits**: Flexible discovery without structural changes + +#### DataHub Actions and Automation + +- **Purpose**: Automate metadata management +- **Use Cases**: + - Auto-tag datasets based on organizational context + - Automatically assign ownership based on business rules + - Sync metadata across platform instances +- **Benefits**: Reduces manual effort and ensures consistency + +### Comparison of Approaches + +| Approach | URN Impact | Flexibility | Complexity | Best Use Case | +| ---------------------- | ----------- | ----------- | ---------- | ------------------------------------------- | +| **Platform Instances** | Changes URN | Low | Low | Technical differentiation needed in URNs | +| **Data Products** | No change | High | High | Business-oriented grouping across platforms | +| **Tags/Labels** | No change | High | Low | Flexible metadata and searchable context | +| **Custom Properties** | No change | Medium | Medium | Structured metadata storage | +| **Glossary Terms** | No change | High | Medium | Business context and domain association | +| **Search Features** | No change | High | Low | Discovery and organization without changes | +| **Automation** | No change | Medium | High | Consistent metadata management | + +### Choosing the Right Approach + +- **Platform Instances**: When you need technical differentiation in URNs +- **Data Products**: When you need business-oriented grouping across platforms +- **Tags/Labels**: When you need flexible, searchable metadata +- **Custom Properties**: When you need structured metadata storage +- **Glossary Terms**: When you need business context association +- **Combined Approach**: Use multiple concepts together for comprehensive organization + +## Summary + +Platform instances and data products each address different aspects of data organization in DataHub. Platform instances modify URNs to include technical identifiers, while data products provide organizational structure without changing the physical identity of the asset. For organizations with evolving taxonomy, the key is to separate technical identifiers (in URNs) from business context (in metadata), ensuring both immutability and flexibility. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/plugins.md b/docs-archive/versioned_docs/version-1.5.0/docs/plugins.md new file mode 100644 index 00000000..979e4aca --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/plugins.md @@ -0,0 +1,323 @@ +--- +title: Plugins Guide +slug: /plugins +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/plugins.md' +--- +# Plugins Guide + +Plugins are way to enhance the basic DataHub functionality in a custom manner. + +Currently, DataHub formally supports 2 types of plugins: + +- [Authentication](#authentication) +- [Authorization](#authorization) + +## Authentication + +> **Note:** This is in BETA version + +> It is recommend that you do not do this unless you really know what you are doing + +Custom authentication plugin makes it possible to authenticate DataHub users against any Identity Management System. +Choose your Identity Management System and write custom authentication plugin as per detail mentioned in this section. + +> Currently, custom authenticators cannot be used to authenticate users of DataHub's web UI. This is because the DataHub web app expects the presence of 2 special cookies PLAY_SESSION and actor which are explicitly set by the server when a login action is performed. +> Instead, custom authenticators are useful for authenticating API requests to DataHub's backend (GMS), and can stand in addition to the default Authentication performed by DataHub, which is based on DataHub-minted access tokens. + +The sample authenticator implementation can be found at [Authenticator Sample](https://github.com/datahub-project/datahub/blob/master/metadata-service/plugin/src/test/sample-test-plugins) + +### Implementing an Authentication Plugin + +1. Add _datahub-auth-api_ as compileOnly dependency: Maven coordinates of _datahub-auth-api_ can be found at [Maven](https://mvnrepository.com/artifact/io.acryl/datahub-auth-api) + + Example of gradle dependency is given below. + + ```groovy + dependencies { + + def auth_api = 'io.acryl:datahub-auth-api:0.9.3-3rc3' + compileOnly "${auth_api}" + testImplementation "${auth_api}" + + } + ``` + +2. Implement the Authenticator interface: Refer [Authenticator Sample](https://github.com/datahub-project/datahub/blob/master/metadata-service/plugin/src/test/sample-test-plugins) + +
+ Sample class which implements the Authenticator interface + + ```java + public class GoogleAuthenticator implements Authenticator { + + @Override + public void init(@Nonnull Map authenticatorConfig, @Nullable AuthenticatorContext context) { + // Plugin initialization code will go here + // DataHub will call this method on boot time + } + + @Nullable + @Override + public Authentication authenticate(@Nonnull AuthenticationRequest authenticationRequest) + throws AuthenticationException { + // DataHub will call this method whenever authentication decisions are need to be taken + // Authenticate the request and return Authentication + } + } + ``` + +
+ +3. Use `getResourceAsStream` to read files: If your plugin read any configuration file like properties or YAML or JSON or xml then use `this.getClass().getClassLoader().getResourceAsStream("")` to read that file from DataHub GMS plugin's class-path. For DataHub GMS resource look-up behavior please refer [Plugin Installation](#plugin-installation) section. Sample code of `getResourceAsStream` is available in sample Authenticator plugin [TestAuthenticator.java](https://github.com/datahub-project/datahub/blob/master/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthenticator.java). + +4. Bundle your Jar: Use `com.gradleup.shadow` gradle plugin to create an uber jar. + + To see an example of building an uber jar, check out the `build.gradle` file for the apache-ranger-plugin file of [Apache Ranger Plugin](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) for reference. + + Exclude signature files as shown in below `shadowJar` task. + + ```groovy + apply plugin: 'com.gradleup.shadow'; + shadowJar { + // Exclude com.datahub.plugins package and files related to jar signature + exclude "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA" + } + ``` + +5. Refer section [Plugin Installation](#plugin-installation) for plugin installation in DataHub environment + +## Enable GMS Authentication + +By default, authentication is disabled in DataHub GMS. + +Follow below steps to enable GMS authentication + +1. Download docker-compose.quickstart.yml: Download docker compose file [docker-compose.quickstart.yml](https://github.com/datahub-project/datahub/blob/master/docker/quickstart/docker-compose.quickstart.yml) + +2. Set environment variable: Set `METADATA_SERVICE_AUTH_ENABLED` environment variable to `true` + +3. Redeploy DataHub GMS: Below is quickstart command to redeploy DataHub GMS + + ```shell + datahub docker quickstart -f docker-compose.quickstart.yml + ``` + +## Authorization + +> **Note:** This is in BETA version + +> It is recommend that you do not do this unless you really know what you are doing + +Custom authorization plugin makes it possible to authorize DataHub users against any Access Management System. +Choose your Access Management System and write custom authorization plugin as per detail mentioned in this section. + +The sample authorizer implementation can be found at [Authorizer Sample](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) + +### Implementing an Authorization Plugin + +1. Add _datahub-auth-api_ as compileOnly dependency: Maven coordinates of _datahub-auth-api_ can be found at [Maven](https://mvnrepository.com/artifact/io.acryl/datahub-auth-api) + + Example of gradle dependency is given below. + + ```groovy + dependencies { + + def auth_api = 'io.acryl:datahub-auth-api:0.9.3-3rc3' + compileOnly "${auth_api}" + testImplementation "${auth_api}" + + } + ``` + +2. Implement the Authorizer interface: [Authorizer Sample](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) + +
+ Sample class which implements the Authorization interface + + ```java + public class ApacheRangerAuthorizer implements Authorizer { + @Override + public void init(@Nonnull Map authorizerConfig, @Nonnull AuthorizerContext ctx) { + // Plugin initialization code will go here + // DataHub will call this method on boot time + } + + @Override + public AuthorizationResult authorize(@Nonnull AuthorizationRequest request) { + // DataHub will call this method whenever authorization decisions are need be taken + // Authorize the request and return AuthorizationResult + } + + @Override + public AuthorizedActors authorizedActors(String privilege, Optional resourceSpec) { + // Need to add doc + } + } + ``` + +
+ +3. Use `getResourceAsStream` to read files: If your plugin read any configuration file like properties or YAML or JSON or xml then use `this.getClass().getClassLoader().getResourceAsStream("")` to read that file from DataHub GMS plugin's class-path. For DataHub GMS resource look-up behavior please refer [Plugin Installation](#plugin-installation) section. Sample code of `getResourceAsStream` is available in sample Authenticator plugin [TestAuthenticator.java](https://github.com/datahub-project/datahub/blob/master/metadata-service/plugin/src/test/sample-test-plugins/src/main/java/com/datahub/plugins/test/TestAuthenticator.java). + +4. Bundle your Jar: Use `com.gradleup.shadow` gradle plugin to create an uber jar. + + To see an example of building an uber jar, check out the `build.gradle` file for the apache-ranger-plugin file of [Apache Ranger Plugin](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) for reference. + + Exclude signature files as shown in below `shadowJar` task. + + ```groovy + apply plugin: 'com.gradleup.shadow'; + shadowJar { + // Exclude com.datahub.plugins package and files related to jar signature + exclude "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA" + } + ``` + +5. Install the Plugin: Refer to the section (Plugin Installation)[#plugin_installation] for plugin installation in DataHub environment + +## Plugin Installation + +DataHub's GMS Service searches for the plugins in container's local directory at location `/etc/datahub/plugins/auth/`. This location will be referred as `plugin-base-directory` hereafter. + +For docker, we set docker-compose to mount `${HOME}/.datahub` directory to `/etc/datahub` directory within the GMS containers. + +### Docker + +Follow below steps to install plugins: + +Lets consider you have created an uber jar for authorizer plugin and jar name is apache-ranger-authorizer.jar and class com.abc.RangerAuthorizer has implemented the [Authorizer](https://github.com/datahub-project/datahub/blob/master/metadata-auth/auth-api/src/main/java/com/datahub/plugins/auth/authorization/Authorizer.java) interface. + +1. Create a plugin configuration file: Create a `config.yml` file at `${HOME}/.datahub/plugins/auth/`. For more detail on configuration refer [Config Detail](#config-detail) section + +2. Create a plugin directory: Create plugin directory as `apache-ranger-authorizer`, this directory will be referred as `plugin-home` hereafter + + ```shell + mkdir -p ${HOME}/.datahub/plugins/auth/apache-ranger-authorizer + ``` + +3. Copy plugin jar to `plugin-home`: Copy `apache-ranger-authorizer.jar` to `plugin-home` + + ```shell + copy apache-ranger-authorizer.jar ${HOME}/.datahub/plugins/auth/apache-ranger-authorizer + ``` + +4. Update plugin configuration file: Add below entry in `config.yml` file, the plugin can take any arbitrary configuration under the "configs" block. in our example, there is username and password + + ```yaml + plugins: + - name: "apache-ranger-authorizer" + type: "authorizer" + enabled: "true" + params: + className: "com.abc.RangerAuthorizer" + configs: + username: "foo" + password: "fake" + ``` + +5. Restart datahub-gms container: + + On startup DataHub GMS service performs below steps + + 1. Load `config.yml` + 2. Prepare list of plugin where `enabled` is set to `true` + 3. Look for directory equivalent to plugin `name` in `plugin-base-directory`. In this case it is `/etc/datahub/plugins/auth/apache-ranger-authorizer/`, this directory will become `plugin-home` + 4. Look for `params.jarFileName` attribute otherwise look for jar having name as <plugin-name>.jar. In this case it is `/etc/datahub/plugins/auth/apache-ranger-authorizer/apache-ranger-authorizer.jar` + 5. Load class given in plugin `params.className` attribute from the jar, here load class `com.abc.RangerAuthorizer` from `apache-ranger-authorizer.jar` + 6. Call `init` method of plugin + +
On method call of `getResourceAsStream` DataHub GMS service looks for the resource in below order. + + 1. Look for the requested resource in plugin-jar file. if found then return the resource as InputStream. + 2. Look for the requested resource in `plugin-home` directory. if found then return the resource as InputStream. + 3. Look for the requested resource in application class-loader. if found then return the resource as InputStream. + 4. Return `null` as requested resource is not found. + +By default, authentication is disabled in DataHub GMS, Please follow section [Enable GMS Authentication](#enable-gms-authentication) to enable authentication. + +### Kubernetes + +Helm support is coming soon. + +## Config Detail + +A sample `config.yml` can be found at [config.yml](https://github.com/datahub-project/datahub/blob/master/metadata-service/plugin/src/test/resources/valid-base-plugin-dir1/config.yml). + +`config.yml` structure: + +| Field | Required | Type | Default | Description | +| ---------------------------- | -------- | ------------------------------- | ------------------------------- | ---------------------------------------------------------------------------------------- | +| plugins[].name | ✅ | string | | name of the plugin | +| plugins[].type | ✅ | enum[authenticator, authorizer] | | type of plugin, possible values are authenticator or authorizer | +| plugins[].enabled | ✅ | boolean | | whether this plugin is enabled or disabled. DataHub GMS wouldn't process disabled plugin | +| plugins[].params.className | ✅ | string | | Authenticator or Authorizer implementation class' fully qualified class name | +| plugins[].params.jarFileName | | string | default to `plugins[].name`.jar | jar file name in `plugin-home` | +| plugins[].params.configs | | map | default to empty map | Runtime configuration required for plugin | + +> plugins[] is an array of plugin, where you can define multiple authenticator and authorizer plugins. plugin name should be unique in plugins array. + +## Plugin Permissions + +Adhere to below plugin access control to keep your plugin forward compatible. + +- Plugin should read/write file to and from `plugin-home` directory only. Refer [Plugin Installation](#plugin-installation) step2 for `plugin-home` definition +- Plugin should access port 80 or 443 or port higher than 1024 + +All other access are forbidden for the plugin. + +> Disclaimer: In BETA version your plugin can access any port and can read/write to any location on file system, however you should implement the plugin as per above access permission to keep your plugin compatible with upcoming release of DataHub. + +## Migration Of Plugins From application.yaml + +If you have any custom Authentication or Authorization plugin define in `authorization` or `authentication` section of [application.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-service/configuration/src/main/resources/application.yaml) then migrate them as per below steps. + +1. Implement Plugin: For Authentication Plugin follow steps of [Implementing an Authentication Plugin](#implementing-an-authentication-plugin) and for Authorization Plugin follow steps of [Implementing an Authorization Plugin](#implementing-an-authorization-plugin) +2. Install Plugin: Install the plugins as per steps mentioned in [Plugin Installation](#plugin-installation). Here you need to map the configuration from [application.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-service/configuration/src/main/resources/application.yaml) to configuration in `config.yml`. This mapping from `application.yaml` to `config.yml` is described below + + **Mapping for Authenticators** + + a. In `config.yml` set `plugins[].type` to `authenticator` + + b. `authentication.authenticators[].type` is mapped to `plugins[].params.className` + + c. `authentication.authenticators[].configs` is mapped to `plugins[].params.configs` + + Example Authenticator Plugin configuration in `config.yml` + + ```yaml + plugins: + - name: "apache-ranger-authenticator" + type: "authenticator" + enabled: "true" + params: + className: "com.abc.RangerAuthenticator" + configs: + username: "foo" + password: "fake" + + ``` + + **Mapping for Authorizer** + + a. In `config.yml` set `plugins[].type` to `authorizer` + + b. `authorization.authorizers[].type` is mapped to `plugins[].params.className` + + c. `authorization.authorizers[].configs` is mapped to `plugins[].params.configs` + + Example Authorizer Plugin configuration in `config.yml` + + ```yaml + plugins: + - name: "apache-ranger-authorizer" + type: "authorizer" + enabled: "true" + params: + className: "com.abc.RangerAuthorizer" + configs: + username: "foo" + password: "fake" + + ``` + +3. Move any other configurations files of your plugin to `plugin_home` directory. The detail about `plugin_home` is mentioned in [Plugin Installation](#plugin-installation) section. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/posts.md b/docs-archive/versioned_docs/version-1.5.0/docs/posts.md new file mode 100644 index 00000000..0b0b7f5b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/posts.md @@ -0,0 +1,156 @@ +--- +title: Posts +slug: /posts +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/posts.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Posts + + +DataHub allows users to make Posts that can be displayed on the app. Currently, Posts are only supported on the Home Page, but may be extended to other surfaces of the app in the future. Posts can be used to accomplish the following: + +- Allowing Admins to post announcements on the home page +- Pinning important DataHub assets or pages +- Pinning important external links + +## Posts Setup, Prerequisites, and Permissions + +Anyone can view Posts on the home page. To create Posts, a user must either have the **Create Global Announcements** Privilege, or possess the **Admin** DataHub Role. + +## Creating Posts + +### Create Posts Using the UI + +To create a post, first navigate to the Settings tab in the top-right menu of DataHub. +Once you're on the Settings page, click 'Home Page Posts'. +To create a new Post, click '+ New Post'. + +

+ Creating a new post +

+DataHub currently supports two types of Post content. Posts can either be of type **Text** or **Link**. Click on "Post Type" to switch between these types. +

+ Selecting text post type +

+

+ Selecting link post type +

+If you choose the text type, enter the title and description as prompted; if you choose the link type, enter the title and the URL of the link and the address of the image as prompted. + +Click 'Create' to complete. + +

+ Viewing posts +

+ +### Create Posts Using the GraphQL + +To create a post via API, you can call the [createPost](../graphql/mutations.md#createPost) GraphQL mutation. +To create a post via API, you can call the [createPost](../graphql/mutations.md#createPost) GraphQL mutation. + +There is only one type of Post that can be currently made, and that is a **Home Page Announcement**. This may be extended in the future to other surfaces. + +DataHub currently supports two types of Post content. Posts can either contain **TEXT** or can be a **LINK**. When creating a post through GraphQL, users will have to supply the post content. + +For **TEXT** posts, the following pieces of information are required in the `content` object (of type [UpdatePostContentInput](../graphql/inputObjects.md#updatepostcontentinput)) of the GraphQL `input` (of type [CreatePostInput](../graphql/inputObjects.md#createpostinput))). **TEXT** posts cannot be clicked. + +- `contentType: TEXT` +- `title` +- `description` + +The `link` and `media` attributes are currently unused for **TEXT** posts. + +For **LINK** posts, the following pieces of information are required in the `content` object (of type [UpdatePostContentInput](../graphql/inputObjects.md#updatepostcontentinput)) of the GraphQL `input` (of type [CreatePostInput](../graphql/inputObjects.md#createpostinput))). **LINK** posts redirect to the provided link when clicked. + +- `contentType: LINK` +- `title` +- `link` +- `media`. Currently only the **IMAGE** type is supported, and the URL of the image must be provided + +The `description` attribute is currently unused for **LINK** posts. + +Here are some examples of Posts displayed on the home page, with one **TEXT** post and two **LINK** posts. + +

+ +

+ +### GraphQL + +- [createPost](../graphql/mutations.md#createpost) +- [listPosts](../graphql/queries.md#listposts) +- [deletePosts](../graphql/queries.md#listposts) + +### Examples + +##### Create Post + +```graphql +mutation test { +  createPost( +    input: { + postType: HOME_PAGE_ANNOUNCEMENT, + content: { + contentType: TEXT, + title: "Planed Upgrade 2023-03-23 20:05 - 2023-03-23 23:05", + description: "datahub upgrade to v0.10.1" + } + } +  ) +} + +``` + +##### List Post + +```graphql +query listPosts($input: ListPostsInput!) { +  listPosts(input: $input) { +    start +    count +    total +    posts { +      urn +      type +      postType +      content { +        contentType +        title +        description +        link +        media { +          type +          location +          __typename +        } +        __typename +      } +      __typename +    } +    __typename +  } +} + +``` + +##### Input for list post + +```shell +{ +  "input": { +    "start": 0, +    "count": 10 +  } +} +``` + +##### Delete Post + +```graphql +mutation deletePosting { + deletePost ( +  urn: "urn:li:post:61dd86fa-9e76-4924-ad45-3a533671835e" + ) +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/configuration.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/configuration.md new file mode 100644 index 00000000..bb68bddc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/configuration.md @@ -0,0 +1,151 @@ +--- +title: Configuration +sidebar_label: Configuration +slug: /quick-ingestion-guides/bigquery/configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/bigquery/configuration.md +--- + +# Configuring Your BigQuery Connector to DataHub + +Now that you have created a Service Account and Service Account Key in BigQuery in [the prior step](setup.md), it's now time to set up a connection via the DataHub UI. + +## Configure Secrets + +1. Within DataHub, navigate to the **Ingestion** tab in the top, right corner of your screen + +

+ Navigate to the "Ingestion Tab" +

+ +:::note +If you do not see the Ingestion tab, please contact your DataHub admin to grant you the correct permissions +::: + +2. Navigate to the **Secrets** tab and click **Create new secret** + +

+ Secrets Tab +

+ +3. Create a Private Key secret + +This will securely store your BigQuery Service Account Private Key within DataHub + +- Enter a name like `BIGQUERY_PRIVATE_KEY` - we will use this later to refer to the secret +- Copy and paste the `private_key` value from your Service Account Key +- Optionally add a description +- Click **Create** + +

+ Private Key Secret +

+ +4. Create a Private Key ID secret + +This will securely store your BigQuery Service Account Private Key ID within DataHub + +- Click **Create new secret** again +- Enter a name like `BIGQUERY_PRIVATE_KEY_ID` - we will use this later to refer to the secret +- Copy and paste the `private_key_id` value from your Service Account Key +- Optionally add a description +- Click **Create** + +

+ Private Key Id Secret +

+ +## Configure Recipe + +5. Navigate to the **Sources** tab and click **Create new source** + +

+ Click "Create new source" +

+ +6. Select BigQuery + +

+ Select BigQuery from the options +

+ +7. Fill out the BigQuery Recipe + +You can find the following details in your Service Account Key file: + +- Project ID +- Client Email +- Client ID + +Populate the Secret Fields by selecting the Private Key and Private Key ID secrets you created in steps 3 and 4. + +

+ Fill out the BigQuery Recipe +

+ +8. Click **Test Connection** + +This step will ensure you have configured your credentials accurately and confirm you have the required permissions to extract all relevant metadata. + +

+ Test BigQuery connection +

+ +After you have successfully tested your connection, click **Next**. + +## Schedule Execution + +Now it's time to schedule a recurring ingestion pipeline to regularly extract metadata from your BigQuery instance. + +9. Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown +

+ schedule selector +

+ +10. Ensure you've configured your correct timezone +

+ timezone_selector +

+ +11. Click **Next** when you are done + +## Finish Up + +12. Name your ingestion source, then click **Save and Run** +

+ Name your ingestion +

+ +You will now find your new ingestion source running + +

+ ingestion_running +

+ +## Validate Ingestion Runs + +13. View the latest status of ingestion runs on the Ingestion page + +

+ ingestion succeeded +

+ +14. Click the plus sign to expand the full list of historical runs and outcomes; click **Details** to see the outcomes of a specific run + +

+ ingestion_details +

+ +15. From the Ingestion Run Details page, pick **View All** to see which entities were ingested + +

+ ingestion_details_view_all +

+ +16. Pick an entity from the list to manually validate if it contains the detail you expected + +

+ ingestion_details_view_all +

+ +**Congratulations!** You've successfully set up BigQuery as an ingestion source for DataHub! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/overview.md new file mode 100644 index 00000000..78b0eec3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/overview.md @@ -0,0 +1,42 @@ +--- +title: Overview +sidebar_label: Overview +slug: /quick-ingestion-guides/bigquery/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/bigquery/overview.md +--- + +# BigQuery Ingestion Guide: Overview + +## What You Will Get Out of This Guide + +This guide will help you set up the BigQuery connector through the DataHub UI to begin ingesting metadata into DataHub. + +Upon completing this guide, you will have a recurring ingestion pipeline that will extract metadata from BigQuery and load it into DataHub. This will include to following BigQuery asset types: + +- [Projects](https://cloud.google.com/bigquery/docs/resource-hierarchy#projects) +- [Datasets](https://cloud.google.com/bigquery/docs/datasets-intro) +- [Tables](https://cloud.google.com/bigquery/docs/tables-intro) +- [Views](https://cloud.google.com/bigquery/docs/views-intro) +- [Materialized Views](https://cloud.google.com/bigquery/docs/materialized-views-intro) + +This recurring ingestion pipeline will also extract: + +- **Usage statistics** to help you understand recent query activity +- **Table-level lineage** (where available) to automatically define interdependencies between datasets +- **Table- and column-level profile statistics** to help you understand the shape of the data + +:::caution +You will NOT have extracted [Routines](https://cloud.google.com/bigquery/docs/routines), [Search Indexes](https://cloud.google.com/bigquery/docs/search-intro) from BigQuery, as the connector does not support ingesting these assets +::: + +## Next Steps + +If that all sounds like what you're looking for, navigate to the [next page](setup.md), where we'll talk about prerequisites + +## Advanced Guides and Reference + +If you're looking to do something more in-depth, want to use CLI instead of the DataHub UI, or just need to look at the reference documentation for this connector, use these links: + +- Learn about CLI Ingestion in the [Introduction to Metadata Ingestion](../../../metadata-ingestion/README.md) +- [BigQuery Ingestion Reference Guide](/docs/generated/ingestion/sources/bigquery/#module-bigquery) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/setup.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/setup.md new file mode 100644 index 00000000..f71156ba --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/bigquery/setup.md @@ -0,0 +1,69 @@ +--- +title: Setup +sidebar_label: Setup +slug: /quick-ingestion-guides/bigquery/setup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/bigquery/setup.md +--- + +# BigQuery Ingestion Guide: Setup & Prerequisites + +To configure ingestion from BigQuery, you'll need a [Service Account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) configured with the proper permission sets and an associated [Service Account Key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + +This setup guide will walk you through the steps you'll need to take via your Google Cloud Console. + +## BigQuery Prerequisites + +If you do not have an existing Service Account and Service Account Key, please work with your BigQuery Admin to ensure you have the appropriate permissions and/or roles to continue with this setup guide. + +When creating and managing new Service Accounts and Service Account Keys, we have found the following permissions and roles to be required: + +- Create a Service Account: `iam.serviceAccounts.create` permission +- Assign roles to a Service Account: `serviceusage.services.enable` permission +- Set permission policy to the project: `resourcemanager.projects.setIamPolicy` permission +- Generate Key for Service Account: Service Account Key Admin (`roles/iam.serviceAccountKeyAdmin`) IAM role + +:::note +Please refer to the BigQuery [Permissions](https://cloud.google.com/iam/docs/permissions-reference) and [IAM Roles](https://cloud.google.com/iam/docs/understanding-roles) references for details +::: + +## BigQuery Setup + +1. To set up a new Service Account follow [this guide](https://cloud.google.com/iam/docs/creating-managing-service-accounts) + +2. When you are creating a Service Account, assign the following predefined Roles: + - [BigQuery Job User](https://cloud.google.com/bigquery/docs/access-control#bigquery.jobUser) + - [BigQuery Metadata Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) + - [BigQuery Resource Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer) -> This role is for Table-Level Lineage and Usage extraction + - [Logs View Accessor](https://cloud.google.com/logging/docs/access-control#logging.viewAccessor) -> This role is for Table-Level Lineage and Usage extraction + - [BigQuery Data Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer) -> This role is for Profiling + - [BigQuery Read Session User](https://cloud.google.com/bigquery/docs/access-control#bigquery.readSessionUser) -> This role is for Profiling + +:::note +You can always add/remove roles to Service Accounts later on. Please refer to the BigQuery [Manage access to projects, folders, and organizations](https://cloud.google.com/iam/docs/granting-changing-revoking-access) guide for more details. +::: + +3. To filter projects based on the `project_labels` configuration, first visit [cloudresourcemanager.googleapis.com](https://console.developers.google.com/apis/api/cloudresourcemanager.googleapis.com/overview) and enable the `Cloud Resource Manager API` + +4. Create and download a [Service Account Key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). We will use this to set up authentication within DataHub. + +The key file looks like this: + +```json +{ + "type": "service_account", + "project_id": "project-id-1234567", + "private_key_id": "d0121d0000882411234e11166c6aaa23ed5d74e0", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-----", + "client_email": "test@suppproject-id-1234567.iam.gserviceaccount.com", + "client_id": "113545814931671546333", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%suppproject-id-1234567.iam.gserviceaccount.com" +} +``` + +## Next Steps + +Once you've confirmed all of the above in BigQuery, it's time to [move on](configuration.md) to configure the actual ingestion source within the DataHub UI. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/configuration.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/configuration.md new file mode 100644 index 00000000..411b68af --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/configuration.md @@ -0,0 +1,213 @@ +--- +title: Configuration +sidebar_label: Configuration +slug: /quick-ingestion-guides/looker/configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/looker/configuration.md +--- + +# Configuring Looker & LookML Connector + +Now that you have created a DataHub-specific API key with the relevant access in [the prior step](setup.md), it's time to set up a connection via the DataHub UI. + +## Configure Secrets + +You must create two secrets to configure a connection with Looker or LookerML. + +- `LOOKER_CLIENT_ID` +- `LOOKER_CLIENT_SECRET` + +On your DataHub instance, navigate to the **Ingestion** tab in your screen's top right corner. + +

+ Navigate to the "Ingestion Tab" +

+ +:::note +If you do not see the Ingestion tab, please get in touch with your DataHub admin to grant you the correct permissions. +::: + +Navigate to the **Secrets** tab and click **Create new secret**. + +

+ Secrets Tab +

+ +First, create a secret for the **Client Id**. The value should be the **Client Id** of the API key created in the [prior step](http://localhost:3000/docs/next/quick-ingestion-guides/looker/setup#create-an-api-key). + +

+ API Key Client ID +

+ +Then, create a secret for the **Client Secret**. The value should be the **Client Secret** of the API key created in the [prior step](http://localhost:3000/docs/next/quick-ingestion-guides/looker/setup#create-an-api-key). + +

+ API Key client secret +

+ +## Configure Looker Ingestion + +### Configure Recipe + +Navigate to the **Sources** tab and click **Create new source**. + +

+ Click "Create new source" +

+ +Choose `Looker`. + +

+ Select Looker from the options +

+ +Enter the details into the Looker Recipe. + +- **Base URL:** This is your looker instance URL. (i.e. `https://.cloud.looker.com`) +- **Client ID:** Use the secret LOOKER_CLIENT_ID with the format `${LOOKER_CLIENT_ID}`. +- **Client Secret:** Use the secret LOOKER_CLIENT_SECRET with the format `${LOOKER_CLIENT_SECRET}`. + +Optionally, use the `dashboard_pattern` and `chart_pattern` fields to filter for specific dashboard and chart. + + config: + ... + dashboard_pattern: + allow: + - "2" + chart_pattern: + allow: + - "258829b1-82b1-4bdb-b9fb-6722c718bbd3" + +Your recipe should look something like this: + +

+ Looker Recipe +

+ +After completing the recipe, click **Next**. + +### Schedule Execution + +Now, it's time to schedule a recurring ingestion pipeline to extract metadata from your Looker instance regularly. + +Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown. + +

+ schedule selector +

+ +Ensure you've configured your correct timezone. + +

+ timezone_selector +

+ +Finally, click **Next** when you are done. + +### Finish Up + +Name your ingestion source, then click **Save and Run**. + +

+ Name your ingestion +

+ +You will now find your new ingestion source running. + +

+ ingestion_running +

+ +## Configure LookML Connector + +Now that you have created a DataHub-specific API key and Deploy Key with the relevant access in [the prior step](setup.md), it's time to set up a connection via the DataHub UI. + +### Configure Recipe + +Navigate to the **Sources** tab and click **Create new source**. + +

+ Click "Create new source" +

+ +Choose `LooML`. + +

+ Select Looker from the options +

+ +Enter the details into the Looker Recipe. You need to set a minimum 5 fields in the recipe for this quick ingestion guide: + +- **GitHub Repository:** This is your GitHub repository where LookML models are stored. You can provide the full URL (example: https://gitlab.com/gitlab-org/gitlab) or organization/repo; in this case, the connector assume it is a GitHub repo +- **GitHub Deploy Key:** Copy the content of `looker_datahub_deploy_key` and paste into this filed. +- **Looker Base URL:** This is your looker instance URL. (i.e. https://abc.cloud.looker.com) +- **Looker Client ID:** Use the secret LOOKER_CLIENT_ID with the format `${LOOKER_CLIENT_ID}`. +- **Looker Client Secret:** Use the secret LOOKER_CLIENT_SECRET with the format `${LOOKER_CLIENT_SECRET}`. + +Your recipe should look something like this: + +

+ LookML Recipe +

+ +After completing the recipe, click **Next**. + +### Schedule Execution + +Now, it's time to schedule a recurring ingestion pipeline to extract metadata from your Looker instance regularly. + +Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown. + +

+ schedule selector +

+ +Ensure you've configured your correct timezone. + +

+ timezone_selector +

+ +Click **Next** when you are done. + +### Finish Up + +Name your ingestion source, then click **Save and Run**. + +

+ Name your ingestion +

+ +You will now find your new ingestion source running. + +

+ ingestion_running +

+ +## Validate Ingestion Runs + +View the latest status of ingestion runs on the Ingestion page. + +

+ ingestion succeeded +

+ +Click the `+` sign to expand the complete list of historical runs and outcomes; click **Details** to see the results of a specific run. + +

+ ingestion_details +

+ +From the Ingestion Run Details page, pick **View All** to see which entities were ingested. + +

+ ingestion_details_view_all +

+ +Pick an entity from the list to manually validate if it contains the detail you expected. + +

+ ingestion_details_view_all +

+ +**Congratulations!** You've successfully set up Looker & LookML as an ingestion source for DataHub! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/overview.md new file mode 100644 index 00000000..28f40b46 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/overview.md @@ -0,0 +1,56 @@ +--- +title: Overview +sidebar_label: Overview +slug: /quick-ingestion-guides/looker/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/looker/overview.md +--- + +# Looker & LookML Ingestion Guide: Overview + +## What You Will Get Out of This Guide + +This guide will help you set up the Looker & LookML connectors to begin ingesting metadata into DataHub. +Upon completing this guide, you will have a recurring ingestion pipeline to extract metadata from Looker & LookML and load it into DataHub. + +### Looker + +Looker connector will ingest Looker asset types: + +- [Dashboards](https://cloud.google.com/looker/docs/dashboards) +- [Charts](https://cloud.google.com/looker/docs/creating-visualizations) +- [Explores](https://cloud.google.com/looker/docs/reference/param-explore-explore) +- [Schemas](https://developers.looker.com/api/explorer/4.0/methods/Metadata/connection_schemas) +- [Owners of Dashboards](https://cloud.google.com/looker/docs/creating-user-defined-dashboards) + +:::note + +To get complete Looker metadata integration (including Looker views and lineage to the underlying warehouse tables), you must also use the [lookml](/docs/generated/ingestion/sources/looker#module-lookml) connector. + +::: + +### LookML + +LookMl connector will include the following LookML asset types: + +- [LookML views from model files in a project](https://cloud.google.com/looker/docs/reference/param-view-view) +- [Metadata for dimensions](https://cloud.google.com/looker/docs/reference/param-field-dimension) +- [Metadata for measures](https://cloud.google.com/looker/docs/reference/param-measure-types) +- [Dimension Groups as tag](https://cloud.google.com/looker/docs/reference/param-field-dimension-group) + +:::note + +To get complete Looker metadata integration (including Looker views and lineage to the underlying warehouse tables), you must also use the [looker](/docs/generated/ingestion/sources/looker#module-looker) connector. + +::: + +## Next Steps + +Please continue to the [setup guide](setup.md), where we'll describe the prerequisites. + +### Reference + +If you want to ingest metadata from Looker using the DataHub CLI, check out the following resources: + +- Learn about CLI Ingestion in the [Introduction to Metadata Ingestion](../../../metadata-ingestion/README.md) +- [Looker Ingestion Source](/docs/generated/ingestion/sources/Looker) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/setup.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/setup.md new file mode 100644 index 00000000..e78bf5bc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/looker/setup.md @@ -0,0 +1,160 @@ +--- +title: Setup +sidebar_label: Setup +slug: /quick-ingestion-guides/looker/setup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/looker/setup.md +--- + +# Looker & LookML Ingestion Guide: Setup + +## Looker Prerequisites + +To configure ingestion from Looker, you'll first have to ensure you have an API key to access the Looker resources. + +### Login To Looker Instance + +Login to your Looker instance(e.g. `https://.cloud.looker.com`). + +Navigate to **Admin Panel** & click **Roles** to open Roles Panel. + +

+ Looker home page +

+ +

+ Looker roles search +

+ +### Create A New Permission Set + +On **Roles Panel**, click **New Permission Set**. + +

+ Looker new permission set +

+ +Set a name for the new permission set (e.g., _DataHub Connector Permission Set_) and select the following permissions. + +
+Permission List + +- access_data +- see_lookml_dashboards +- see_looks +- see_user_dashboards +- explore +- see_sql +- see_lookml +- clear_cache_refresh +- manage_models +- see_datagroups +- see_pdts +- see_queries +- see_schedules +- see_system_activity +- see_users + +
+ +After selecting all permissions mentioned above, click **New Permission Set** at the bottom of the page. + +

+Looker permission set window +

+ +### Create A Role + +On the **Roles** Panel, click **New Role**. + +

+Looker new role button +

+ +Set the name for the new role (e.g., _DataHub Extractor_) and set the following fields on this window. + +- Set **Permission Set** to permission set created in previous step (i.e _DataHub Connector Permission Set_) +- Set **Model Set** to `All` + +Finally, click **New Role** at the bottom of the page. + +

+ Looker new role window +

+ +### Create A New User + +On the **Admin** Panel, click **Users** to open the users panel. + +

+ Looker user search +

+ +Click **Add Users**. + +

+ Looker add user +

+ +On **Adding a new user**, set details in the following fields. + +- Add user's **Email Addresses**. +- Set **Roles** to the role created in previous step (e.g. _DataHub Extractor_) + +Finally, click **Save**. + +

+Looker new user window +

+ +### Create An API Key + +On the **User** Panel, click on the newly created user. + +

+Looker user panel +

+ +Click **Edit Keys** to open the **API Key** Panel. + +

+Looker user view +

+ +On the **API Key** Panel, click **New API Key** to generate a new **Client ID** and **Client Secret**. + +

+Looker new api key +

+ +## LookML Prerequisites + +Follow the below steps to create the GitHub Deploy Key. + +### Generate a private-public SSH key pair + +```bash +ssh-keygen -t rsa -f looker_datahub_deploy_key +# If prompted, don't add a passphrase to the key +``` + +This will typically generate two files like the one below. + +- `looker_datahub_deploy_key` (private key) +- `looker_datahub_deploy_key.pub` (public key) + +### Add Deploy Key to GitHub Repository + +First, log in to [GitHub](https://github.com). + +Navigate to **GitHub Repository** -> **Settings** -> **Deploy Keys** and add a public key (e.g. `looker_datahub_deploy_key.pub`) as deploy key with read access. + +

+Looker home page +

+ +Make a note of the private key file. You must paste the file's contents into the GitHub Deploy Key field later while [configuring](./configuration.md) ingestion on the DataHub Portal. + +## Next Steps + +Once you've done all the above steps, it's time to move on to [configuring the actual ingestion source](configuration.md) within DataHub. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/configuration.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/configuration.md new file mode 100644 index 00000000..e9c3586c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/configuration.md @@ -0,0 +1,142 @@ +--- +title: Configuration +sidebar_label: Configuration +slug: /quick-ingestion-guides/powerbi/configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/powerbi/configuration.md +--- + +# Configuring Your Power BI Connector to DataHub + +Now that you have created a DataHub-specific Microsoft Entra application with the relevant access to Power BI in [the prior step](setup.md), it's now time to set up a connection via the DataHub UI. + +## Configure Secrets + +1. Within DataHub, navigate to the **Ingestion** tab in the top, right corner of your screen + +

+ Navigate to the "Ingestion Tab" +

+ +:::note +If you do not see the Ingestion tab, please contact your DataHub admin to grant you the correct permissions +::: + +2. Navigate to the **Secrets** tab and click **Create new secret**. + +

+ Secrets Tab +

+ +3. Create a secret to store the Entra app's Client Secret value + +This will securely store your client secret value. + +- Enter a name like `POWER_BI_CLIENT_SECRET` - we will use this later to refer to the client secret +- Enter the client secret +- Optionally add a description +- Click **Create** + +

+ Entra app client secret +

+ +## Configure Recipe + +1. Navigate to the **Sources** tab and click **Create new source** + +

+ Click "Create new source" +

+ +2. Choose Power BI + +

+ Select Power BI from the options +

+ +3. Enter details into the Power BI Recipe + + You need to set minimum 3 field in the recipe: + + a. **tenant_id:** Use the `Directory (tenant) ID` from your Microsoft Entra application. + + b. **client_id:** Use the `Application (client) ID` from your Microsoft Entra application. + + c. **client_secret:** Use the secret POWER_BI_CLIENT_SECRET with the format "${POWER_BI_CLIENT_SECRET}". + + d. **environment:** (Optional) Specify the Power BI environment to connect to. Use 'commercial' for commercial Power BI (default) or 'government' for Power BI Government Community Cloud (GCC). + +Optionally, use the `workspace_id_pattern` field to filter for specific workspaces. + + config: + ... + workspace_id_pattern: + allow: + - "258829b1-82b1-4bdb-b9fb-6722c718bbd3" + +Your recipe should look something like this: + +

+ tenant id +

+ +After completing the recipe, click **Next**. + +## Schedule Execution + +Now it's time to schedule a recurring ingestion pipeline to regularly extract metadata from your Power BI instance. + +1. Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown + +

+ schedule selector +

+ +2. Ensure you've configured your correct timezone +

+ timezone_selector +

+ +3. Click **Next** when you are done + +## Finish Up + +1. Name your ingestion source, then click **Save and Run** +

+ Name your ingestion +

+ +You will now find your new ingestion source running + +

+ ingestion_running +

+ +## Validate Ingestion Runs + +1. View the latest status of ingestion runs on the Ingestion page + +

+ ingestion succeeded +

+ +2. Click the plus sign to expand the full list of historical runs and outcomes; click **Details** to see the outcomes of a specific run + +

+ ingestion_details +

+ +3. From the Ingestion Run Details page, pick **View All** to see which entities were ingested + +

+ ingestion_details_view_all +

+ +4. Pick an entity from the list to manually validate if it contains the detail you expected + +

+ ingestion_details_view_all +

+ +**Congratulations!** You've successfully set up Power BI as an ingestion source for DataHub! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/overview.md new file mode 100644 index 00000000..f59f1800 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/overview.md @@ -0,0 +1,35 @@ +--- +title: Overview +sidebar_label: Overview +slug: /quick-ingestion-guides/powerbi/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/powerbi/overview.md +--- + +# Power BI Ingestion Guide: Overview + +## What You Will Get Out of This Guide + +This guide will help you set up the Power BI connector to begin ingesting metadata into DataHub. + +Upon completing this guide, you will have a recurring ingestion pipeline that will extract metadata from Power BI and load it into DataHub. This will include to following Power BI asset types: + +- Dashboards +- Tiles +- Reports +- Pages +- Datasets +- Lineage + +_To learn more about setting these advanced values, check out the [Power BI Ingestion Source](/docs/generated/ingestion/sources/powerbi)._ + +## Next Steps + +Continue to the [setup guide](setup.md), where we'll describe the prerequisites. + +## Advanced Guides and Reference + +If you want to ingest metadata from Power BI using the DataHub CLI, check out the following resources: + +- Learn about CLI Ingestion in the [Introduction to Metadata Ingestion](../../../metadata-ingestion/README.md) +- [Power BI Ingestion Source](/docs/generated/ingestion/sources/powerbi) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/setup.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/setup.md new file mode 100644 index 00000000..fdf7e896 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/powerbi/setup.md @@ -0,0 +1,108 @@ +--- +title: Setup +sidebar_label: Setup +slug: /quick-ingestion-guides/powerbi/setup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/powerbi/setup.md +--- + +# Power BI Ingestion Guide: Setup & Prerequisites + +## Power BI Prerequisites + +DataHub connects to Power BI using a service principal / Microsoft Entra application. + +In order to configure ingestion from Power BI, you'll first have to ensure you have a Microsoft Entra application with permission to access the Power BI resources. + +### Register a new Microsoft Entra application + +Follow the below steps to register an Entra application: + +1. Login to the Azure portal at https://portal.azure.com + +2. Go to `Microsoft Entra ID` + +3. Navigate to `App registrations` + +4. Click on `+ New registration` + +5. In the `Register an application` window, fill out the `Name` of the application (e.g. `datahub-powerbi-connector-app`) and keep the other defaults as-is, then click `Register`. + +

+app_registration +

+ +6. Once the app is finished registering, the Azure portal will open up the application overview, as shown below. On this screen, note down the `Application (client) ID` and `Directory (tenant) ID`. + +

+powerbi_app_connector +

+ +### Create a client secret for the Entra application + +1. Navigate to `Certificates & secrets` on the Entra application you just created. + +2. Click on `New client secret` + +3. Generate a new secret and note down the secret `Value` + +### Create a new Microsoft Entra group + +The application you registered will need to be a member of an Entra group in order to be granted Power BI/Fabric permissions. Follow the below steps to create a new Microsoft Entra Group: + +1. Go to `Microsoft Entra ID` + +2. Navigate to `Groups` and click on `New group` + +3. In the `New group` window, leave `Group type` as the default (`Security`) and fill out the `Group name` (e.g. `datahub-powerbi-connector-group`), as shown in the below screenshot: + +

+powerbi_app_connector +

+ +### Add the Entra application as a member of the Entra group + +1. Navigate to `All groups` and click on your newly created group. + +2. Navigate to `Members` and click `Add members`. + +3. Add your Microsoft Entra application (e.g. `datahub-powerbi-connector-app`) as a member. + +### Grant permissions to access Power BI APIs + +You need to add the created Entra group under your Power BI/Fabric tenant settings in order to grant resource access. Follow the below steps to grant privileges to the group and all applications within it: + +1. Login to Power BI: https://app.powerbi.com/ + +2. Go to `Settings` -> `Admin portal` + +3. In the `Admin portal`, navigate to `Tenant settings`. + +4. For each of the following options, enable the option and add the previously created security group (e.g. _datahub-powerbi-connector-group_) under `Specific security groups`: + + - `Developer settings > Service principals can call Fabric public APIs` (or `Allow service principals to use Power BI APIs` in older versions of Power BI) + - `Admin API settings > Service principals can access read-only admin APIs` + - `Admin API settings > Enhance admin APIs responses with detailed metadata` + - `Admin API settings > Enhance admin APIs responses with DAX and mashup expressions` + +### Add the Entra application as a member of your Power BI / Fabric workspaces + +For workspaces which you want to ingest into DataHub, add the Entra application as a member. + +1. Navigate to `Workspaces` + +2. Open the workspace which you want to ingest, as shown in the below screenshot: + +

+workspace-window-underlined +

+ +3. Click `Manage Access` + +4. Click `Add people or groups` + +5. Add your Microsoft Entra application (e.g. `datahub-powerbi-connector-app`) as a member. For most cases `Viewer` role is enough, but for profiling the `Contributor` role is required. + +## Next Steps + +Once you've done all of the above steps, it's time to [move on](configuration.md) to configuring the actual ingestion source within DataHub. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/configuration.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/configuration.md new file mode 100644 index 00000000..fbd3d796 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/configuration.md @@ -0,0 +1,139 @@ +--- +title: Configuration +sidebar_label: Configuration +slug: /quick-ingestion-guides/redshift/configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/redshift/configuration.md +--- + +# Configuring Your Redshift Connector to DataHub + +Now that you have created a DataHub user in Redshift in [the prior step](setup.md), it's time to set up a connection via the DataHub UI. + +## Configure Secrets + +1. Within DataHub, navigate to the **Ingestion** tab in the top, right corner of your screen + +

+ Navigate to the "Ingestion Tab" +

+ +:::note +If you do not see the Ingestion tab, please contact your DataHub admin to grant you the correct permissions +::: + +2. Navigate to the **Secrets** tab and click **Create new secret** + +

+ Secrets Tab +

+ +3. Create a Redshift User's Password secret + +This will securely store your Redshift User's password within DataHub + +- Click **Create new secret** again +- Enter a name like `REDSHIFT_PASSWORD` - we will use this later to refer to the secret +- Enter your `datahub` redshift user's password +- Optionally add a description +- Click **Create** + +

+ Redshift Password Secret +

+ +## Configure Recipe + +4. Navigate to the **Sources** tab and click **Create new source** + +

+ Click "Create new source" +

+ +5. Select Redshift + +

+ Select BigQuery from the options +

+ +6. Fill out the Redshift Recipe + +Populate the Password field by selecting Redshift Password secrets you created in steps 3 and 4. + +

+ Fill out the Redshift Recipe +

+ + + +## Schedule Execution + +Now it's time to schedule a recurring ingestion pipeline to regularly extract metadata from your Redshift instance. + +7. Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown + +

+ schedule selector +

+ +8. Ensure you've configured your correct timezone + +

+ timezone_selector +

+ +9. Click **Next** when you are done + +## Finish Up + +10. Name your ingestion source, then click **Save and Run** + +

+ Name your ingestion +

+ +You will now find your new ingestion source running + +

+ ingestion_running +

+ +## Validate Ingestion Runs + +11. View the latest status of ingestion runs on the Ingestion page + +

+ ingestion succeeded +

+ +12. Click the plus sign to expand the full list of historical runs and outcomes; click **Details** to see the outcomes of a specific run + +

+ ingestion_details +

+ +13. From the Ingestion Run Details page, pick **View All** to see which entities were ingested + +

+ ingestion_details_view_all +

+ +14. Pick an entity from the list to manually validate if it contains the detail you expected + +

+ ingestion_details_view_all +

+ +**Congratulations!** You've successfully set up Redshift as an ingestion source for DataHub! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/overview.md new file mode 100644 index 00000000..a66a967c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/overview.md @@ -0,0 +1,41 @@ +--- +title: Overview +sidebar_label: Overview +slug: /quick-ingestion-guides/redshift/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/redshift/overview.md +--- + +# Redshift Ingestion Guide: Overview + +## What You Will Get Out of This Guide + +This guide will help you set up the Redshift connector through the DataHub UI to begin ingesting metadata into DataHub. + +Upon completing this guide, you will have a recurring ingestion pipeline that will extract metadata from Redshift and load it into DataHub. This will include to following Redshift asset types: + +- Database +- Schemas (External and Internal) +- Tables (External and Internal) +- Views + +This recurring ingestion pipeline will also extract: + +- **Usage statistics** to help you understand recent query activity +- **Table-level lineage** (where available) to automatically define interdependencies between datasets +- **Table- and column-level profile statistics** to help you understand the shape of the data + +:::caution +The source currently can ingest one database with one recipe +::: + +## Next Steps + +If that all sounds like what you're looking for, navigate to the [next page](setup.md), where we'll talk about prerequisites + +## Advanced Guides and Reference + +If you're looking to do something more in-depth, want to use CLI instead of the DataHub UI, or just need to look at the reference documentation for this connector, use these links: + +- Learn about CLI Ingestion in the [Introduction to Metadata Ingestion](../../../metadata-ingestion/README.md) +- [Redshift Ingestion Reference Guide](/docs/generated/ingestion/sources/redshift/#module-redshift) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/setup.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/setup.md new file mode 100644 index 00000000..dbaae6a6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/redshift/setup.md @@ -0,0 +1,135 @@ +--- +title: Setup +sidebar_label: Setup +slug: /quick-ingestion-guides/redshift/setup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/redshift/setup.md +--- + +# Redshift Ingestion Guide: Setup & Prerequisites + +To configure ingestion from Redshift, you'll need a [User](https://docs.aws.amazon.com/redshift/latest/gsg/t_adding_redshift_user_cmd.html) configured with the proper permission sets. + +This setup guide will walk you through the steps you'll need to take in your Amazon Redshift cluster. + +## Redshift Prerequisites + +1. Connect to your Amazon Redshift cluster using an SQL client such as SQL Workbench/J or Amazon Redshift Query Editor with your Admin user. +2. Create a [Redshift User](https://docs.aws.amazon.com/redshift/latest/gsg/t_adding_redshift_user_cmd.html) that will be used to perform the metadata extraction if you don't have one already. + For example: + +```sql +CREATE USER datahub WITH PASSWORD 'Datahub1234'; +``` + +## Redshift Setup + +1. Grant the following permissions to your `datahub` user. For most users, the **recommended set** below will be sufficient: + +### Recommended Permissions + +For a typical provisioned cluster with default settings: + +```sql +-- Core system access (required for lineage and usage statistics) +ALTER USER datahub WITH SYSLOG ACCESS UNRESTRICTED; + +-- Core metadata extraction (always required) +GRANT SELECT ON pg_catalog.svv_redshift_databases TO datahub; +GRANT SELECT ON pg_catalog.svv_redshift_schemas TO datahub; +GRANT SELECT ON pg_catalog.svv_external_schemas TO datahub; +GRANT SELECT ON pg_catalog.svv_table_info TO datahub; +GRANT SELECT ON pg_catalog.svv_external_tables TO datahub; +GRANT SELECT ON pg_catalog.svv_external_columns TO datahub; +GRANT SELECT ON pg_catalog.pg_class_info TO datahub; + +-- Essential pg_catalog tables for table discovery +GRANT SELECT ON pg_catalog.pg_class TO datahub; +GRANT SELECT ON pg_catalog.pg_namespace TO datahub; +GRANT SELECT ON pg_catalog.pg_description TO datahub; +GRANT SELECT ON pg_catalog.pg_database TO datahub; +GRANT SELECT ON pg_catalog.pg_attribute TO datahub; +GRANT SELECT ON pg_catalog.pg_attrdef TO datahub; + +-- Datashare lineage (enabled by default) +GRANT SELECT ON pg_catalog.svv_datashares TO datahub; + +-- Provisioned cluster materialized views +GRANT SELECT ON pg_catalog.stv_mv_info TO datahub; +``` + +### Additional Permissions Based on Your Configuration + +**For Serverless Workgroups:** + +```sql +-- Use these instead of stv_mv_info (from Provisioned section above) +GRANT SELECT ON pg_catalog.svv_user_info TO datahub; +GRANT SELECT ON pg_catalog.svv_mv_info TO datahub; +``` + +**For Shared Databases (Datashare Consumers):** + +```sql +-- Required when is_shared_database = True +GRANT SELECT ON pg_catalog.svv_redshift_tables TO datahub; +GRANT SELECT ON pg_catalog.svv_redshift_columns TO datahub; +``` + +### Data Access Permissions (Required for Profiling/Classification) + +**Important**: The above permissions only provide access to metadata. For data profiling, classification, or any feature that reads actual table data, you need: + +```sql +-- Schema access (required to access tables within schemas) +GRANT USAGE ON SCHEMA public TO datahub; +GRANT USAGE ON SCHEMA your_schema_name TO datahub; + +-- Table data access (required for profiling and classification) +GRANT SELECT ON ALL TABLES IN SCHEMA public TO datahub; +GRANT SELECT ON ALL TABLES IN SCHEMA your_schema_name TO datahub; + +-- For production environments (future tables/views): +-- IMPORTANT: Default privileges only apply to objects created by the user who runs this command +-- Option 1: If you (as admin) will create all future tables/views: +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO datahub; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON VIEWS TO datahub; +ALTER DEFAULT PRIVILEGES IN SCHEMA your_schema_name GRANT SELECT ON TABLES TO datahub; +ALTER DEFAULT PRIVILEGES IN SCHEMA your_schema_name GRANT SELECT ON VIEWS TO datahub; + +-- Option 2: If other users will create tables/views, run this for each user: +-- ALTER DEFAULT PRIVILEGES FOR ROLE other_user_name IN SCHEMA public GRANT SELECT ON TABLES TO datahub; +-- ALTER DEFAULT PRIVILEGES FOR ROLE other_user_name IN SCHEMA public GRANT SELECT ON VIEWS TO datahub; + +-- Option 3: For all future users (requires superuser): +-- ALTER DEFAULT PRIVILEGES FOR ALL ROLES IN SCHEMA public GRANT SELECT ON TABLES TO datahub; +-- ALTER DEFAULT PRIVILEGES FOR ALL ROLES IN SCHEMA public GRANT SELECT ON VIEWS TO datahub; +``` + +:::caution Data Access vs Metadata Access + +**The permissions are split into two categories:** + +1. **System table permissions** (above) - Required for metadata extraction, lineage, and usage statistics +2. **Data access permissions** (this section) - Required for data profiling, classification, and any feature that reads actual table content + +**Default privileges only apply to objects created by the user who ran the ALTER DEFAULT PRIVILEGES command.** If multiple users create tables in your schemas, you need to: + +1. **Run the commands as each user**, OR +2. **Use `FOR ROLE other_user_name`** for each user who creates objects, OR +3. **Use `FOR ALL ROLES`** (requires superuser privileges) + +**Common gotcha**: If User A runs `ALTER DEFAULT PRIVILEGES` and User B creates a table, DataHub won't have access to User B's table unless you used Option 2 or 3 above. + +**Alternative approach**: Instead of default privileges, consider using a scheduled job to periodically grant access to new tables: + +```sql +-- Run this periodically to catch new tables +GRANT SELECT ON ALL TABLES IN SCHEMA your_schema_name TO datahub; +``` + +::: + +## Next Steps + +Once you've confirmed all of the above in Redshift, it's time to [move on](configuration.md) to configure the actual ingestion source within the DataHub UI. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/configuration.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/configuration.md new file mode 100644 index 00000000..6883c8ca --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/configuration.md @@ -0,0 +1,149 @@ +--- +title: Configuration +sidebar_label: Configuration +slug: /quick-ingestion-guides/snowflake/configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/snowflake/configuration.md +--- + +# Configuring Your Snowflake Connector to DataHub + +Now that you have created a DataHub-specific user with the relevant roles in Snowflake in [the prior step](setup.md), it's now time to set up a connection via the DataHub UI. + +## Configure Secrets + +1. Within DataHub, navigate to the **Ingestion** tab in the top, right corner of your screen + +

+ Navigate to the "Ingestion Tab" +

+ +:::note +If you do not see the Ingestion tab, please contact your DataHub admin to grant you the correct permissions +::: + +2. Navigate to the **Secrets** tab and click **Create new secret** + +

+ Secrets Tab +

+ +3. Create a Password secret + +This will securely store your Snowflake password within DataHub + +- Enter a name like `SNOWFLAKE_PASSWORD` - we will use this later to refer to the secret +- Enter the password configured for the DataHub user in the previous step +- Optionally add a description +- Click **Create** + +

+ Snowflake Password Secret +

+ +## Configure Recipe + +4. Navigate to the **Sources** tab and click **Create new source** + +

+ Click "Create new source" +

+ +5. Select Snowflake + +

+ Select Snowflake from the options +

+ +6. Fill out the Snowflake Recipe + +Enter the Snowflake Account Identifier as **Account ID** field. Account identifier is the part before `.snowflakecomputing.com` in your snowflake host URL: + +

+ Account Id Field +

+ +_Learn more about Snowflake Account Identifiers [here](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html#account-identifiers)_ + +Add the previously added Password secret to **Password** field: + +- Click on the Password input field +- Select `SNOWFLAKE_PASSWORD` secret + +

+ Password field +

+ +Populate the relevant fields using the same **Username**, **Role**, and **Warehouse** you created and/or specified in [Snowflake Prerequisites](setup.md). + +

+ Warehouse Field +

+ +7. Click **Test Connection** + +This step will ensure you have configured your credentials accurately and confirm you have the required permissions to extract all relevant metadata. + +

+ Test Snoflake connection +

+ +After you have successfully tested your connection, click **Next**. + +## Schedule Execution + +Now it's time to schedule a recurring ingestion pipeline to regularly extract metadata from your Snowflake instance. + +8. Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown + +

+ schedule selector +

+ +9. Ensure you've configured your correct timezone +

+ timezone_selector +

+ +10. Click **Next** when you are done + +## Finish Up + +11. Name your ingestion source, then click **Save and Run** +

+ Name your ingestion +

+ +You will now find your new ingestion source running + +

+ ingestion_running +

+ +## Validate Ingestion Runs + +12. View the latest status of ingestion runs on the Ingestion page + +

+ ingestion succeeded +

+ +13. Click the plus sign to expand the full list of historical runs and outcomes; click **Details** to see the outcomes of a specific run + +

+ ingestion_details +

+ +14. From the Ingestion Run Details page, pick **View All** to see which entities were ingested + +

+ ingestion_details_view_all +

+ +15. Pick an entity from the list to manually validate if it contains the detail you expected + +

+ ingestion_details_view_all +

+ +**Congratulations!** You've successfully set up Snowflake as an ingestion source for DataHub! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/overview.md new file mode 100644 index 00000000..d78b98da --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/overview.md @@ -0,0 +1,51 @@ +--- +title: Overview +sidebar_label: Overview +slug: /quick-ingestion-guides/snowflake/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/snowflake/overview.md +--- + +# Snowflake Ingestion Guide: Overview + +## What You Will Get Out of This Guide + +This guide will help you set up the Snowflake connector to begin ingesting metadata into DataHub. + +Upon completing this guide, you will have a recurring ingestion pipeline that will extract metadata from Snowflake and load it into DataHub. This will include to following Snowflake asset types: + +- Databases +- Schemas +- Tables +- External Tables +- Views +- Materialized Views + +The pipeline will also extract: + +- **Usage statistics** to help you understand recent query activity (available if using Snowflake Enterprise edition or above) +- **Table- and Column-level lineage** to automatically define interdependencies between datasets and columns (available if using Snowflake Enterprise edition or above) +- **Table-level profile statistics** to help you understand the shape of the data + +:::caution +You will NOT have extracted Stages, Snowpipes, or Tasks from Snowflake, as the connector does not support ingesting these assets yet. +::: + +### Caveats + +By default, DataHub only profiles datasets that have changed in the past 1 day. This can be changed in the YAML editor by setting the value of `profile_if_updated_since_days` to something greater than 1. + +Additionally, DataHub only extracts usage and lineage information based on operations performed in the last 1 day. This can be changed by setting a custom value for `start_time` and `end_time` in the YAML editor. + +_To learn more about setting these advanced values, check out the [Snowflake Ingestion Source](/docs/generated/ingestion/sources/snowflake/#module-snowflake)._ + +## Next Steps + +If that all sounds like what you're looking for, navigate to the [next page](setup.md), where we'll talk about prerequisites. + +## Advanced Guides and Reference + +If you want to ingest metadata from Snowflake using the DataHub CLI, check out the following resources: + +- Learn about CLI Ingestion in the [Introduction to Metadata Ingestion](../../../metadata-ingestion/README.md) +- [Snowflake Ingestion Source](/docs/generated/ingestion/sources/snowflake/#module-snowflake) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/setup.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/setup.md new file mode 100644 index 00000000..9c69842c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/snowflake/setup.md @@ -0,0 +1,78 @@ +--- +title: Setup +sidebar_label: Setup +slug: /quick-ingestion-guides/snowflake/setup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/snowflake/setup.md +--- + +# Snowflake Ingestion Guide: Setup & Prerequisites + +In order to configure ingestion from Snowflake, you'll first have to ensure you have a Snowflake user with the `ACCOUNTADMIN` role or `MANAGE GRANTS` privilege. + +## Snowflake Prerequisites + +1. Create a DataHub-specific role by executing the following queries in Snowflake. Replace `` with an existing warehouse that you wish to use for DataHub ingestion. + + ```sql + create or replace role datahub_role; + -- Grant access to a warehouse to run queries to view metadata + grant operate, usage on warehouse "" to role datahub_role; + ``` + + Make note of this role and warehouse. You'll need this in the next step. + +2. Create a DataHub-specific user by executing the following queries. Replace `` with a strong password. Replace `` with the same warehouse used above. + + ```sql + create user datahub_user display_name = 'DataHub' password='' default_role = datahub_role type='LEGACY_SERVICE' default_warehouse = ''; + -- Grant access to the DataHub role created above + grant role datahub_role to user datahub_user; + ``` + + Make note of the user and its password. You'll need this in the next step. + +3. Assign privileges to read metadata about your assets by executing the following queries. Replace `` with an existing database. Repeat for all databases from your Snowflake instance that you wish to integrate with DataHub. + + ```sql + set db_var = '""'; + -- Grant access to view database and schema in which your tables/views exist + grant usage on DATABASE identifier($db_var) to role datahub_role; + grant usage on all schemas in database identifier($db_var) to role datahub_role; + grant usage on future schemas in database identifier($db_var) to role datahub_role; + + -- Grant Select acccess enable Data Profiling + grant select on all tables in database identifier($db_var) to role datahub_role; + grant select on future tables in database identifier($db_var) to role datahub_role; + grant select on all external tables in database identifier($db_var) to role datahub_role; + grant select on future external tables in database identifier($db_var) to role datahub_role; + grant select on all views in database identifier($db_var) to role datahub_role; + grant select on future views in database identifier($db_var) to role datahub_role; + grant select on all dynamic tables in database identifier($db_var) to role datahub_role; + grant select on future dynamic tables in database identifier($db_var) to role datahub_role; + + -- Grant access to view tables and views + grant references on all tables in database identifier($db_var) to role datahub_role; + grant references on future tables in database identifier($db_var) to role datahub_role; + grant references on all external tables in database identifier($db_var) to role datahub_role; + grant references on future external tables in database identifier($db_var) to role datahub_role; + grant references on all views in database identifier($db_var) to role datahub_role; + grant references on future views in database identifier($db_var) to role datahub_role; + -- Grant access to dynamic tables + grant monitor on all dynamic tables in database identifier($db_var) to role datahub_role; + grant monitor on future dynamic tables in database identifier($db_var) to role datahub_role; + + -- Assign privileges to extract lineage and usage statistics from Snowflake by executing the below query. + grant imported privileges on database snowflake to role datahub_role; + + ``` + + If you have imported databases in your Snowflake instance that you wish to integrate with DataHub, you'll need to use the below query for them. + + ```sql + grant IMPORTED PRIVILEGES on database "" to role datahub_role; + ``` + +## Next Steps + +Once you've done all of the above in Snowflake, it's time to [move on](configuration.md) to configuring the actual ingestion source within DataHub. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/configuration.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/configuration.md new file mode 100644 index 00000000..589532d2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/configuration.md @@ -0,0 +1,155 @@ +--- +title: Configuration +sidebar_label: Configuration +slug: /quick-ingestion-guides/tableau/configuration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/tableau/configuration.md +--- + +# Configuring Your Tableau Connector to DataHub + +Now that you have created a DataHub-specific user with the relevant access in Tableau in [the prior step](setup.md), it's now time to set up a connection via the DataHub UI. + +## Configure Secrets + +1. Within DataHub, navigate to the **Ingestion** tab in the top, right corner of your screen + +

+ Navigate to the "Ingestion Tab" +

+ +:::note +If you do not see the Ingestion tab, please contact your DataHub admin to grant you the correct permissions +::: + +2. Navigate to the **Secrets** tab and click **Create new secret** + +

+ Secrets Tab +

+ +3. Create a `username` secret + +This will securely store your Tableau `username` within DataHub + +- Enter a name like `TABLEAU_USERNAME` - we will use this later to refer in recipe +- Enter the `username`, setup in the [setup guide](setup.md) +- Optionally add a description +- Click **Create** + +

+ Tableau Username Secret +

+ +4. Create a `password` secret + +This will securely store your Tableau `password` within DataHub + +- Enter a name like `TABLEAU_PASSWORD` - we will use this later to refer in recipe +- Enter the `password` of the user, setup in the [setup guide](setup.md) +- Optionally add a description +- Click **Create** + +

+ Tableau Password Secret +

+ +## Configure Recipe + +5. Navigate to on the **Sources** tab and then **Create new source** + +

+ Click "Create new source" +

+ +6. Select Tableau + +

+ Select Tableau from the options +

+ +7. Fill in the Tableau Recipe form: + + You need to set minimum following fields in the recipe: + + a. **Host URL:** URL of your Tableau instance (e.g., https://15az.online.tableau.com/). It is available in browser address bar on Tableau Portal. + + b. **Username:** Use the TABLEAU_USERNAME secret (e.g., "${TABLEAU_USERNAME}"). + + c. **Password:** Use the TABLEAU_PASSWORD secret (e.g., "${TABLEAU_PASSWORD}"). + + d. **Site**: Required only if using tableau cloud/ tableau online + +To filter specific project, use `project_pattern` fields. + + config: + ... + project_pattern: + allow: + - "SalesProject" + +Your recipe should look something like this: + +

+ tableau recipe in form format +

+ +Click **Next** when you're done. + +## Schedule Execution + +Now it's time to schedule a recurring ingestion pipeline to regularly extract metadata from your Tableau instance. + +8. Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown + +

+ schedule selector +

+ +9. Ensure you've configured your correct timezone +

+ timezone_selector +

+ +10. Click **Next** when you are done + +## Finish Up + +11. Name your ingestion source, then click **Save and Run** +

+ Name your ingestion +

+ +You will now find your new ingestion source running + +

+ ingestion_running +

+ +## Validate Ingestion Runs + +12. View the latest status of ingestion runs on the Ingestion page + +

+ ingestion succeeded +

+ +13. Click the plus sign to expand the full list of historical runs and outcomes; click **Details** to see the outcomes of a specific run + +

+ ingestion_details +

+ +14. From the Ingestion Run Details page, pick **View All** to see which entities were ingested + +

+ ingestion_details_view_all +

+ +15. Pick an entity from the list to manually validate if it contains the detail you expected + +

+ ingestion_details_view_all +

+ +**Congratulations!** You've successfully set up Tableau as an ingestion source for DataHub! diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/overview.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/overview.md new file mode 100644 index 00000000..f546bb1a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/overview.md @@ -0,0 +1,41 @@ +--- +title: Overview +sidebar_label: Overview +slug: /quick-ingestion-guides/tableau/overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/tableau/overview.md +--- + +# Tableau Ingestion Guide: Overview + +## What You Will Get Out of This Guide + +This guide will help you set up the Tableau connector to begin ingesting metadata into DataHub. + +Upon completing this guide, you will have a recurring ingestion pipeline that will extract metadata from Tableau and load it into DataHub. This will include to following Tableau asset types: + +- Dashboards +- Sheets +- Embedded DataSource +- Published DataSource +- Custom SQL Table +- Embedded or External Tables +- User +- Workbook +- Tag + +The pipeline will also extract: + +- **Usage statistics** help you understand top viewed Dashboard/Chart +- **Table- and Column-level lineage** automatically index relationships between datasets and columns + +## Next Steps + +Continue to the [setup guide](setup.md), where we'll describe the prerequisites. + +## Advanced Guides and Reference + +If you want to ingest metadata from Tableau using the DataHub CLI, check out the following resources: + +- Learn about CLI Ingestion in the [Introduction to Metadata Ingestion](../../../metadata-ingestion/README.md) +- [Tableau Ingestion Source](/docs/generated/ingestion/sources/tableau) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/setup.md b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/setup.md new file mode 100644 index 00000000..8b3c11e3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quick-ingestion-guides/tableau/setup.md @@ -0,0 +1,68 @@ +--- +title: Setup +sidebar_label: Setup +slug: /quick-ingestion-guides/tableau/setup +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/quick-ingestion-guides/tableau/setup.md +--- + +# Tableau Ingestion Guide: Setup & Prerequisites + +In order to configure ingestion from Tableau, you'll first have to enable Tableau Metadata API and you should have a user with Site Administrator Explorer permissions. + +## Tableau Prerequisites + +1. Grant `Site Administrator Explorer permissions` to a user + + A. Log in to Tableau Cloud https://sso.online.tableau.com/public/idp/SSO. + + B. Navigate to `Users`. + +

+ Navigate to the Users tab +

+ + C. **For New User**: Follow below steps to grant permission for new user. + + - Click `Add Users` -> `Add Users by Email` + +

+ Navigate to the Users tab +

+ + - Fill `Enter email addresses`, set `Site role` to `Site Administrator Explorer` and Click `Add Users` + +

+ Navigate to the Users tab +

+ + D. **For Existing User:** Follow below steps to grant permission for existing user. + + - Select a user and click `Actions` -> `Site Role` + +

+ Actions Site Role +

+ + - Change user role to `Site Administrator Explorer` + +

+ tableau site role +

+ +2. **Enable Tableau Metadata API:** This step is required only for Tableau Server. The Metadata API is installed with Tableau Server but disabled by default. + + - Open a command prompt as an admin on the initial node (_where TSM is installed_) in the cluster + - Run the command: `tsm maintenance metadata-services enable` + +3. **Enable Derived Permissions:** This step is required only when the site is using external assets. For more detail, refer to the tableau documentation [Manage Permissions for External Assets](https://help.tableau.com/current/online/en-us/dm_perms_assets.htm). + + Follow the below steps to enable the derived permissions: + + - Sign in to Tableau Cloud or Tableau Server as an admin. + - From the left navigation pane, click Settings. + - On the General tab, under Automatic Access to Metadata about Databases and Tables, select the `Automatically grant authorized users access to metadata about databases and tables` check box. + +## Next Steps + +Once you've done all of the above in Tableau, it's time to [move on](configuration.md) to configuring the actual ingestion source within DataHub. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/quickstart.md b/docs-archive/versioned_docs/version-1.5.0/docs/quickstart.md new file mode 100644 index 00000000..6b6a7737 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/quickstart.md @@ -0,0 +1,330 @@ +--- +title: DataHub Quickstart Guide +sidebar_label: Quickstart Guide +slug: /quickstart +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/quickstart.md' +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# DataHub Quickstart Guide + +:::tip DataHub Cloud + +This guide provides instructions on deploying the open source DataHub locally. +If you're interested in a managed version, [DataHub](https://www.datahub.com) provides a fully managed, premium version of DataHub.
+**[Get Started with DataHub Cloud](./managed-datahub/welcome-acryl.md)** + +::: + +## Prerequisites + +- Install **Docker** and **Docker Compose** v2 for your platform. + + | Platform | Application | + | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | + | Windows | [Docker Desktop](https://www.docker.com/products/docker-desktop/) | + | Mac | [Docker Desktop](https://www.docker.com/products/docker-desktop/) | + | Linux | [Docker for Linux](https://docs.docker.com/desktop/install/linux-install/) and [Docker Compose](https://docs.docker.com/compose/install/linux/) | + +- **Launch the Docker engine** from command line or the desktop app. +- Ensure you have **Python 3.10+** installed & configured. (Check using `python3 --version`). + +:::note Docker Resource Allocation + +Make sure to allocate enough hardware resources for Docker engine.
+Tested & confirmed config: 2 CPUs, 8GB RAM, 2GB Swap area, and 12GB disk space. + +::: + +## Install the DataHub CLI + + + + +```bash +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub +datahub version +``` + +:::note Command Not Found + +If you see `command not found`, try running cli commands like `python3 -m datahub version`.
+Note that DataHub CLI does not support Python 2.x. + +::: + + + + +```bash +poetry add acryl-datahub +poetry shell +datahub version +``` + + + + +## Start DataHub + +Run the following CLI command from your terminal. + +```bash +datahub docker quickstart +``` + +This will deploy a DataHub instance using [docker-compose](https://docs.docker.com/compose/). +If you are curious, the `docker-compose.yaml` file is downloaded to your home directory under the `.datahub/quickstart` directory. + +Starting CLI version 1.5 has changed how the signing key for generationg tokens via [Metadata Service Authentication](./authentication/introducing-metadata-service-authentication.md) is configured. + +Quickstart resolves the values in the following order: + +1. If the envrionment variables `DATAHUB_TOKEN_SERVICE_SIGNING_KEY` and `DATAHUB_TOKEN_SERVICE_SALT` defined, their values are used. +2. If the file `~/.datahub/quickstart/.local-secrets.env` exists and the variables mentioned above are defined in it thier values are used. +3. If both of the above are not available, new random values are generated and used. The values are written to the file mentioned above and used in subsequent invocations. + +It is recommended that users provide their own stable values for the environment values before running quickstart. + +``` +export DATAHUB_TOKEN_SERVICE_SIGNING_KEY= +export DATAHUB_TOKEN_SERVICE_SALT= +``` + +To generate values to use, you can use the output of the following + +``` +openssl rand -base64 32 +``` + +If things go well, you should see messages like the ones below: + +```shell-session +Fetching docker-compose file https://raw.githubusercontent.com/datahub-project/datahub/master/docker/quickstart/docker-compose.quickstart-profile.yml from GitHub +Pulling docker images... +Finished pulling docker images! + +Starting up DataHub... +[+] Running 14/14 + ✔ Network datahub_network Created 0.0s + ✔ Volume "datahub_broker" Created 0.0s + ✔ Volume "datahub_mysqldata" Created 0.0s + ✔ Volume "datahub_osdata" Created 0.0s + ✔ Container datahub-mysql-1 Healthy 11.6s + ✔ Container datahub-opensearch-1 Healthy 11.6s + ✔ Container datahub-kafka-broker-1 Healthy 6.0s + ✔ Container datahub-system-update-quickstart-1 Exited 26.6s + ✔ Container datahub-datahub-gms-quickstart-1 Healthy 42.1s + ✔ Container datahub-frontend-quickstart-1 Started 26.6s + ✔ Container datahub-datahub-actions-quickstart-1 Started 42.1s + +✔ DataHub is now running +Ingest some demo data using `datahub docker ingest-sample-data`, +or head to http://localhost:9002 (username: datahub, password: datahub) to play around with the frontend. +Need support? Get in touch on Slack: https://datahub.com/slack/ +``` + +:::note Breaking changes + +### Docker Compose File version change + +From version 1.2 onwards, the `datahub docker quickstart` command uses a version of docker-compose file that is incompatible with datahub that was installed using earlier versions of the CLI. + +If you have datahub already installed using an earlier version of CLI, additional steps are needed to upgrade. + +Required steps to upgrade: + +1. Backup your data (recommended): datahub docker quickstart --backup + Guide: https://docs.datahub.com/docs/quickstart#back-up-datahub + This step can be skipped if data does not need to be preserved. + +2. Remove old installation: datahub docker nuke + +3. Start fresh installation: datahub docker quickstart + +⚠️ Without backup, all existing data will be lost. + +### DataHub Authentication Changes in default signing key + +From version 1.5 DataHub quickstart now generates a random signing key and salt for use when generating and validating authentication tokens instead of a hardcoded default key used previously if the user does not provide thier own keys. + +⚠️ For users upgrading from previous versions of the cli, due to the change in the signing key, existing PAT tokens will be invalidated. +::: + +### Sign In + +Upon completion of this step, you should be able to navigate to the DataHub UI at [http://localhost:9002](http://localhost:9002) in your browser. +You can sign in using the default credentials below. + +```json +username: datahub +password: datahub +``` + +To change the default credentials, please refer to [Change the default user datahub in quickstart](authentication/changing-default-credentials.md#quickstart). + +### Ingest Sample Data + +To ingest the sample metadata, run the following CLI command from your terminal + +```bash +datahub docker ingest-sample-data +``` + +:::note Token Authentication + +If you've enabled [Metadata Service Authentication](authentication/introducing-metadata-service-authentication.md), you'll need to provide a Personal Access Token +using the `--token ` parameter in the command. + +::: + +That's it! Now feel free to play around with DataHub! + +--- + +## Common Operations + +### Stop DataHub + +To stop DataHub's quickstart, you can issue the following command. + +```bash +datahub docker quickstart --stop +``` + +### Reset DataHub + +To cleanse DataHub of all of its state (e.g. before ingesting your own), you can use the CLI `nuke` command. + +```bash +datahub docker nuke +``` + +### Upgrade DataHub + +If you have been testing DataHub locally, a new version of DataHub got released and you want to try the new version then you can just issue the quickstart command again. It will pull down newer images and restart your instance without losing any data. + +```bash +datahub docker quickstart +``` + +By default, quickstart will install the latest released version of datahub. You can pick a specific version by specifying a release explicitly. + +```bash +datahub docker quickstart --version v1.2.0 +``` + +You can see the releases available on the [github releases](https://github.com/datahub-project/datahub/releases) page +You can also specify `head` as the version to get the latest development version on the master branch. + +### Customize installation + +If you would like to customize the DataHub installation further, please download the [docker-compose.yaml](https://raw.githubusercontent.com/datahub-project/datahub/master/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml) used by the cli tool, modify it as necessary and deploy DataHub by passing the downloaded docker-compose file: + +```bash +datahub docker quickstart --quickstart-compose-file +``` + +### Back up DataHub + +The quickstart image is not recommended for use as a production instance.
+However, in case you want to take a backup of your current quickstart state (e.g. you have a demo to your company coming up and you want to create a copy of the quickstart data so you can restore it at a future date), you can supply the `--backup` flag to quickstart. + +```bash +datahub docker quickstart --backup +``` + +This will take a backup of your MySQL image and write it by default to your `~/.datahub/quickstart/` directory as the file `backup.sql`. + +You can customize the backup file path by passing a `--backup-file` argument: + +```bash +datahub docker quickstart --backup --backup-file +``` + +:::caution + +Note that the Quickstart backup does not include any timeseries data (dataset statistics, profiles, etc.), so you will lose that information if you delete all your indexes and restore from this backup. + +::: + +### Restore DataHub + +As you might imagine, these backups are restore-able. The following section describes a few different options you have to restore your backup. + +#### General Restoring + +To restore a previous backup, run the following command: + +```bash +datahub docker quickstart --restore +``` + +This command will pick up the `backup.sql` file located under `~/.datahub/quickstart` and restore your primary database as well as the elasticsearch indexes with it. + +To supply a specific backup file, use the `--restore-file` option. + +```bash +datahub docker quickstart --restore --restore-file /home/my_user/datahub_backups/quickstart_backup_2002_22_01.sql +``` + +#### Restore Only Index + +Another situation that can come up is the index can get corrupt, or be missing some update. In order to re-bootstrap the index from the primary store, you can run this command to sync the index with the primary store. + +```bash +datahub docker quickstart --restore-indices +``` + +#### Restore Only Primary + +Sometimes, you might want to just restore the state of your primary database (MySQL), but not re-index the data. To do this, you have to explicitly disable the restore-indices capability. + +```bash +datahub docker quickstart --restore --no-restore-indices +``` + +--- + +## Next Steps + +- [Quickstart Debugging Guide](./troubleshooting/quickstart.md) +- [Ingest metadata through the UI](./ui-ingestion.md) +- [Ingest metadata through the CLI](../metadata-ingestion/README.md) +- [Add Users to DataHub](authentication/guides/add-users.md) +- [Configure OIDC Authentication](authentication/guides/sso/configure-oidc-react.md) +- [Configure JaaS Authentication](authentication/guides/jaas.md) +- [Configure authentication in DataHub's backend](authentication/introducing-metadata-service-authentication.md#configuring-metadata-service-authentication). +- [Change the default user datahub in quickstart](authentication/changing-default-credentials.md#quickstart) + +### Move To Production + +:::caution + +Quickstart is not intended for a production environment. We recommend deploying DataHub to production using Kubernetes. +We provide helpful [Helm Charts](https://artifacthub.io/packages/helm/datahub/datahub) to help you quickly get up and running. +Check out [Deploying DataHub to Kubernetes](./deploy/kubernetes.md) for a step-by-step walkthrough. + +::: + +The `quickstart` method of running DataHub is intended for local development and a quick way to experience the features that DataHub has to offer. +It is not intended for a production environment. This recommendation is based on the following points. + +#### Default Credentials + +`quickstart` uses docker compose configuration which includes default credentials for both DataHub, and it's underlying +prerequisite data stores, such as MySQL. Additionally, other components are unauthenticated out of the box. This is a +design choice to make development easier and is not best practice for a production environment. + +#### Exposed Ports + +DataHub's services, and it's backend data stores use the docker default behavior of binding to all interface addresses. +This makes it useful for development but is not recommended in a production environment. + +#### Performance & Management + +`quickstart` is limited by the resources available on a single host, there is no ability to scale horizontally. +Rollout of new versions often requires downtime and the configuration is largely pre-determined and not easily managed. +Lastly, by default, `quickstart` follows the most recent builds forcing updates to the latest released and unreleased builds. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/rfc.md b/docs-archive/versioned_docs/version-1.5.0/docs/rfc.md new file mode 100644 index 00000000..8602ec0f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/rfc.md @@ -0,0 +1,126 @@ +--- +title: DataHub RFC Process +sidebar_label: RFC Process +slug: /rfc +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/rfc.md' +--- +# DataHub RFC Process + +## What is an RFC? + +The "RFC" (request for comments) process is intended to provide a consistent and controlled path for new features, +significant modifications, or any other significant proposal to enter DataHub and its related frameworks. + +Many changes, including bug fixes and documentation improvements can be implemented and reviewed via the normal GitHub +pull request workflow. + +Some changes though are "substantial", and we ask that these be put through a bit of a design process and produce a +consensus among the DataHub core teams. + +## The RFC life-cycle + +An RFC goes through the following stages: + +- _Discussion_ (Optional): Create an issue with the "RFC" label to have a more open ended, initial discussion around + your proposal (useful if you don't have a concrete proposal yet). Consider posting to #contribute-code in [Slack](./slack.md) + for more visibility. +- _Pending_: when the RFC is submitted as a PR. Please add the "RFC" label to the PR. +- _Active_: when an RFC PR is merged and undergoing implementation. +- _Landed_: when an RFC's proposed changes are shipped in an actual release. +- _Rejected_: when an RFC PR is closed without being merged. + +[Pending RFC List](https://github.com/datahub-project/datahub/pulls?q=is%3Apr+is%3Aopen+label%3ARFC) + +## When to follow this process + +You need to follow this process if you intend to make "substantial" changes to any components in the DataHub git repo, +their documentation, or any other projects under the purview of the DataHub core teams. What constitutes a "substantial" +change is evolving based on community norms, but may include the following: + +- A new feature that creates new API surface area, and would require a feature flag if introduced. +- The removal of features that already shipped as part of the release channel. +- The introduction of new idiomatic usage or conventions, even if they do not include code changes to DataHub itself. + +Some changes do not require an RFC: + +- Rephrasing, reorganizing or refactoring +- Addition or removal of warnings +- Additions that strictly improve objective, numerical quality criteria (speedup) + +If you submit a pull request to implement a new, major feature without going through the RFC process, it may be closed +with a polite request to submit an RFC first. + +## Gathering feedback before submitting + +It's often helpful to get feedback on your concept before diving into the level of API design detail required for an +RFC. You may open an issue on this repo to start a high-level discussion, with the goal of eventually formulating an RFC +pull request with the specific implementation design. We also highly recommend sharing drafts of RFCs in #contribute-code on the +[DataHub Slack](./slack.md) for early feedback. + +## The process + +In short, to get a major feature added to DataHub, one must first get the RFC merged into the main DataHub repository as a markdown +file. At that point the RFC is 'active' and may be implemented with the goal of eventual inclusion into DataHub. + +- Fork the [datahub-project/datahub repository](https://github.com/datahub-project/datahub). +- Copy the `template.md` file from `docs/rfcs/` to `docs/rfcs/active/000-my-feature.md`, where `my-feature` is more + descriptive. Don't assign an RFC number yet. +- Fill in the RFC. Put care into the details. _RFCs that do not present convincing motivation, demonstrate understanding + of the impact of the design, or are disingenuous about the drawback or alternatives tend to be poorly-received._ +- Submit a pull request with the "RFC" label. As a pull request the RFC will receive design feedback from the larger community, and the + author should be prepared to revise it in response. +- Update the pull request to add the number of the PR to the filename and add a link to the PR in the header of the RFC. +- Build consensus and integrate feedback. RFCs that have broad support are much more likely to make progress than those + that don't receive any comments. +- Eventually, the DataHub team will decide whether the RFC is a candidate for inclusion. +- RFCs that are candidates for inclusion will enter a "final comment period" lasting 7 days. The beginning of this + period will be signaled with a comment and tag on the pull request. Furthermore, an announcement will be made in the + \#contribute-code Slack channel for further visibility. +- An RFC can be modified based upon feedback from the DataHub team and community. Significant modifications may trigger + a new final comment period. +- An RFC may be rejected by the DataHub team after public discussion has settled and comments have been made summarizing + the rationale for rejection. The RFC will enter a "final comment period to close" lasting 7 days. At the end of the "FCP + to close" period, the PR will be closed. +- An RFC author may withdraw their own RFC by closing it themselves. Please state the reason for the withdrawal. +- An RFC may be accepted at the close of its final comment period. A DataHub team member will merge the RFC's associated + pull request, at which point the RFC will become 'active'. + +## Details on Active RFCs + +Once an RFC becomes active then authors may implement it and submit the feature as a pull request to the DataHub repo. +Becoming 'active' is not a rubber stamp, and in particular still does not mean the feature will ultimately be merged; it +does mean that the core team has agreed to it in principle and are amenable to merging it. + +Furthermore, the fact that a given RFC has been accepted and is 'active' implies nothing about what priority is assigned +to its implementation, nor whether anybody is currently working on it. + +Modifications to active RFC's can be done in followup PR's. We strive to write each RFC in a manner that it will reflect +the final design of the feature; but the nature of the process means that we cannot expect every merged RFC to actually +reflect what the end result will be at the time of the next major release; therefore we try to keep each RFC document +somewhat in sync with the language feature as planned, tracking such changes via followup pull requests to the document. + +## Implementing an RFC + +The author of an RFC is not obligated to implement it. Of course, the RFC author (like any other developer) is welcome +to post an implementation for review after the RFC has been accepted. + +An active RFC should have the link to the implementation PR(s) listed, if there are any. Feedback to the actual +implementation should be conducted in the implementation PR instead of the original RFC PR. + +If you are interested in working on the implementation for an 'active' RFC, but cannot determine if someone else is +already working on it, feel free to ask (e.g. by leaving a comment on the associated issue). + +## Implemented RFCs + +Once an RFC has finally been implemented, first off, congratulations! And thank you for your contribution! Second, to +help track the status of the RFC, please make one final PR to move the RFC from `docs/rfcs/active` to +`docs/rfcs/accepted`. + +## Reviewing RFCs + +Most of the DataHub team will attempt to review some set of open RFC pull requests on a regular basis. If a DataHub +team member believes an RFC PR is ready to be accepted into active status, they can approve the PR using GitHub's +review feature to signal their approval of the RFCs. + +_DataHub's RFC process is inspired by many others, including [Vue.js](https://github.com/vuejs/rfcs) and +[Ember](https://github.com/emberjs/rfcs)._ diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/rfcs/README.md b/docs-archive/versioned_docs/version-1.5.0/docs/rfcs/README.md new file mode 100644 index 00000000..2230c6df --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/rfcs/README.md @@ -0,0 +1,48 @@ +--- +title: DataHub RFCs +sidebar_label: RFCs +slug: /rfcs +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/rfcs/README.md' +--- +# DataHub RFCs + +This directory contains Request for Comments (RFC) documents for substantial changes to DataHub. + +## What is an RFC? + +RFCs are design documents that propose significant changes to DataHub. See [the RFC process documentation](../rfc.md) for full details on when and how to submit an RFC. + +## Fresh Start (2025) + +**Note:** RFCs have moved back to the main DataHub repository (from the separate `datahub-project/rfcs` repo) to improve visibility and reduce contributor friction. + +This represents a fresh start for the RFC process. Historical RFCs from before 2025 remain accessible in: + +- Git history (commit `dbb4c84cb2` and earlier) +- Archived external repository: https://github.com/datahub-project/rfcs + +Historical RFCs will be migrated to this directory on an as-needed basis when they become relevant for discussion or implementation. + +## Directory Structure + +- **[active/](https://github.com/datahub-project/datahub/blob/master/docs/rfcs/active/)** - RFCs currently under discussion or implementation +- **[accepted/](https://github.com/datahub-project/datahub/blob/master/docs/rfcs/accepted/)** - RFCs that have been implemented and shipped +- **template.md** - Template for creating new RFCs (available in the repository) + +## Active RFCs + +Currently, there are no active RFCs. Check the [pull requests with the "RFC" label](https://github.com/datahub-project/datahub/pulls?q=is%3Apr+is%3Aopen+label%3ARFC) for pending RFC proposals. + +## Accepted RFCs + +No RFCs have been accepted in this new structure yet. + +## Contributing an RFC + +1. Read the [RFC process documentation](../rfc.md) +2. Copy `template.md` to `active/000-my-feature.md` +3. Fill in the template with your proposal +4. Submit a pull request with the "RFC" label +5. Gather feedback and iterate on the design + +For questions or early feedback, post in the #contribute-code channel on [DataHub Slack](../slack.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/roadmap.md b/docs-archive/versioned_docs/version-1.5.0/docs/roadmap.md new file mode 100644 index 00000000..549bb973 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/roadmap.md @@ -0,0 +1,177 @@ +--- +title: DataHub Roadmap +sidebar_label: Roadmap +slug: /roadmap +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/roadmap.md' +--- +# DataHub Roadmap + +## [The DataHub Roadmap has a new home!](https://feature-requests.datahubproject.io/roadmap) + +Please refer to the [new DataHub Roadmap](https://feature-requests.datahubproject.io/roadmap) for the most up-to-date details of what we are working on! + +_If you have suggestions about what we should consider in future cycles, feel free to submit a [feature request](https://feature-requests.datahubproject.io/) and/or upvote existing feature requests so we can get a sense of level of importance!_ + +## Historical Roadmap + +_This following represents the progress made on historical roadmap items as of January 2022. For incomplete roadmap items, we have created Feature Requests to gauge current community interest & impact to be considered in future cycles. If you see something that is still of high-interest to you, please up-vote via the Feature Request portal link and subscribe to the post for updates as we progress through the work in future cycles._ + +### Q4 2021 [Oct - Dec 2021] + +#### Data Lake Ecosystem Integration + +- [ ] Spark Delta Lake - [View in Feature Reqeust Portal](https://feature-requests.datahubproject.io/b/feedback/p/spark-delta-lake) +- [ ] Apache Iceberg - [Included in Q1 2022 Roadmap - Community-Driven Metadata Ingestion Sources](https://feature-requests.datahubproject.io/roadmap/540) +- [ ] Apache Hudi - [View in Feature Request Portal](https://feature-requests.datahubproject.io/b/feedback/p/apachi-hudi-ingestion-support) + +#### Metadata Trigger Framework + +[View in Feature Request Portal](https://feature-requests.datahubproject.io/b/User-Experience/p/ability-to-subscribe-to-an-entity-to-receive-notifications-when-something-changes) + +- [ ] Stateful sensors for Airflow +- [ ] Receive events for you to send alerts, email +- [ ] Slack integration + +#### ML Ecosystem + +- [x] Features (Feast) +- [x] Models (Sagemaker) +- [ ] Notebooks - View in Feature Request Portal](https://feature-requests.datahubproject.io/admin/p/jupyter-integration) + +#### Metrics Ecosystem + +[View in Feature Request Portal](https://feature-requests.datahubproject.io/b/User-Experience/p/ability-to-define-metrics-and-attach-them-to-entities) + +- [ ] Measures, Dimensions +- [ ] Relationships to Datasets and Dashboards + +#### Data Mesh oriented features + +- [ ] Data Product modeling +- [ ] Analytics to enable Data Meshification + +#### Collaboration + +[View in Feature Reqeust Portal](https://feature-requests.datahubproject.io/b/User-Experience/p/collaboration-within-datahub-ui) + +- [ ] Conversations on the platform +- [ ] Knowledge Posts (Gdocs, Gslides, Gsheets) + +### Q3 2021 [Jul - Sept 2021] + +#### Data Profiling and Dataset Previews + +Use Case: See sample data for a dataset and statistics on the shape of the data (column distribution, nullability etc.) + +- [x] Support for data profiling and preview extraction through ingestion pipeline (column samples, not rows) + +#### Data Quality + +Included in Q1 2022 Roadmap - [Display Data Quality Checks in the UI](https://feature-requests.datahubproject.io/roadmap/544) + +- [x] Support for data profiling and time-series views +- [ ] Support for data quality visualization +- [ ] Support for data health score based on data quality results and pipeline observability +- [ ] Integration with systems like Great Expectations, AWS deequ, dbt test etc. + +#### Fine-grained Access Control for Metadata + +- [x] Support for role-based access control to edit metadata +- Scope: Access control on entity-level, aspect-level and within aspects as well. + +#### Column-level lineage + +Included in Q1 2022 Roadmap - [Column Level Lineage](https://feature-requests.datahubproject.io/roadmap/541) + +- [ ] Metadata Model +- [ ] SQL Parsing + +#### Operational Metadata + +- [ ] Partitioned Datasets - - [View in Feature Request Portal](https://feature-requests.datahubproject.io/b/User-Experience/p/advanced-dataset-schema-properties-partition-support) +- [x] Support for operational signals like completeness, freshness etc. + +### Q2 2021 (Apr - Jun 2021) + +#### Cloud Deployment + +- [x] Production-grade Helm charts for Kubernetes-based deployment +- [ ] How-to guides for deploying DataHub to all the major cloud providers + - [x] AWS + - [ ] Azure + - [x] GCP + +#### Product Analytics for DataHub + +- [x] Helping you understand how your users are interacting with DataHub +- [x] Integration with common systems like Google Analytics etc. + +#### Usage-Based Insights + +- [x] Display frequently used datasets, etc. +- [ ] Improved search relevance through usage data + +#### Role-based Access Control + +- Support for fine-grained access control for metadata operations (read, write, modify) +- Scope: Access control on entity-level, aspect-level and within aspects as well. +- This provides the foundation for Tag Governance, Dataset Preview access control etc. + +#### No-code Metadata Model Additions + +Use Case: Developers should be able to add new entities and aspects to the metadata model easily + +- [x] No need to write any code (in Java or Python) to store, retrieve, search and query metadata +- [ ] No need to write any code (in GraphQL or UI) to visualize metadata + +### Q1 2021 [Jan - Mar 2021] + +#### React UI + +- [x] Build a new UI based on React +- [x] Deprecate open-source support for Ember UI + +#### Python-based Metadata Integration + +- [x] Build a Python-based Ingestion Framework +- [x] Support common people repositories (LDAP) +- [x] Support common data repositories (Kafka, SQL databases, AWS Glue, Hive) +- [x] Support common transformation sources (dbt, Looker) +- [x] Support for push-based metadata emission from Python (e.g. Airflow DAGs) + +#### Dashboards and Charts + +- [x] Support for dashboard and chart entity page +- [x] Support browse, search and discovery + +#### SSO for Authentication + +- [x] Support for Authentication (login) using OIDC providers (Okta, Google etc) + +#### Tags + +Use-Case: Support for free-form global tags for social collaboration and aiding discovery + +- [x] Edit / Create new tags +- [x] Attach tags to relevant constructs (e.g. datasets, dashboards, users, schema_fields) +- [x] Search using tags (e.g. find all datasets with this tag, find all entities with this tag) + +#### Business Glossary + +- [x] Support for business glossary model (definition + storage) +- [ ] Browse taxonomy +- [x] UI support for attaching business terms to entities and fields + +#### Jobs, Flows / Pipelines + +Use case: Search and Discover your Pipelines (e.g. Airflow DAGs) and understand data lineage with datasets + +- [x] Support for Metadata Models + Backend Implementation +- [x] Metadata Integrations with systems like Airflow. + +#### Data Profiling and Dataset Previews + +Use Case: See sample data for a dataset and statistics on the shape of the data (column distribution, nullability etc.) + +- [ ] Support for data profiling and preview extraction through ingestion pipeline +- Out of scope for Q1: Access control of data profiles and sample data diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/schema-history.md b/docs-archive/versioned_docs/version-1.5.0/docs/schema-history.md new file mode 100644 index 00000000..bb530f1f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/schema-history.md @@ -0,0 +1,67 @@ +--- +title: Schema History +slug: /schema-history +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/schema-history.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Schema History + + + +Schema History is a valuable tool for understanding how a Dataset changes over time and gives insight into the following cases, +along with informing Data Practitioners when these changes happened. + +- A new field is added +- An existing field is removed +- An existing field changes type + +Schema History uses DataHub's [Timeline API](/docs/dev-guides/timeline/) to compute schema changes. + +## Schema History Setup, Prerequisites, and Permissions + +Schema History is viewable in the DataHub UI for any Dataset that has had at least one schema change. To view a Dataset, a user +must have the **View Entity Page** privilege, or be assigned to **any** DataHub Role. + +## Using Schema History + +You can view the Schema History for a Dataset by navigating to that Dataset's Schema Tab. As long as that Dataset has more than +one version, you can view what a Dataset looked like at any given version by using the version selector. +Here's an example from DataHub's official Demo environment with the +Snowflake pets dataset. + +

+ +

+ +If you click on an older version in the selector, you'll be able to see what the schema looked like back then. Notice +the changes here to the glossary terms for the `status` field, and to the descriptions for the `created_at` and `updated_at` +fields. + +

+ +

+ +In addition to this, you can also toggle the Audit view that shows you when the most recent changes were made to each field. +You can active this by clicking on the Audit icon you see above the top right of the table. + +

+ +

+ +You can see here that some of these fields were added at the oldest dataset version, while some were added only at this latest +version. Some fields were even modified and had a type change at the latest version! + +### GraphQL + +- [getSchemaBlame](../graphql/queries.md#getSchemaBlame) +- [getSchemaVersionList](../graphql/queries.md#getSchemaVersionList) + +## FAQ and Troubleshooting + +**What updates are planned for the Schema History feature?** + +In the future, we plan on adding the following features + +- Supporting a linear timeline view where you can see what changes were made to various schema fields over time +- Adding a diff viewer that highlights the differences between two versions of a Dataset diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/slack.md b/docs-archive/versioned_docs/version-1.5.0/docs/slack.md new file mode 100644 index 00000000..cd4a0a52 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/slack.md @@ -0,0 +1,53 @@ +--- +title: Slack +slug: /slack +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/slack.md' +--- +# Slack + +The DataHub Slack is a thriving and rapidly growing community - we can't wait for you to join us! + +_[Sign up here](https://datahub.com/slack?utm_source=docs&utm_medium=page_link&utm_campaign=docs_page_link) to join us on Slack and to subscribe to the DataHub Community newsletter. Already a member? [Log in here](https://datahub.com/slack?utm_source=docs&utm_medium=docs&utm_campaign=docs_page_link)._ + +## Slack Guidelines + +In addition to our [Code of Conduct](CODE_OF_CONDUCT.md), we expect all Slack Community Members to respect the following guidelines: + +### Avoid using DMs and @mentions + +Whenever possible, post your questions and responses in public channels so other Community Members can benefit from the conversation and outcomes. Limit the use of @mentions of other Community Members to be considerate of notification noise. + +### Make use of threads + +Threads help us keep conversations contained and help us ensure we help you find a resolution and get you the support you need. + +Use threads when posting long messages and large blocks of code and/or stack trace - it is a MASSIVE help for us to keep track of the large volume of questions across our various support channels. + +### Do not post the same question across multiple channels + +If you're having a tough time getting the support you need (or aren't sure where to go!), please DM [@Maggie](https://datahubspace.slack.com/team/U0121TRV0FL) for support + +### Do not solicit members of our Slack + +The DataHub Community exists to collaborate with, learn from, and support one another. It is not a space to pitch your products or services directly to our members via public channels, private channels, or direct messages. + +We are excited to have a growing presence from vendors to help answer questions from Community Members as they may arise, but we have a strict 3-strike policy against solicitation: + +1. First occurrence: We'll give you a friendly but public reminder that the behavior is inappropriate according to our guidelines. +2. Second occurrence: We'll send you a DM warning that any additional violations will result in removal from the community. +3. Third occurrence: We'll delete or ban your account. + +We reserve the right to ban users without notice if they are clearly spamming our Community Members. + +## Navigating DataHub Slack + +Lets get you settled in -- + +- **Head over to [#introduce-yourself](https://datahubspace.slack.com/archives/C01PU1K6GDP) to, well, introduce yourself!** We'd love to learn more about you, what brings you here, and how we can support you +- **Not sure how to start?** You guessed it, check out [#getting-started](https://datahubspace.slack.com/archives/CV2KB471C) - we'll point you in the right direction +- **Looking for general debugging help?** [#troubleshoot](https://datahubspace.slack.com/archives/C029A3M079U) is the place to go +- **Need some live support from the Core DataHub Team?** Join us during our 2xWeek Office Hours via Zoom! Check out [#office-hours](https://datahubspace.slack.com/archives/C02AD211493) for more details +- **Looking for ways to contribute to the DataHub project?** Tell us all about it in [#contribute](https://datahubspace.slack.com/archives/C017W0NTZHR) +- **Have suggestions on how to make DataHub better?** We can't wait to hear them in [#feature-requests](https://datahubspace.slack.com/archives/C02FWNS2F08) +- **Excited to share your experience working with DataHub?** [#show-and-tell](https://datahubspace.slack.com/archives/C02FD9PLCA0) is the perfect channel for you +- **Need something else?** Reach out to [@Maggie](https://datahubspace.slack.com/team/U0121TRV0FL) - our Community Product Manager diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/sync-status.md b/docs-archive/versioned_docs/version-1.5.0/docs/sync-status.md new file mode 100644 index 00000000..ad30fc08 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/sync-status.md @@ -0,0 +1,49 @@ +--- +title: Sync Status +slug: /sync-status +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/sync-status.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Sync Status + + + +When looking at metadata in DataHub, it's useful to know if the information you're looking at is relevant. +Specifically, if metadata is stale, or hasn't been updated in a while, then you should consider refreshing that metadata +using [metadata ingestion](./../metadata-ingestion/README.md) or [deleting](./how/delete-metadata.md) it if it no longer exists. + +## Sync Status Setup, Prerequisites, and Permissions + +The sync status feature is enabled by default and does not require any special setup. + +## Using Sync Status + +The DataHub UI will display the sync status in the top right corner of the page. + +The last synchronized date is basically the last time an ingestion run saw an entity. It is computed as the most recent update to the entity, excluding changes done through the UI. If an ingestion run restates an entity but doesn't actually cause any changes, we still count that as an update for the purposes of sync status. + +
+ Technical details: computing the last synchronized timestamp + +To compute the last synchronized timestamp, we look at the system metadata of all aspects associated with the entity. +We exclude any aspects where the system metadata `runId` value is unset or equal to `no-run-id-provided`, as this is what filters out changes made through the UI. +Finally, we take the most recent system metadata `lastObserved` timestamp across the aspects and use that as the last synchronized timestamp. + +
+ +

+ +

+ +We'll automatically assign a color based on the sync status recency: + +- Green: last synchronized in the past week +- Yellow: last synchronized in the past month +- Red: last synchronized more than a month ago + +You can hover over the sync status message in the UI to view the exact timestamp of the most recent sync. + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/tags.md b/docs-archive/versioned_docs/version-1.5.0/docs/tags.md new file mode 100644 index 00000000..85ec52a1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/tags.md @@ -0,0 +1,114 @@ +--- +title: Tags +slug: /tags +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/tags.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Tags + + + +Tags are informal, loosely controlled labels that help in search & discovery. They can be added to datasets, dataset schemas, or containers, for an easy way to label or categorize entities – without having to associate them to a broader business glossary or vocabulary. + +Tags can help help you in: + +- Querying: Tagging a dataset with a phrase that a co-worker can use to query the same dataset +- Mapping assets to a category or group of your choice + +## Tags Setup, Prerequisites, and Permissions + +What you need to add tags: + +- **Edit Tags** metadata privilege to add tags at the entity level +- **Edit Dataset Column Tags** to edit tags at the column level + +You can create these privileges by creating a new [Metadata Policy](./authorization/policies.md). + +## Using DataHub Tags + +### Adding a Tag + +To add a tag at the dataset or container level, simply navigate to the page for that entity and click on the **Add Tag** button. + +

+ +

+ +Type in the name of the tag you want to add. You can add a new tag, or add a tag that already exists (the autocomplete will pull up the tag if it already exists). + +

+ +

+ +Click on the "Add" button and you'll see the tag has been added! + +

+ +

+ +If you would like to add a tag at the schema level, hover over the "Tags" column for a schema until the "Add Tag" button shows up, and then follow the same flow as above. + +

+ +

+ +### Removing a Tag + +To remove a tag, simply click on the "X" button in the tag. Then click "Yes" when prompted to confirm tag removal. + +### Searching by a Tag + +You can search for a tag in the search bar, and even filter entities by the presence of a specific tag. + +

+ +

+ +## Additional Resources + +### Videos + +**Add Ownership, Tags, Terms, and more to DataHub via CSV!** + +

+ +

+ +### GraphQL + +- [addTag](../graphql/mutations.md#addtag) +- [addTags](../graphql/mutations.md#addtags) +- [batchAddTags](../graphql/mutations.md#batchaddtags) +- [removeTag](../graphql/mutations.md#removetag) +- [batchRemoveTags](../graphql/mutations.md#batchremovetags) +- [createTag](../graphql/mutations.md#createtag) +- [updateTag](../graphql/mutations.md#updatetag) +- [deleteTag](../graphql/mutations.md#deletetag) + +You can easily fetch the Tags for an entity with a given its URN using the **tags** property. Check out [Working with Metadata Entities](./api/graphql/how-to-set-up-graphql.md#querying-for-tags-of-an-asset) for an example. + +### DataHub Blog + +- [Tags and Terms: Two Powerful DataHub Features, Used in Two Different Scenarios + Managing PII in DataHub: A Practitioner’s Guide](https://medium.com/datahub-project/tags-and-terms-two-powerful-datahub-features-used-in-two-different-scenarios-b5b4791e892e) + +## FAQ and Troubleshooting + +**What is the difference between DataHub Tags and Glossary Terms?** + +DataHub Tags are informal, loosely controlled labels while Terms are part of a controlled vocabulary, with optional hierarchy. Tags have no element of formal, central management. + +Usage and applications: + +- An asset may have multiple tags. +- Tags serve as a tool for search & discovery while Terms are typically used to standardize types of leaf-level attributes (i.e. schema fields) for governance. E.g. (EMAIL_PLAINTEXT) + +**How are DataHub Tags different from Domains?** + +Domains are a set of top-level categories usually aligned to business units/disciplines to which the assets are most relevant. They rely on central or distributed management. A single domain is assigned per data asset. + +### Related Features + +- [Glossary Terms](./glossary/business-glossary.md) +- [Domains](./domains.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/tests/metadata-tests.md b/docs-archive/versioned_docs/version-1.5.0/docs/tests/metadata-tests.md new file mode 100644 index 00000000..10d87872 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/tests/metadata-tests.md @@ -0,0 +1,303 @@ +--- +title: Metadata Tests +slug: /tests/metadata-tests +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/tests/metadata-tests.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Metadata Tests + + + +DataHub includes a highly configurable, no-code framework that allows you to configure broad-spanning monitors & continuous actions +for the data assets - datasets, dashboards, charts, pipelines - that make up your enterprise Metadata Graph. +At the center of this framework is the concept of a Metadata Test. + +There are two powerful use cases that are uniquely enabled by the Metadata Tests framework: + +1. Automated Asset Classification +2. Automated Metadata Completion Monitoring + +### Automated Asset Classification + +Metadata Tests allows you to define conditions for selecting a subset of data assets (e.g. datasets, dashboards, etc), +along with a set of actions to take for entities that are selected. After the test is defined, the actions +will be applied continuously over time, as the selection set evolves & changes with your data ecosystem. + +When defining selection criteria, you'll be able to choose from a range of useful technical signals (e.g. usage, size) that are automatically +extracted by DataHub (which vary by integration). This makes automatically classifying the "important" assets in your organization quite easy, which +is in turn critical for running effective Data Governance initiatives within your organization. + +For example, we can define a Metadata Test which selects all Snowflake Tables which are in the top 10% of "most queried" +for the past 30 days, and then assign those Tables to a special "Tier 1" group using DataHub Tags, Glossary Terms, or Domains. + +### Automated Data Governance Monitoring + +Metadata Tests allow you to define & monitor a set of rules that apply to assets in your data ecosystem (e.g. datasets, dashboards, etc). This is particularly useful when attempting to govern +your data, as it allows for the (1) definition and (2) measurement of centralized metadata standards, which are key for both bootstrapping +and maintaining a well-governed data ecosystem. + +For example, we can define a Metadata Test which requires that all "Tier 1" data assets (e.g. those marked with a special Tag or Glossary Term), +must have the following metadata: + +1. At least 1 explicit owner _and_ +2. High-level, human-authored documentation _and_ +3. At least 1 Glossary Term from the "Classification" Term Group + +Then, we can closely monitor which assets are passing and failing these rules as we work to improve things over time. +We can easily identify assets that are _in_ and _out of_ compliance with a set of centrally-defined standards. + +By applying automation, Metadata Tests +can enable the full lifecycle of complex Data Governance initiatives - from scoping to execution to monitoring. + +## Metadata Tests Setup, Prerequisites, and Permissions + +What you need to manage Metadata Tests on DataHub: + +- **Manage Tests** Privilege + +This Platform Privilege allows users to create, edit, and remove all Metadata Tests on DataHub. Therefore, it should only be +given to those users who will be serving as metadata Admins of the platform. The default `Admin` role has this Privilege. + +> Note that the Metadata Tests feature is currently limited in support for the following DataHub Asset Types: +> +> - Dataset +> - Dashboard +> - Chart +> - Data Flow (e.g. Pipeline) +> - Data Job (e.g. Task) +> - Container (Database, Schema, Project) +> +> If you'd like to see Metadata Tests for other asset types, please let your DataHub Cloud CustomerSuccess partner know! + +## Using Metadata Tests + +Metadata Tests can be created by first navigating to **Govern > Tests**. + +To begin building a new Metadata, click **Create new Test**. + +

+ +

+ +### Creating a Metadata Test + +Inside the Metadata Test builder, we'll need to construct the 3 parts of a Metadata Test: + +1. **Selection Criteria** - Select assets that are in the scope of the test +2. **Rules** - Define rules that selected assets can either pass or fail +3. **Actions (Optional)** - Define automated actions to be taken assets that are passing + or failing the test + +

+ +

+ +#### Step 1. Defining Selection Criteria (Scope) + +In the first step, we define a set of conditions that are used to select a subset of the assets in our Metadata Graph +that will be "in the scope" of the new test. Assets that **match** the selection conditions will be considered in scope, while those which do not are simply not applicable for the test. +Once the test is created, the test will be evaluated for any assets which fall in scope on a continuous basis (when an asset changes on DataHub +or once every day). + +##### Selecting Asset Types + +You must select at least one asset _type_ from a set that includes Datasets, Dashboards, Charts, Data Flows (Pipelines), Data Jobs (Tasks), +and Containers. + +

+ +

+ +Entities will the selected types will be considered in scope, while those of other types will be considered out of scope and +thus omitted from evaluation of the test. + +##### Building Conditions + +**Property** conditions are the basic unit of comparison used for selecting data assets. Each **Property** condition consists of a target _property_, +an _operator_, and an optional _value_. + +A _property_ is an attribute of a data asset. It can either be a technical signal (e.g. **metric** such as usage, storage size) or a +metadata signal (e.g. owners, domain, glossary terms, tags, and more), depending on the asset type and applicability of the signal. +The full set of supported _properties_ can be found in the table below. + +An _operator_ is the type of predicate that will be applied to the selected _property_ when evaluating the test for an asset. The types +of operators that are applicable depend on the selected property. Some examples of operators include `Equals`, `Exists`, `Matches Regex`, +and `Contains`. + +A _value_ defines the right-hand side of the condition, or a pre-configured value to evaluate the property and operator against. The type of the value +is dependent on the selected _property_ and *operator. For example, if the selected *operator\* is `Matches Regex`, the type of the +value would be a string. + +By selecting a property, operator, and value, we can create a single condition (or predicate) used for +selecting a data asset to be tested. For example, we can build property conditions that match: + +- All datasets in the top 25% of query usage in the past 30 days +- All assets that have the "Tier 1" Glossary Term attached +- All assets in the "Marketing" Domain +- All assets without owners +- All assets without a description + +To create a **Property** condition, simply click **Add Condition** then select **Property** condition. + +

+ +

+ +We can combine **Property** conditions using boolean operators including `AND`, `OR`, and `NOT`, by +creating **Logical** conditions. To create a **Logical** condition, simply click **Add Condition** then select an +**And**, **Or**, or **Not** condition. + +

+ +

+ +Logical conditions allow us to accommodate complex real-world selection requirements: + +- All Snowflake Tables that are in the Top 25% of most queried AND do not have a Domain +- All Looker Dashboards that do not have a description authored in Looker OR in DataHub + +#### Step 2: Defining Rules + +In the second step, we can define a set of conditions that selected assets must match in order to be "passing" the test. +To do so, we can construct another set of **Property** conditions (as described above). + +> **Pro-Tip**: If no rules are supplied, then all assets that are selected by the criteria defined in Step 1 will be considered "passing". +> If you need to apply an automated Action to the selected assets, you can leave the Rules blank and continue to the next step. + +

+ +

+ +When combined with the selection criteria, Rules allow us to define complex, highly custom **Data Governance** policies such as: + +- All datasets in the top 25% of query usage in the past 30 days **must have an owner**. +- All assets in the "Marketing" Domain **must have a description** +- All Snowflake Tables that are in the Top 25% of most queried AND do not have a Domain **must have + a Glossary Term from the Classification Term Group** + +##### Validating Test Conditions + +During Step 2, we can quickly verify that the Selection Criteria & Rules we've authored +match our expectations by testing them against some existing assets indexed by DataHub. + +To verify your Test conditions, simply click **Try it out**, find an asset to test against by searching & filtering down your assets, +and finally click **Run Test** to see whether the asset is passes or fails the provided conditions. + +

+ +

+ +#### Step 3: Defining Actions (Optional) + +> If you don't wish to take any actions for assets that pass or fail the test, simply click 'Skip'. + +In the third step, we can define a set of Actions that will be automatically applied to each selected asset which passes or fails the Rules conditions. + +For example, we may wish to mark **passing** assets with a special DataHub Tag or Glossary Term (e.g. "Tier 1"), or remove these special marking for those which are failing. +This allows us to automatically control classifications of data assets as they move in and out of compliance with the Rules defined in Step 2. + +A few of the supported Action types include: + +- Adding or removing specific Tags +- Adding or removing specific Glossary Terms +- Adding or removing specific Owners +- Adding or removing to a specific Domain + +

+ +

+ +#### Step 4: Name, Category, Description + +In the final step, we can add a freeform name, category, and description for our new Metadata Test. + +### Viewing Test Results + +Metadata Test results can be viewed in 2 places: + +1. On an asset profile page (e.g. Dataset profile page), inside the **Validation** tab. +2. On the Metadata Tests management page. To view all assets passing or failing a particular test, + simply click on the labels which showing the number of passing or failing assets. + +

+ +

+ +### Updating an Existing Test + +To update an existing Test, simply click **Edit** on the test you wish to change. + +Then, make the changes required and click **Save**. When you save a Test, it may take up to 2 minutes for changes +to be reflected across DataHub. + +### Removing a Test + +To remove a Test, simply click on the trashcan icon located on the Tests list. This will remove the Test and +deactivate it so that it no is evaluated. + +When you delete a Test, it may take up to 2 minutes for changes to be reflected. + +### GraphQL + +- [listTests](../../graphql/queries.md#listtests) +- [createTest](../../graphql/mutations.md#createtest) +- [deleteTest](../../graphql/mutations.md#deletetest) + +## Performance Tuning for Batch Evaluation + +This section covers performance considerations for scheduled batch evaluation of Metadata Tests. + +### Batch Processing Architecture + +The batch evaluation job runs in the `datahub-upgrade` container with direct local access to `EntityService` and `EntitySearchService`. Test results and actions are written asynchronously to Kafka, then processed by the MCE Consumer (embedded in GMS by default) which writes to the database. + +### When ElasticSearch Executor is Used + +- Enabled by default (`METADATA_TESTS_ELASTICSEARCH_EXECUTOR_ENABLED=true`) +- Used when test selection criteria and rules can be expressed as ElasticSearch queries +- Provides faster evaluation for large entity counts +- Falls back to SQL queries via local EntityService for unsupported predicates + +### Key Configuration Variable + +The primary performance tuning parameter is **`METADATA_TESTS_ACTIONS_CONCURRENCY`** (default: `2`), which controls the number of threads generating action MCPs. Increasing this value allows more actions to be produced in parallel, improving throughput when processing large numbers of test results. + +### Scaling Recommendations + +Since test evaluation uses local services, the bottleneck is MCE Consumer throughput processing async writes from Kafka. To scale effectively: + +- Adequate Kafka partition count on `MetadataChangeProposal_v1` topic +- Sufficient MCE consumer capacity (embedded or standalone) + +For standalone MCE/MAE consumer deployment and Kafka configuration, see [Configuring Kafka](../how/kafka-config.md). For complete environment variable reference, see [Environment Variables](../deploy/environment-vars.md#component-configuration). + +## FAQ and Troubleshooting + +**When are Metadata Tests evaluated?** + +Metadata Tests are evaluated in two scenarios: + +1. **Real-time evaluation**: When an individual asset changes in DataHub, all tests that include it in scope are evaluated. This feature is typically disabled by default. It can be enabled on demand, subject to disussion. + +2. **Scheduled evaluation**: A dedicated Metadata Test evaluator runs on a recurring schedule (typically every 24 hours) and evaluates all tests against the entire Metadata Graph. The cadence can be adjusted. It can be made more frequent in limited cases where metadata is small, subject to discussion. + +Both scenarios have increased server load concerns which may require additional server resources and consultation with your Acryl representative for the associated cost. + +**Can I configure a custom evaluation schedule for my Metadata Test?** + +No, you cannot. Currently, the internal evaluator will ensure that tests are run continuously for +each asset, regardless of whether it is being changed on DataHub. + +**How is a Metadata Test different from an Assertion?** + +An Assertion is a specific test, similar to a unit test, that is defined for a single data asset. Typically, +it will include domain-specific knowledge about the asset and test against physical attributes of it. For example, an Assertion +may verify that the number of rows for a specific table in Snowflake falls into a well-defined range. + +A Metadata Test is a broad spanning predicate which applies to a subset of the Metadata Graph (e.g. across multiple +data assets). Typically, it is defined against _metadata_ attributes, as opposed to the physical data itself. For example, +a Metadata Test may verify that ALL tables in Snowflake have at least 1 assigned owner, and a human-authored description. +Metadata Tests allow you to manage broad policies across your entire data ecosystem driven by metadata, for example to +augment a larger scale Data Governance initiative. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/townhall-history.md b/docs-archive/versioned_docs/version-1.5.0/docs/townhall-history.md new file mode 100644 index 00000000..61f35160 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/townhall-history.md @@ -0,0 +1,557 @@ +--- +title: Town Hall History +slug: /townhall-history +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/townhall-history.md +--- +# Town Hall History + +:::note +For the Town Hall meetings after June 2023, please refer to our [LinkedIn Live event history](https://www.linkedin.com/company/acryl-data/events/). +::: + +### June 2023 + +[Full YouTube video](https://www.youtube.com/watch?v=1QVcUmRQK5E) + +- Community & Project Updates - Maggie Hays & Shirshanka Das (DataHub) +- Community Case Study: Dataset Joins - Raj Tekal & Bobbie-Jean Nowak (Optum) +- DataHub 201: Column-Level Data Lineage - Hyejin Yoon (DataHub) +- Sneak Peek: BigQuery Column-Level Lineage with SQL Parsing - Harshal Sheth (DataHub) +- DataHub Performance Tuning – Indy Prentice (DataHub) + +### May 2023 + +[Full YouTube video](https://www.youtube.com/watch?v=KHNPjSbbZR8) + +**Agenda** + +- Community - Maggie Hays & Shirshanka Das (DataHub) +- Community Case Study: Jira + DataHub for Access Requests - Joshua Garza (Sharp Healthcare) +- Sneak Peek: Use your own ownership types - Pedro Silva (DataHub) +- Sneak Peek: Data Contracts are coming! – John Joyce, Shirshanka (DataHub) +- Bring DataHub into your BI Tools — Chris Collins (DataHub) + +### Apr 2023 + +[Full YouTube video](https://www.youtube.com/watch?v=D5YYGu-ZIBo) + +**Agenda** + +- Community & Roadmap Updates - Maggie Hays & Shirshanka Das (DataHub) +- DataHub 201: Python SDK - Hyejin Yoon (DataHub) +- Streamlined Search & Browse Experience - Chris Collins (DataHub) +- DataHub GitHub Actions - Harshal Sheth (DataHub) +- Data Products in DataHub - Shirshanka Das & Chris Collins (DataHub) +- DataHub Docs Bot - Maggie Hays (DataHub) + +### Mar 2023 + +[Full YouTube video](https://youtu.be/BTX8rIBe0yo) + +**Agenda** + +- Community & Roadmap Update +- Recent Releases +- Community Case Study — Jumio’s DataHub adoption journey +- DataHub 201: Data Debugging +- Sneak Peek: Streamlined Filtering Experience + +### Feb 2023 + +[Full YouTube video](https://youtu.be/UItt4ppJSFc) + +**Agenda** + +- Community & Roadmap Update +- Recent Releases +- Community Case Study - How the Hurb Team successfully implemented and adopted DataHub within their organization +- Sneak Peek: Subscriptions and Notifications +- Search Improvements - API support for pagination +- New Feature - Custom Queries +- Simplifying Metadata Ingestion +- DataHub 201: Rolling Out DataHub + +### Jan 2023 (26th) + +[Full YouTube video](https://youtu.be/A3mSiGHZ6Rc) + +**Agenda** + +- What’s to Come - Q1 2023 Roadmap: Data Products, Data Contracts and more +- Community Case Study - Notion: Automating annotations and metadata propagation +- Community Contribution - Grab: Improvements to documentation editing +- Simplifying DataHub - Removing Schema Registry requirement and introducing DataHub Lite + +### Jan 2023 (5th) + +[Full YouTube video](https://youtu.be/ECxIMbKwuOY) + +**Agenda** + +- DataHub Community: 2022 in Review - Our Community of Data Practitioners is one of a kind. We’ll take the time to celebrate who we are, what we’ve built, and how we’ve collaborated in the past 12 months. +- Search Improvements - Learn how we’re making the Search experience smarter and faster to connect you with the most relevant resources during data discovery. +- Removing Schema Registry Requirement - Hear all about ongoing work to simplify the DataHub deployment process. +- Smart Data Profiling - We’re making big improvements to data profiling! Smart data profiling will reduce processing time by only scanning datasets that have recently changed. +- Sneak Peek: Time-based Lineage - Get a preview of how you’ll soon be able to trace lineage between datasets across different points in time to understand how interdependencies have evolved. +- Sneak Peek: Chrome Extension - Soon, you’ll be able to quickly access rich metadata from DataHub while exploring resources in Looker via our upcoming Chrome Extension. + +### Dec 2023 + +[Full YouTube video](https://youtu.be/BlCLhG8lGoY) + +**Agenda** + +November Town Hall (in December!) + +- Community Case Study - The Pinterest Team will share how they have integrated DataHub + Thrift and extended the Metadata Model with a Data Element entity to capture semantic types. +- NEW! Ingestion Quickstart Guides - DataHub newbies, this one is for you! We’re rolling out ingestion quickstart guides to help you quickly get up and running with DataHub + Snowflake, BigQuery, and more! +- NEW! In-App Product Tours - We’re making it easier than ever for end-users to get familiar with all that DataHub has to offer - hear all about the in-product onboarding resources we’re rolling out soon! +- DataHub UI Navigation and Performance - Learn all about upcoming changes to our user experience to make it easier (and faster!) for end users to work within DataHub. +- Sneak Peek! Manual Lineage via the UI - The Community asked and we’re delivering! Soon you’ll be able to manually add lineage connections between Entities in DataHub. +- NEW! Slack + Microsoft Teams Integrations - Send automated alerts to Slack and/or Teams to keep track of critical events and changes within DataHub. +- Hacktoberfest Winners Announced - We’ll recap this year’s Hacktoberfest and announce three winners of a $250 Amazon gift card & DataHub Swag. + +### Oct 2022 + +[Full YouTube video](https://youtu.be/B74WHxX5EMk) + +**Agenda** + +- Conquer Data Governance with DataHub’s Metadata Tests - Learn how to tackle Data Governance with incremental, automation-driven governance using Metadata Tests provided in DataHub’s DataHub Cloud offering +- Community Case Study - The Grab Team shares how they are using DataHub for data discoverability, automated classification and governance workflows, data quality observability, and beyond! +- Upcoming Ingestion Sources - We’ll tell you the ins and outs of our upcoming dbt Cloud and Unity Catalog connectors +- Sneak Peek! Saved Views - Learn how you can soon use Saved Views to help end-users navigate entities in DataHub with more precision and focus +- Performance Improvements - Hear about the latest upgrades to DataHub performance + +### Sep 2022 + +[Full YouTube video](https://youtu.be/FjkNySWkghY) + +**Agenda** + +- Column Level Lineage is here! - Demo of column-level lineage and impact analysis in the DataHub UI +- Community Case Study - The Stripe Team shares how they leverage DataHub to power observability within their Airflow-based ecosystem +- Sneak Peek! Automated PII Classification - Preview upcoming functionality to automatically identify data fields that likely contain sensitive data +- Ingestion Improvements Galore - Improved performance and functionality for dbt, Looker, Tableau, and Presto ingestion sources + +### Aug 2022 + +[Full YouTube video](https://youtu.be/EJCKxKBvCwo) + +**Agenda** + +- Community Case Study - The Etsy Team shares their journey of adopting DataHub +- Looker & DataHub Improvements - surface the most relevant Looks and Dashboards +- Home Page Improvements to tailor the Browse experience +- Unified Ingestion Summaries - View live logs for UI-based ingestion and see historical ingestion reports across CLI and UI-based ingestion +- Patch Support - Native support for PATCH in the metadata protocol to support efficient updates to add & remove owners, lineage, tags and more +- Sneak Peek! Advanced Search + +### Jul 2022 + +[Full YouTube video](https://youtu.be/Zrkf3Mzcvc4) + +**Agenda** + +- Community Updates +- Project Updates +- Improvements to UI-Based Ingestion +- Sneak Preview - Bulk Edits via the UI +- Streamlined Metadata Ingestion +- DataHub 201: Metadata Enrichment + +### Jun 2022 + +[Full YouTube video](https://youtu.be/fAD53fEJ6m0) + +**Agenda** + +- Community Updates +- Project Updates +- dbt Integration Updates +- CSV Ingestion Support +- DataHub 201 - Glossary Term Deep Dive + +### May 2022 + +[Full YouTube video](https://youtu.be/taKb_zyowEE) + +**Agenda** + +- Community Case Study: Hear how the G-Research team is using Cassandra as DataHub’s Backend +- Creating & Editing Glossary Terms from the DataHub UI +- DataHub User Onboarding via the UI +- DataHub 201: Impact Analysis +- Sneak Peek: Data Reliability with DataHub +- Metadata Day Hackathon Winners + +### Apr 2022 + +[Full YouTube video](https://www.youtube.com/watch?v=7iwNxHgqxtg) + +**Agenda** + +- Community Case Study: Hear from Included Health about how they are embedding external tools into the DataHub UI +- New! Actions Framework: run custom code when changes happen within DataHub +- UI Refresh for ML Entities +- Improved deletion support for time-series aspects, tags, terms, & more +- OpenAPI Improvements + +### Mar 2022 + +[Full YouTube video](https://www.youtube.com/watch?v=IVazVgcNRdw) + +**Agenda** + +- Community Case Study: Hear from Zendesk about how they are applying “shift left” principles by authoring metadata in their Protobuf schemas +- RBAC Functionality: View-Based Policies +- Schema Version History - surfacing the history of schema changes in DataHub's UI +- Improvements to Airflow Ingestion, including Run History +- Container/Domain-Level Property Inheritance +- Delete API + +### Feb 2022 + +[Full YouTube video](https://www.youtube.com/watch?v=enBqB2Dbuv4) + +**Agenda** + +- Lineage Impact Analysis - using DataHub to understand the impact of changes on downstream dependencies +- Displaying Data Quality Checks in the UI +- Roadmap update: Schema Version History & Column-Level Lineage +- Community Case Study: Managing Data Lineage via YAML + +### Jan 2022 + +[Full YouTube video](https://youtu.be/ShlSR3dMUnE) + +**Agenda** + +- Community & Roadmap Updates by Maggie Hays (DataHub) +- Project Updates by Shirshanka Das (DataHub) +- Community Case Study: Adding Dataset Transformers by Eric Cooklin (Stash) +- Demo: Data Domains & Containers by John Joyce (DataHub) +- DataHub Basics — Data Profiling & Usage Stats 101 by Maggie Hays & Tamás Németh (DataHub) +- Demo: Spark Lineage by Mugdha Hardikar (GS Lab) & Shirshanka Das + +### Dec 2021 + +[Full YouTube video](https://youtu.be/rYInKCwxu7o) + +**Agenda** + +- Community & Roadmap Updates by Maggie Hays (DataHub) +- Project Updates by Shirshanka Das (DataHub) +- 2021 DataHub Community in Review by Maggie Hays +- DataHub Basics -- Users, Groups, & Authentication 101 by Pedro Silva (DataHub) +- Sneak Peek: UI-Based Ingestion by John Joyce (DataHub) +- Case Study — DataHub at Grofers by Shubham Gupta +- Top DataHub Contributors of 2021 - Maggie Hays +- Final Surprise! We Interviewed a 10yo and a 70yo about DataHub + +### Nov 2021 + +[Full YouTube video](https://youtu.be/to80sEDZz7k) + +**Agenda** + +- Community & Roadmap Updates by Maggie Hays (DataHub) +- Project Updates by Shirshanka Das (DataHub) +- DataHub Basics -- Lineage 101 by John Joyce & Surya Lanka (DataHub) +- Introducing No-Code UI by Gabe Lyons & Shirshanka Das (DataHub) +- DataHub API Authentication by John Joyce (DataHub) +- Case Study: LinkedIn pilot to extend the OSS UI by Aikepaer Abuduweili & Joshua Shinavier + +### Oct 2021 + +[Full YouTube video](https://youtu.be/GrS_uZhYNm0) + +**Agenda** + +- DataHub Community & Roadmap Update - Maggie Hays (DataHub) +- October Project Updates - Shirshanka Das (DataHub) +- Introducing Recommendations - John Joyce & Dexter Lee (DataHub) +- Case Study: DataHub @ hipages - Chris Coulson (hipages) +- Data Profiling Improvements - Surya Lanka & Harshal Sheth (DataHub) +- Lineage Improvements & BigQuery Dataset Lineage by Gabe Lyons & Varun Bharill (DataHub) + +### Sep 2021 + +[Full YouTube video](https://youtu.be/nQDiKPKnLLQ) + +**Agenda** + +- Project Updates and Callouts by Shirshanka + - GraphQL Public API Annoucement +- Demo: Faceted Search by Gabe Lyons (DataHub) +- Stateful Ingestion by Shirshanka Das & Surya Lanka (DataHub) +- Case-Study: DataHub @ Adevinta by Martinez de Apellaniz +- Recent Improvements to the Looker Connector by Shirshanka Das & Maggie Hays (DataHub) +- Offline + - Foreign Key and Related Term Mapping by Gabe Lyons (Acryl Data) [video](https://www.loom.com/share/79f27c2d9f6c4a3b8aacbc48c19add18) + +### Aug 2021 + +[Full YouTube video](https://youtu.be/3joZINi3ti4) + +**Agenda** + +- Project Updates and Callouts by Shirshanka + - Business Glossary Demo + - 0.8.12 Upcoming Release Highlights + - Users and Groups Management (Okta, Azure AD) +- Demo: Fine Grained Access Control by John Joyce (DataHub) +- Community Case-Study: DataHub @ Warung Pintar and Redash integration by Taufiq Ibrahim (Bizzy Group) +- New User Experience by John Joyce (DataHub) +- Offline + - Performance Monitoring by Dexter Lee (DataHub) [video](https://youtu.be/6Xfr_Y9abZo) + +### Jul 2021 + +[Full YouTube video](https://www.youtube.com/watch?v=rZsiB8z5rG4) + +[Medium Post](https://medium.com/datahub-project/datahub-project-updates-f4299cd3602e?source=friends_link&sk=27af7637f7ae44786ede694c3af512a5) + +**Agenda** + +- Project Updates by Shirshanka + - Release highlights +- Deep Dive: Data Observability: Phase 1 by Harshal Sheth, Dexter Lee (DataHub) +- Case Study: Building User Feedback into DataHub by Melinda Cardenas (NY Times) +- Demo: AWS SageMaker integration for Models and Features by Kevin Hu (DataHub) + +### Jun 2021 + +[Full YouTube video](https://www.youtube.com/watch?v=xUHOdDfdFpY) + +[Medium Post](https://medium.com/datahub-project/datahub-project-updates-ed3155476408?source=friends_link&sk=02816a16ff2acd688e6db8eb55808d31) + +**Agenda** + +- Project Updates by Shirshanka + - Release notes + - RBAC update + - Roadmap for H2 2021 +- Demo: Table Popularity powered by Query Activity by Harshal Sheth (DataHub) +- Case Study: Business Glossary in production at Saxo Bank by Sheetal Pratik (Saxo Bank), Madhu Podila (ThoughtWorks) +- Developer Session: Simplified Deployment for DataHub by John Joyce, Gabe Lyons (DataHub) + +### May 2021 + +[Full YouTube video](https://www.youtube.com/watch?v=qgW_xpIr1Ho) + +[Medium Post](https://medium.com/datahub-project/linkedin-datahub-project-updates-ed98cdf913c1?source=friends_link&sk=9930ec5579299b155ea87c747683d1ad) + +**Agenda** + +- Project Updates by Shirshanka - 10 mins + - 0.8.0 Release + - AWS Recipe by Dexter Lee (DataHub) +- Demo: Product Analytics design sprint (Maggie Hays (SpotHero), Dexter Lee (DataHub)) - 10 mins +- Use-Case: DataHub on GCP by Sharath Chandra (Confluent) - 10 mins +- Deep Dive: No Code Metadata Engine by John Joyce (DataHub) - 20 mins +- General Q&A and closing remarks + +### Apr 2021 + +[Full YouTube video](https://www.youtube.com/watch?v=dlFa4ubJ9ho) + +[Medium Digest](https://medium.com/datahub-project/linkedin-datahub-project-updates-2b0d26066b8f?source=friends_link&sk=686c47219ed294e0838ae3e2fe29084d) + +**Agenda** + +- Welcome - 5 mins +- Project Updates by Shirshanka - 10 mins + - 0.7.1 Release and callouts (dbt by Gary Lucas) + - Product Analytics design sprint announcement (Maggie Hayes) +- Use-Case: DataHub at DefinedCrowd ([video](https://www.youtube.com/watch?v=qz5Rpmw8I5E)) by Pedro Silva - 15 mins +- Deep Dive + Demo: Data Lineage! Airflow, Superset integration ([video](https://www.youtube.com/watch?v=3wiaqhb8UR0)) by Harshal Sheth and Gabe Lyons - 10 mins +- Use-Case: DataHub Hackathon at Depop ([video](https://www.youtube.com/watch?v=SmOMyFc-9J0)) by John Cragg - 10 mins +- Observability Feedback share out - 5 mins +- General Q&A and closing remarks - 5 mins + +### Mar 2021 + +[YouTube video](https://www.youtube.com/watch?v=xE8Uc27VTG4) + +[Medium Digest](https://medium.com/datahub-project/linkedin-datahub-project-updates-697f0faddd10?source=friends_link&sk=9888633c5c7219b875125e87a703ec4d) + +**Agenda** + +- Welcome - 5 mins +- Project Updates ([slides](https://drive.google.com/file/d/1c3BTP3oDAzJr07l6pY6CkDZi5nT0cLRs/view?usp=sharing)) by [Shirshanka](https://www.linkedin.com/in/shirshankadas/) - 10 mins + - 0.7.0 Release + - Project Roadmap +- Demo Time: Themes and Tags in the React App! by [Gabe Lyons](https://www.linkedin.com/in/gabe-lyons-9a574543/) - 10 mins +- Use-Case: DataHub at [Wolt](https://www.linkedin.com/company/wolt-oy/) ([slides](https://drive.google.com/file/d/1za7NKbnXpFV2bBDblP35CbQEIDwc9tog/view?usp=sharing)) by [Fredrik](https://www.linkedin.com/in/fredriksannholm/?originalSubdomain=fi) and Matti - 15 mins +- Poll Time: Observability Mocks! ([slides](https://drive.google.com/file/d/1Ih2EGf-76jhbNAjr2EsBLb7n8bra2WIz/view?usp=sharing)) - 5 mins +- General Q&A from sign up sheet, slack, and participants - 10 mins +- Closing remarks - 5 mins + +### Feb 2021 + +[YouTube video](https://www.youtube.com/watch?v=Z9ImbcsAVl0) + +[Medium Digest](https://medium.com/datahub-project/linkedin-datahub-project-updates-february-2021-edition-338d2c6021f0) +**Agenda** + +- Welcome - 5 mins +- Latest React App Demo! ([video](https://www.youtube.com/watch?v=RQBEJhcen5E)) by John Joyce and Gabe Lyons - 5 mins +- Use-Case: DataHub at Geotab ([video](https://www.youtube.com/watch?v=boyjT2OrlU4)) by [John Yoon](https://www.linkedin.com/in/yhjyoon/) - 15 mins +- Tech Deep Dive: Tour of new pull-based Python Ingestion scripts ([slides](https://docs.google.com/presentation/d/15Xay596WDIhzkc5c8DEv6M-Bv1N4hP8quup1tkws6ms/edit#slide=id.gb478361595_0_10),[video](https://www.youtube.com/watch?v=u0IUQvG-_xI)) by [Harshal Sheth](https://www.linkedin.com/in/hsheth2/) - 15 mins +- General Q&A from sign up sheet, slack, and participants - 15 mins +- Closing remarks - 5 mins + +### Jan 2021 + +[Full Recording](https://youtu.be/r862MZTLAJ0) + +[Slide-deck](https://docs.google.com/presentation/d/e/2PACX-1vQ2B0iHb2uwege1wlkXHOgQer0myOMEE5EGnzRjyqw0xxS5SaAc8VMZ_1XVOHuTZCJYzZZW4i9YnzSN/pub?start=false&loop=false&delayms=3000) + +**Agenda** + +- Announcements - 2 mins +- Community Updates ([video](https://youtu.be/r862MZTLAJ0?t=99)) - 10 mins +- Use-Case: DataHub at Viasat ([slides](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/ViasatMetadataJourney.pdf),[video](https://youtu.be/2SrDAJnzkjE)) by [Anna Kepler](https://www.linkedin.com/in/akepler) - 15 mins- Tech Deep Dive: GraphQL + React RFCs readout and discussion ([slides](https://docs.google.com/presentation/d/e/2PACX-1vRtnINnpi6PvFw7-5iW8PSQoT9Kdf1O_0YW7QAr1_mSdJMNftYFTVCjKL-e3fpe8t6IGkha8UpdmoOI/pub?start=false&loop=false&delayms=3000) ,[video](https://www.youtube.com/watch?v=PrBaFrb7pqA)) by [John Joyce](https://www.linkedin.com/in/john-joyce-759883aa) and [Arun Vasudevan](https://www.linkedin.com/in/arun-vasudevan-55117368/) - 15 mins +- General Q&A from sign up sheet, slack, and participants - 15 mins +- Closing remarks - 3 mins +- General Q&A from sign up sheet, slack, and participants - 15 mins +- Closing remarks - 5 minutes + +### Dec 2020 + +[Recording](https://linkedin.zoom.us/rec/share/8E7-lFnCi_kQ8OvXR9kW6fn-AjvV8VlqOO2xYR8b5Y_UeWI_ODcKFlxlHqYgBP7j.S-c8C1YMrz7d3Mjq) + +**Agenda** + +- Quick intro - 5 mins +- [Why did Grofers choose DataHub for their data catalog?](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Datahub_at_Grofers.pdf) by [Shubham Gupta](https://www.linkedin.com/in/shubhamg931/) - 15 minutes +- [DataHub UI development - Part 2](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Town_Hall_Presentation_-_12-2020_-_UI_Development_Part_2.pdf) by [Charlie Tran](https://www.linkedin.com/in/charlie-tran/) (LinkedIn) - 20 minutes +- General Q&A from sign up sheet, slack, and participants - 15 mins +- Closing remarks - 5 minutes + +### Nov 2020 + +[Recording](https://linkedin.zoom.us/rec/share/0yvjZ2fOzVmD8aaDo3lC59fXivmYG3EnF0U9tMVgKs827595usvSoIhtFUPjZCsU.b915nLRkw6iQlnoD) + +**Agenda** + +- Quick intro - 5 mins +- [Lightning talk on Metadata use-cases at LinkedIn](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Metadata_Use-Cases_at_LinkedIn_-_Lightning_Talk.pdf) by [Shirshanka Das](https://www.linkedin.com/in/shirshankadas/) (LinkedIn) - 5 mins +- [Strongly Consistent Secondary Index (SCSI) in GMA](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Datahub_-_Strongly_Consistent_Secondary_Indexing.pdf), an upcoming feature by [Jyoti Wadhwani](https://www.linkedin.com/in/jyotiwadhwani/) (LinkedIn) - 15 minutes +- [DataHub UI overview](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/DataHub-UIOverview.pdf) by [Ignacio Bona](https://www.linkedin.com/in/ignaciobona) (LinkedIn) - 20 minutes +- General Q&A from sign up sheet, slack, and participants - 10 mins +- Closing remarks - 5 minutes + +### Sep 2020 + +[Recording](https://linkedin.zoom.us/rec/share/uEQ2pRY0BHbVqk_sOTVRm05VXJ0xM_zKJ26yzfCBqNZItiBht__k_juCCahJ37QK.IKAU9qA_0qdURX4_) + +**Agenda** + +- Quick intro - 5 mins +- [Data Discoverability at SpotHero](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Data_Discoverability_at_SpotHero.pdf) by [Maggie Hays](https://www.linkedin.com/in/maggie-hays/) (SpotHero) - 20 mins +- [Designing the next generation of metadata events for scale](https://github.com/acryldata/static-assets-test/raw/master/imgs/demo/Designing_the_next_generation_of_metadata_events_for_scale.pdf) by [Chris Lee](https://www.linkedin.com/in/chrisleecmu/) (LinkedIn) - 15 mins +- General Q&A from sign up sheet, slack, and participants - 15 mins +- Closing remarks - 5 mins + +### Aug 2020 + +[Recording](https://linkedin.zoom.us/rec/share/vMBfcb31825IBZ3T71_wffM_GNv3T6a8hicf8_dcfzQlhfFxl5i_CPVKcmYaZA) + +**Agenda** + +- Quick intro - 5 mins +- [Data Governance look for a Digital Bank](https://www.slideshare.net/SheetalPratik/linkedinsaxobankdataworkbench) by [Sheetal Pratik](https://www.linkedin.com/in/sheetalpratik/) (Saxo Bank) - 20 mins +- Column level lineage for datasets demo by [Nagarjuna Kanamarlapudi](https://www.linkedin.com/in/nagarjunak/) (LinkedIn) - 15 mins +- General Q&A from sign up sheet and participants - 15 mins +- Closing remarks - 5 mins + +### Jul 2020 + +[Recording](https://bluejeans.com/s/wjnDRJevi5z/) + +**Agenda** + +- Quick intro - 5 mins +- Showcasing new entities onboarded to internal LinkedIn DataHub (Data Concepts, Schemas) by [Nagarjuna Kanamarlapudi](https://www.linkedin.com/in/nagarjunak) (LinkedIn) - 15 mins +- Showcasing new Lineage UI in internal LinkedIn DataHub By [Ignacio Bona](https://www.linkedin.com/in/ignaciobona) (LinkedIn) - 10 mins +- New [RFC Process](./rfc.md) by [John Plaisted](https://www.linkedin.com/in/john-plaisted-49a00a78/) (LinkedIn) - 2 mins +- Answering questions from the signup sheet - 13 mins +- Questions from the participants - 10 mins +- Closing remarks - 5 mins + +### June 2020 + +[Recording](https://bluejeans.com/s/yILyR/) + +**Agenda** + +- Quick intro - 5 mins +- Onboarding Data Process entity by [Liangjun Jiang](https://github.com/liangjun-jiang) (Expedia) - 15 mins +- How to onboard a new relationship to metadata graph by [Kerem Sahin](https://github.com/keremsahin1) (Linkedin) - 15 mins +- Answering questions from the signup sheet - 15 mins +- Questions from the participants - 10 mins +- Closing remarks - 5 mins + +### May 2020 + +[Recording](https://bluejeans.com/s/GCAzY) + +**Agenda** + +- Quick intro - 5 mins +- How to add a new aspect/feature for an existing entity in UI by [Charlie Tran](https://www.linkedin.com/in/charlie-tran/) (LinkedIn) - 10 mins +- How to search over a new field by [Jyoti Wadhwani](https://www.linkedin.com/in/jyotiwadhwani/) (LinkedIn) - 10 mins +- Answering questions from the signup sheet - 15 mins +- Questions from the participants - 10 mins +- Closing remarks - 5 mins + +### Apr 2020 (17th) + +[Recording](https://bluejeans.com/s/eYRD4) + +**Agenda** + +- Quick intro - 5 mins +- [DataHub Journey with Expedia Group](https://www.youtube.com/watch?v=ajcRdB22s5o&ab_channel=ArunVasudevan) by [Arun Vasudevan](https://www.linkedin.com/in/arun-vasudevan-55117368/) (Expedia) - 10 mins +- Deploying DataHub using Nix by [Larry Luo](https://github.com/clojurians-org) (Shanghai HuaRui Bank) - 10 mins +- Answering questions from the signup sheet - 15 mins +- Questions from the participants - 10 mins +- Closing remarks - 5 mins + +### Apr 2020 (3rd) + +[Recording](https://bluejeans.com/s/vzYpa) + +[Q&A](https://docs.google.com/document/d/1ChF9jiJWv9wj3HLLkFYRg7NSYg8Kb0PT7COd7Hf9Zpk/edit?usp=sharing) + +- **Agenda** + - Quick intro - 5 mins + - Creating Helm charts for deploying DataHub on Kubernetes by [Bharat Akkinepalli](https://www.linkedin.com/in/bharat-akkinepalli-ba0b7223/) (ThoughtWorks) - 10 mins + - How to onboard a new metadata aspect by [Mars Lan](https://www.linkedin.com/in/marslan) (LinkedIn) - 10 mins + - Answering questions from the signup sheet - 15 mins + - Questions from the participants - 10 mins + - Closing remarks - 5 mins + +### Mar 2020 (20th) + +[Recording](https://bluejeans.com/s/FSKEF) + +[Q&A](https://docs.google.com/document/d/1vQ6tAGXsVafnPIcZv1GSYgnTJJXFOACa1aWzOQjiGHI/edit) + +**Agenda** + +- Quick intro - 5 mins +- Internal DataHub demo - 10 mins +- What's coming up next for DataHub (what roadmap items we are working on) - 10 mins +- Answering questions from the signup sheet - 15 mins +- Questions from the participants - 10 mins +- Closing remarks - 5 mins + +### Mar 2020 (6th) + +[Recording](https://bluejeans.com/s/vULMG) + +[Q&A](https://docs.google.com/document/d/1N_VGqlH9CD-54LBsVlpcK2Cf2Mgmuzq79EvN9qgBqtQ/edit) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/townhalls.md b/docs-archive/versioned_docs/version-1.5.0/docs/townhalls.md new file mode 100644 index 00000000..02b6c7eb --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/townhalls.md @@ -0,0 +1,60 @@ +--- +title: DataHub Town Halls +sidebar_label: Town Halls +slug: /townhalls +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/townhalls.md' +--- +# DataHub Town Halls + +Join us for community gatherings where we discuss roadmap updates, demo new features, and hear from DataHub users around the world. + +## 🗓️ Next Town Hall + +[Register for the February Town Hall](https://luma.com/zp3h4ex8) + +--- + +## 📺 Recent Town Halls + +**January 2026: Powering AI Agents with DataHub Context** + +- 📺 [Watch recording](https://www.youtube.com/playlist?list=PLdCtLs64vZvHTXGqybmOfyxXbGDn2Reb9) + +**December 2025: Building AI Agents You'd Trust in Production** + +- 📺 [Watch recording](https://www.youtube.com/watch?v=oWiT7tkSZgo) + +**[→ View all Town Hall recordings](https://www.youtube.com/playlist?list=PLdCtLs64vZvHTXGqybmOfyxXbGDn2Reb9)** + +--- + +## 💡 What We Typically Cover + +- **Product Roadmap Updates** - What's coming next in DataHub +- **Feature Demos** - Live walkthroughs of new capabilities +- **Community Case Studies** - How teams are using DataHub in production +- **Open Q&A** - Ask the team anything + +## 🙋 Want to Participate? + +- **Ask Questions:** Submit topics via [Slack #townhall channel](https://datahub.com/slack) +- **Share Your Story:** Using DataHub at your company? We'd love to feature you - reach out in Slack +- **Suggest Topics:** Drop ideas in the #townhall channel + +--- + +## 📅 Upcoming Community Events + +Check out our DataHub Community Calendar to RSVP to office hours and upcoming events! + +

+ +

diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/build.md b/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/build.md new file mode 100644 index 00000000..ed1897ee --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/build.md @@ -0,0 +1,48 @@ +--- +title: Build Debugging Guide +slug: /troubleshooting/build +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/troubleshooting/build.md +--- +# Build Debugging Guide + +For when [Local Development](/docs/developers.md) did not work out smoothly. + +## Getting `Unsupported class file major version 57` + +You're probably using a Java version that's too new for gradle. Run the following command to check your Java version + +``` +java --version +``` + +While it may be possible to build and run DataHub using newer versions of Java, we currently only support [Java 17](https://openjdk.org/projects/jdk/17/) (aka Java 17). + +## Getting `cannot find symbol` error for `javax.annotation.Generated` + +Similar to the previous issue, please use Java 17 to build the project. +You can install multiple version of Java on a single machine and switch between them using the `JAVA_HOME` environment variable. See [this document](https://docs.oracle.com/cd/E21454_01/html/821-2531/inst_jdk_javahome_t.html) for more details. + +## `:metadata-models:generateDataTemplate` task fails with `java.nio.file.InvalidPathException: Illegal char <:> at index XX` or `Caused by: java.lang.IllegalArgumentException: 'other' has different root` error + +This is a [known issue](https://github.com/linkedin/rest.li/issues/287) when building the project on Windows due a bug in the Pegasus plugin. Please refer to [Windows Compatibility](/docs/developers.md#windows-compatibility). + +## Various errors related to `generateDataTemplate` or other `generate` tasks + +As we generate quite a few files from the models, it is possible that old generated files may conflict with new model changes. When this happens, a simple `./gradlew clean` should resolve the issue. + +## `Execution failed for task ':metadata-service:restli-servlet-impl:checkRestModel'` + +This generally means that an [incompatible change](https://linkedin.github.io/rest.li/modeling/compatibility_check) was introduced to the rest.li API in GMS. You'll need to rebuild the snapshots/IDL by running the following command once + +``` +./gradlew :metadata-service:restli-servlet-impl:build -Prest.model.compatibility=ignore +``` + +## `java.io.IOException: No space left on device` + +This means you're running out of space on your disk to build. Please free up some space or try a different disk. + +## `Build failed` for task `./gradlew :datahub-frontend:dist -x yarnTest -x yarnLint` + +This could mean that you need to update your [Yarn](https://yarnpkg.com/getting-started/install) version diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/general.md b/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/general.md new file mode 100644 index 00000000..ffe767be --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/general.md @@ -0,0 +1,18 @@ +--- +title: General Debugging Guide +slug: /troubleshooting/general +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/troubleshooting/general.md +--- +# General Debugging Guide + +## Logo for my platform is not appearing on the Home Page or search results + +Please see if either of these guides help you + +- [Adding a custom Dataset Data Platform](../how/add-custom-data-platform.md) +- [DataHub CLI put platform command](../cli.md#put-platform) + +## How do I add dataset freshness indicator for datasets? + +You can emit an [operation aspect](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Operation.pdl) for the same. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/quickstart.md b/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/quickstart.md new file mode 100644 index 00000000..bc0ac16b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/troubleshooting/quickstart.md @@ -0,0 +1,313 @@ +--- +title: Quickstart Debugging Guide +slug: /troubleshooting/quickstart +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/troubleshooting/quickstart.md +--- +# Quickstart Debugging Guide + +For when [Quickstart](/docs/quickstart.md) did not work out smoothly. + +## Common Problems + +
+Command not found: datahub + + +If running the datahub cli produces "command not found" errors inside your terminal, your system may be defaulting to an +older version of Python. Try prefixing your `datahub` commands with `python3 -m`: + +```bash +python3 -m datahub docker quickstart +``` + +Another possibility is that your system PATH does not include pip's `$HOME/.local/bin` directory. On linux, you can add this to your `~/.bashrc`: + +```bash +if [ -d "$HOME/.local/bin" ] ; then + PATH="$HOME/.local/bin:$PATH" +fi +``` + +
+ +
+ +Port Conflicts + + +By default the quickstart deploy will require the following ports to be free on your local machine: + +- 3306 for MySQL +- 9200 for Elasticsearch +- 9092 for the Kafka broker +- 8081 for Schema Registry +- 2181 for ZooKeeper +- 9002 for the DataHub Web Application (datahub-frontend) +- 8080 for the DataHub Metadata Service (datahub-gms) + +In case the default ports conflict with software you are already running on your machine, you can override these ports by passing additional flags to the `datahub docker quickstart` command. +e.g. To override the MySQL port with 53306 (instead of the default 3306), you can say: `datahub docker quickstart --mysql-port 53306`. Use `datahub docker quickstart --help` to see all the supported options. +For the metadata service container (datahub-gms), you need to use an environment variable, `DATAHUB_MAPPED_GMS_PORT`. So for instance to use the port 58080, you would say `DATAHUB_MAPPED_GMS_PORT=58080 datahub docker quickstart` + +
+ +
+ +no matching manifest for linux/arm64/v8 in the manifest list entries + +On Mac computers with Apple Silicon (M1, M2 etc.), you might see an error like `no matching manifest for linux/arm64/v8 in the manifest list entries`, this typically means that the datahub cli was not able to detect that you are running it on Apple Silicon. To resolve this issue, override the default architecture detection by issuing `datahub docker quickstart --arch m1` + +
+
+ +Miscellaneous Docker issues + +There can be misc issues with Docker, like conflicting containers and dangling volumes, that can often be resolved by +pruning your Docker state with the following command. Note that this command removes all unused containers, networks, +images (both dangling and unreferenced), and optionally, volumes. + +``` +docker system prune +``` + +
+ +## How can I confirm if all Docker containers are running as expected after a quickstart? + +If you set up the `datahub` CLI tool (see [here](../../metadata-ingestion/README.md)), you can use the built-in check utility: + +```shell +datahub docker check +``` + +You can list all Docker containers in your local by running `docker container ls`. You should expect to see a log similar to the below: + +``` +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +979830a342ce acryldata/datahub-mce-consumer:latest "bash -c 'while ping…" 10 hours ago Up 10 hours datahub-mce-consumer +3abfc72e205d acryldata/datahub-frontend-react:latest "datahub-frontend…" 10 hours ago Up 10 hours 0.0.0.0:9002->9002/tcp datahub-frontend +50b2308a8efd acryldata/datahub-mae-consumer:latest "bash -c 'while ping…" 10 hours ago Up 10 hours datahub-mae-consumer +4d6b03d77113 acryldata/datahub-gms:latest "bash -c 'dockerize …" 10 hours ago Up 10 hours 0.0.0.0:8080->8080/tcp datahub-gms +c267c287a235 landoop/schema-registry-ui:latest "/run.sh" 10 hours ago Up 10 hours 0.0.0.0:8000->8000/tcp schema-registry-ui +4b38899cc29a confluentinc/cp-schema-registry:5.2.1 "/etc/confluent/dock…" 10 hours ago Up 10 hours 0.0.0.0:8081->8081/tcp schema-registry +37c29781a263 confluentinc/cp-kafka:5.2.1 "/etc/confluent/dock…" 10 hours ago Up 10 hours 0.0.0.0:9092->9092/tcp, 0.0.0.0:29092->29092/tcp broker +15440d99a510 docker.elastic.co/kibana/kibana:5.6.8 "/bin/bash /usr/loca…" 10 hours ago Up 10 hours 0.0.0.0:5601->5601/tcp kibana +943e60f9b4d0 neo4j:4.0.6 "/sbin/tini -g -- /d…" 10 hours ago Up 10 hours 0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp neo4j +6d79b6f02735 confluentinc/cp-zookeeper:5.2.1 "/etc/confluent/dock…" 10 hours ago Up 10 hours 2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp zookeeper +491d9f2b2e9e docker.elastic.co/elasticsearch/elasticsearch:5.6.8 "/bin/bash bin/es-do…" 10 hours ago Up 10 hours 0.0.0.0:9200->9200/tcp, 9300/tcp elasticsearch +ce14b9758eb3 mysql:8.2 +``` + +Also you can check individual Docker container logs by running `docker logs <>`. For `datahub-gms`, you should see a log similar to this at the end of the initialization: + +``` +2020-02-06 09:20:54.870:INFO:oejs.Server:main: Started @18807ms +``` + +For `datahub-frontend-react`, you should see a log similar to this at the end of the initialization: + +``` +09:20:22 [main] INFO play.core.server.AkkaHttpServer - Listening for HTTP on /0.0.0.0:9002 +``` + +## My elasticsearch or broker container exited with error or was stuck forever + +If you're seeing errors like below, chances are you didn't give enough resource to docker. Please make sure to allocate at least 8GB of RAM + 2GB swap space. + +``` +datahub-gms | 2020/04/03 14:34:26 Problem with request: Get http://elasticsearch:9200: dial tcp 172.19.0.5:9200: connect: connection refused. Sleeping 1s +broker | [2020-04-03 14:34:42,398] INFO Client session timed out, have not heard from server in 6874ms for sessionid 0x10000023fa60002, closing socket connection and attempting reconnect (org.apache.zookeeper.ClientCnxn) +schema-registry | [2020-04-03 14:34:48,518] WARN Client session timed out, have not heard from server in 20459ms for sessionid 0x10000023fa60007 (org.apache.zookeeper.ClientCnxn) +``` + +## How can I check if [MXE](../what/mxe.md) Kafka topics are created? + +You can use a utility like [kafkacat](https://github.com/edenhill/kafkacat) to list all topics. +You can run below command to see the Kafka topics created in your Kafka broker. + +```bash +kafkacat -L -b localhost:9092 +``` + +Confirm that `MetadataChangeEvent`, `MetadataAuditEvent`, `MetadataChangeProposal_v1` and `MetadataChangeLog_v1` topics exist besides the default ones. + +## How can I check if search indices are created in Elasticsearch? + +You can run below command to see the search indices created in your Elasticsearch. + +```bash +curl http://localhost:9200/_cat/indices +``` + +Confirm that `datasetindex_v2` & `corpuserindex_v2` indices exist besides the default ones. Example response as below: + +```bash +yellow open dataset_datasetprofileaspect_v1 HnfYZgyvS9uPebEQDjA1jg 1 1 0 0 208b 208b +yellow open datajobindex_v2 A561PfNsSFmSg1SiR0Y0qQ 1 1 2 9 34.1kb 34.1kb +yellow open mlmodelindex_v2 WRJpdj2zT4ePLSAuEvFlyQ 1 1 1 12 24.2kb 24.2kb +yellow open dataflowindex_v2 FusYIc1VQE-5NaF12uS8dA 1 1 1 3 23.3kb 23.3kb +yellow open mlmodelgroupindex_v2 QOzAaVx7RJ2ovt-eC0hg1w 1 1 0 0 208b 208b +yellow open datahubpolicyindex_v2 luXfXRlSRoS2-S_tvfLjHA 1 1 0 0 208b 208b +yellow open corpuserindex_v2 gbNXtnIJTzqh3vHSZS0Fwg 1 1 2 2 18.4kb 18.4kb +yellow open dataprocessindex_v2 9fL_4iCNTLyFv8MkDc6nIg 1 1 0 0 208b 208b +yellow open chartindex_v2 wYKlG5ylQe2dVKHOaswTww 1 1 2 7 29.4kb 29.4kb +yellow open tagindex_v2 GBQSZEvuRy62kpnh2cu1-w 1 1 2 2 19.7kb 19.7kb +yellow open mlmodeldeploymentindex_v2 UWA2ltxrSDyev7Tmu5OLmQ 1 1 0 0 208b 208b +yellow open dashboardindex_v2 lUjGAVkRRbuwz2NOvMWfMg 1 1 1 0 9.4kb 9.4kb +yellow open .ds-datahub_usage_event-000001 Q6NZEv1UQ4asNHYRywxy3A 1 1 36 0 54.8kb 54.8kb +yellow open datasetindex_v2 bWE3mN7IRy2Uj0QzeCt1KQ 1 1 7 47 93.7kb 93.7kb +yellow open mlfeatureindex_v2 fvjML5xoQpy8oxPIwltm8A 1 1 20 39 59.3kb 59.3kb +yellow open dataplatformindex_v2 GihumZfvRo27vt9yRpoE_w 1 1 0 0 208b 208b +yellow open glossarynodeindex_v2 ABKeekWTQ2urPWfGDsS4NQ 1 1 1 1 18.1kb 18.1kb +yellow open graph_service_v1 k6q7xV8OTIaRIkCjrzdufA 1 1 116 25 77.1kb 77.1kb +yellow open system_metadata_service_v1 9-FKAqp7TY2hs3RQuAtVMw 1 1 303 0 55.9kb 55.9kb +yellow open schemafieldindex_v2 Mi_lqA-yQnKWSleKEXSWeg 1 1 0 0 208b 208b +yellow open mlfeaturetableindex_v2 pk98zrSOQhGr5gPYUQwvvQ 1 1 5 14 36.4kb 36.4kb +yellow open glossarytermindex_v2 NIyi3WWiT0SZr8PtECo0xQ 1 1 3 8 23.1kb 23.1kb +yellow open mlprimarykeyindex_v2 R1WFxD9sQiapIZcXnDtqMA 1 1 7 6 35.5kb 35.5kb +yellow open corpgroupindex_v2 AYxVtFAEQ02BsJdahYYvlA 1 1 2 1 13.3kb 13.3kb +yellow open dataset_datasetusagestatisticsaspect_v1 WqPpDCKZRLaMIcYAAkS_1Q 1 1 0 0 208b 208b +``` + +## How can I check if data has been loaded into MySQL properly? + +Once the mysql container is up and running, you should be able to connect to it directly on `localhost:3306` using tools such as [MySQL Workbench](https://www.mysql.com/products/workbench/). You can also run the following command to invoke [MySQL Command-Line Client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html) inside the mysql container. + +``` +docker exec -it mysql /usr/bin/mysql datahub --user=datahub --password=datahub +``` + +Inspect the content of `metadata_aspect_v2` table, which contains the ingested aspects for all entities. + +## Getting error while starting Docker containers + +There can be different reasons why a container fails during initialization. Below are the most common reasons: + +### `bind: address already in use` + +This error means that the network port (which is supposed to be used by the failed container) is already in use on your system. You need to find and kill the process which is using this specific port before starting the corresponding Docker container. If you don't want to kill the process which is using that port, another option is to change the port number for the Docker container. You need to find and change the [ports](https://docs.docker.com/compose/compose-file/#ports) parameter for the specific Docker container in the `docker-compose.yml` configuration file. + +``` +Example : On macOS + +ERROR: for mysql Cannot start service mysql: driver failed programming external connectivity on endpoint mysql (5abc99513affe527299514cea433503c6ead9e2423eeb09f127f87e2045db2ca): Error starting userland proxy: listen tcp 0.0.0.0:3306: bind: address already in use + + 1) sudo lsof -i :3306 + 2) kill -15 +``` + +### `OCI runtime create failed` + +If you see an error message like below, please make sure to git update your local repo to HEAD. + +``` +ERROR: for datahub-mae-consumer Cannot start service datahub-mae-consumer: OCI runtime create failed: container_linux.go:349: starting container process caused "exec: \"bash\": executable file not found in $PATH": unknown +``` + +### `failed to register layer: devmapper: Unknown device` + +This most means that you're out of disk space (see [#1879](https://github.com/datahub-project/datahub/issues/1879)). + +### `ERROR: for kafka-rest-proxy Get https://registry-1.docker.io/v2/confluentinc/cp-kafka-rest/manifests/5.4.0: EOF` + +This is most likely a transient issue with [Docker Registry](https://docs.docker.com/registry/). Retry again later. + +## toomanyrequests: too many failed login attempts for username or IP address + +Try the following + +```bash +rm ~/.docker/config.json +docker login +``` + +More discussions on the same issue https://github.com/docker/hub-feedback/issues/1250 + +## Seeing `Table 'datahub.metadata_aspect' doesn't exist` error when logging in + +This means the database wasn't properly initialized as part of the quickstart processs (see [#1816](https://github.com/datahub-project/datahub/issues/1816)). Please run the following command to manually initialize it. + +``` +docker exec -i mysql sh -c 'exec mysql datahub -udatahub -pdatahub' < docker/mysql/init.sql +``` + +## I've messed up my docker setup. How do I start from scratch? + +Run the following script to remove all the containers and volumes created during the quickstart tutorial. Note that you'll also lose all the data as a result. + +``` +datahub docker nuke +``` + +## I'm seeing exceptions in DataHub GMS container like "Caused by: java.lang.IllegalStateException: Duplicate key com.linkedin.metadata.entity.ebean.EbeanAspectV2@dd26e011". What do I do? + +This is related to a SQL column collation issue. The default collation we previously used (prior to Oct 26, 2021) for URN fields was case-insensitive (utf8mb4_unicode_ci). We've recently moved +to deploying with a case-sensitive collation (utf8mb4_bin) by default. In order to update a deployment that was started before Oct 26, 2021 (v0.8.16 and below) to have the new collation, you must run this command against your SQL DB directly: + +``` +ALTER TABLE metadata_aspect_v2 CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_bin; +``` + +## I've modified the default user.props file to include a custom username and password, but I don't see the new user(s) inside the Users & Groups tab. Why not? + +Currently, `user.props` is a file used by the JAAS PropertyFileLoginModule solely for the purpose of **Authentication**. The file is not used as an source from which to +ingest additional metadata about the user. For that, you'll need to ingest some custom information about your new user using the Rest.li APIs or the [Metadata File ingestion source](../generated/ingestion/sources/metadata-file.md). + +For an example of a file that ingests user information, check out [single_mce.json](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/mce_files/single_mce.json), which ingests a single user object into DataHub. Notice that the "urn" field provided +will need to align with the custom username you've provided in user.props file. For example, if your user.props file contains: + +``` +my-custom-user:my-custom-password +``` + +You'll need to ingest some metadata of the following form to see it inside the DataHub UI: + +``` +{ + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:my-custom-user", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": { + "string": "The name of the custom user" + }, + "email": "my-custom-user-email@example.io", + "title": { + "string": "Engineer" + }, + "managerUrn": null, + "departmentId": null, + "departmentName": null, + "firstName": null, + "lastName": null, + "fullName": { + "string": "My Custom User" + }, + "countryCode": null + } + } + ] + } + }, + "proposedDelta": null +} +``` + +## I've configured OIDC, but I cannot login. I get continuously redirected. What do I do? + +Sorry to hear that! + +This phenomena may be due to the size of a Cookie DataHub uses to authenticate its users. If it's too large ( > 4096), then you'll see this behavior. The cookie embeds an encoded version of the information returned by your OIDC Identity Provider - if they return a lot of information, this can be the root cause. + +One solution is to use Play Cache to persist this session information for a user. This means the attributes about the user (and their session info) will be stored in an in-memory store in the `datahub-frontend` service, instead of a browser-side cookie. + +To configure the Play Cache session store, you can set the env variable "PAC4J_SESSIONSTORE_PROVIDER" as "PlayCacheSessionStore" for the `datahub-frontend` container. + +Do note that there are downsides to using the Play Cache. Specifically, it will make `datahub-frontend` a stateful server. If you have multiple instances of `datahub-frontend` deployed, you'll need to ensure that the same user is deterministically routed to the same service container (since the sessions are stored in memory). If you're using a single instance of `datahub-frontend` (the default), then things should "just work". + +For more details, please refer to https://github.com/datahub-project/datahub/pull/5114 diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/ui-ingestion.md b/docs-archive/versioned_docs/version-1.5.0/docs/ui-ingestion.md new file mode 100644 index 00000000..641d9169 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/ui-ingestion.md @@ -0,0 +1,379 @@ +--- +title: Metadata Ingestion +slug: /ui-ingestion +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/ui-ingestion.md' +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; + +# Metadata Ingestion + + + +DataHub helps you discover and understand your organization's data by automatically collecting information about your data sources. This process is called **metadata ingestion**, allowing DataHub to automatically pull in: + +- **Table and column names** from your databases +- **Asset Lineage** showing how information flows between systems +- **Usage statistics** revealing which datasets are most popular +- **Data quality information** including freshness and completeness +- **Business context** like ownership and documentation + +This makes it simple to connect to popular platforms like Snowflake, BigQuery, dbt, and more, schedule automatic updates, and manage credentials securely. + +## Prerequisites and Permissions + +To manage metadata ingestion in DataHub, you need appropriate permissions. + +:::note Ask DataHub for Ingestion (Public Beta - DataHub Cloud) +**Ask DataHub** (Public Beta) is available within the ingestion creation and troubleshooting workflow for DataHub Cloud deployments. Get AI-powered assistance with configuration, filtering, troubleshooting, and best practices—right in your workflow. +::: + +### Option 1: Admin-Level Access + +Users can be granted the following privileges for full administrative access to all ingestion sources: + +- **`Manage Metadata Ingestion`** - Provides complete access to create, edit, run, and delete all ingestion sources +- **`Manage Secrets`** - Allows creation and management of encrypted credentials used in ingestion configurations + +These privileges can be granted in two ways: + +1. **Admin Role Assignment** - Users assigned to the **Admin Role** receive these privileges by default +2. **Custom Policy with Platform Privileges** - Create a [Custom Policy](authorization/policies.md) that grants the `Manage Metadata Ingestion` and `Manage Secrets` platform privileges to specific users or groups + +

+ +

+ +### Option 2: Resource-Specific Policies + +For more granular control, administrators can create [Custom Policies](authorization/policies.md) that apply specifically to **Ingestion Sources**, allowing different users to have different levels of access: + +- **View** - View ingestion source configurations and run history +- **Edit** - Modify ingestion source configurations +- **Delete** - Remove ingestion sources +- **Execute** - Run ingestion sources on-demand + +**Prerequisites:** + +- **DataHub Core**: Enable the `VIEW_INGESTION_SOURCE_PRIVILEGES_ENABLED` feature flag +- **DataHub Cloud**: Work with your customer success team to get the feature enabled + +:::caution +**Important**: Once this feature flag is enabled, any policies that apply to "All" resource types will now include Ingestion Sources, including the default read-only policies. This will make the Ingestion tab visible and potentially actionable depending on the applied privileges. Implement this with care if you have view-only policies that should not expose the Data Sources page. +::: + +## Creating an Ingestion Source + +Once you have the appropriate privileges, navigate to the **Ingestion** tab in DataHub. + +

+ +

+ +On this page, you'll see a list of active **Ingestion Sources**. An Ingestion Source represents a configured connection to an external data system from which DataHub extracts metadata. + +If you're just getting started, you won't have any sources configured. The following sections will guide you through creating your first ingestion source. + +### Step 1: Select a Data Source + +Begin by clicking **+ Create source** to start the ingestion source creation process. + +

+ +

+ +Next, select the type of data source you want to connect. DataHub provides pre-built templates for popular platforms including: + +- **Data Warehouses**: Snowflake, BigQuery, Redshift, Databricks +- **Databases**: MySQL, PostgreSQL, SQL Server, Oracle +- **Business Intelligence**: Looker, Tableau, PowerBI +- **Streaming**: Kafka, Pulsar +- **And many more...** + +

+ +

+ +Select the template that matches your data source. If your specific platform isn't listed, you can choose **Custom** to configure a source manually, though this requires more technical knowledge. + +### Step 2: Configure Connection Details + +After selecting your data source template, you'll configure how DataHub connects to and extracts metadata from your source. + +

+ +

+ +_Ask DataHub (Public Beta - Cloud only) provides contextual assistance throughout the ingestion configuration process_ + +**Name and Owners**: First, provide a descriptive name for your ingestion source that will help you and your team identify it later. You can also assign **Users** and/or **Groups** as owners of this ingestion source. By default, you (the creator) will be assigned as an owner, but you can add additional owners or change this at any time after creation. + +**Connection Information**: Next, you'll configure the connection details using a user-friendly form. The exact fields will vary depending on your chosen platform, but typically include: + +- Host/server address and port +- Database or project names +- Authentication credentials + +**Asset Filters**: Configure what metadata to extract: + +- Which databases, schemas, or tables to include +- Filtering options to exclude certain data + +**Ingestion Settings**: Configure ingestion behavior including profiling, stale metadata handling, and other operational settings. The defaults represent best practices for most use cases. + +:::note Ask DataHub for Configuration Help (Public Beta) +**Ask DataHub** (Public Beta) can help you understand the behavior and options of each configuration setting. Get tailored recommendations for your data source and use case. +::: + +

+ +

+ +_Ask DataHub (Public Beta - Cloud only) helps you understand configuration options and provides tailored recommendations for your data source_ + +#### Managing Sensitive Information with Secrets + +For production environments, sensitive information like passwords and API keys should be stored securely using DataHub's **Secrets** functionality. + +To create a secret: + +1. Navigate to the **Secrets** tab in the Ingestion interface +2. Click **Create new secret** +3. Provide a descriptive name (e.g., `BIGQUERY_PRIVATE_KEY`) +4. Enter the sensitive value +5. Optionally add a description +6. Click **Create** + +Once created, secrets can be referenced in your ingestion configuration forms using the dropdown menus provided for credential fields. + +:::caution Security Note +Users with the `Manage Secrets` privilege can retrieve plaintext secret values through DataHub's GraphQL API. Ensure secrets are only accessible to trusted administrators. +::: + +#### Test Your Connection + +Before proceeding, it's important to verify that DataHub can successfully connect to your data source. Most ingestion source forms include a **Test Connection** button that validates: + +- Network connectivity to your data source +- Authentication credentials +- Required permissions for metadata extraction + +

+ Test BigQuery connection +

+ +If the connection test fails, review your configuration and ensure that: + +- Network access is available between DataHub and your data source +- Credentials are correct and have sufficient permissions +- Any firewall rules allow the connection + +#### Advanced Settings + +For users who need additional control, DataHub provides advanced configuration options accessible in the Advanced Settings section: + +- **CLI Version:** Specify a particular version of the DataHub CLI for ingestion execution +- **Environment Variables:** Set custom environment variables for the ingestion process +- **Executor ID:** Configure remote execution if needed +- **Debug Mode:** Enable detailed logging for troubleshooting + +### Step 3: Sync Schedule + +Configure how often DataHub should sync metadata from your source. You can enable or disable scheduled execution using the toggle (recommended: enabled). This ensures your metadata stays up-to-date without manual intervention. + +

+ +

+ +If you prefer to run ingestion manually or on an ad-hoc basis, you can skip the scheduling step entirely. + +### Step 4: Review and Save + +Review your configuration to ensure all settings are correct. When you're ready, you have two options: + +- **Save**: Save the ingestion source configuration without executing it immediately +- **Save and Run**: Save and immediately execute your first ingestion run + +Once you're happy with your configurations, click your preferred save option to finalize your source. + +## Running and Monitoring Ingestion + +### Executing an Ingestion Source + +Once you've created your Ingestion Source, you can run it by clicking the 'Play' button. Shortly after, you should see the 'Last Status' column of the ingestion source change to `Running`, indicating that DataHub has successfully queued the ingestion job. + +

+ +

+ +When ingestion completes successfully, the status will show as `Success` in green. + +

+ +

+ +### Viewing Run History + +The **Run History** tab shows you a complete history of all your ingestion runs. Here you can: + +- **See all runs**: View every ingestion execution across all your sources +- **Check recent activity**: Runs are listed with the most recent at the top +- **Filter by source**: Use the dropdown to see runs from a specific ingestion source +- **Access from Sources tab**: Click on any source's **Last Run** status or select **View Run History** from the source menu + +This makes it easy to track your ingestion performance and troubleshoot any issues over time. + +

+ +

+ +### Viewing Ingestion Results + +After successful ingestion, you can view detailed information about what was extracted: + +1. Click the **Success** status button on a completed ingestion run +2. Select **View All** to see the list of ingested entities +3. Click on individual entities to validate the extracted metadata + +

+ ingestion_details_view_all +

+ +### Cancelling Running Ingestion + +If an ingestion run is taking too long or appears to be stuck, you can cancel it by clicking the 'Stop' button on the running job. + +

+ +

+ +This is useful when encountering issues like: + +- Network timeouts +- Ingestion source bugs +- Resource constraints + +## Troubleshooting Failed Ingestion + +### Common Failure Reasons + +When an ingestion run fails, you'll see a failed status indicator in your sources list. + +

+ +

+ +Common causes of ingestion failures include: + +1. **Configuration Errors**: Incorrect connection details, missing required fields, or invalid parameter values +2. **Authentication Issues**: Wrong credentials, expired tokens, or insufficient permissions +3. **Network Connectivity**: DNS resolution failures, firewall blocks, or unreachable data sources +4. **Secret Resolution Problems**: Referenced secrets that don't exist or have incorrect names +5. **Resource Constraints**: Memory limits, timeouts, or processing capacity issues + +### Viewing Detailed Logs + +To diagnose ingestion failures, click on a run history status (Failed, Aborted) value to view and download comprehensive ingestion run logs. + +

+ +

+ +The logs provide detailed information about: + +- Connection attempts and errors +- Authentication failures +- Data extraction progress +- Error messages and stack traces + +### Authentication for Secured DataHub Instances + +If your DataHub instance has [Metadata Service Authentication](authentication/introducing-metadata-service-authentication.md) enabled, you'll need to provide a Personal Access Token in your configuration. + +

+ +

+ +## Advanced Configuration with YAML + +While the UI-based forms handle most common ingestion scenarios, advanced users may need direct access to YAML configuration for: + +- Custom ingestion sources not available in the UI +- Complex transformation pipelines +- Advanced filtering and processing logic +- Integration with external systems + +For these advanced use cases, DataHub supports direct YAML recipe configuration. For detailed information about YAML-based configuration, including syntax and examples, see the [Recipe Overview Guide](metadata-ingestion/recipe_overview.md). + +### Deploying Recipes (CLI) + +You can deploy recipes using the CLI as mentioned in the [CLI documentation for uploading ingestion recipes](./cli.md#ingest-deploy). + +```bash +datahub ingest deploy --name "My Test Ingestion Source" --schedule "5 * * * *" --time-zone "UTC" -c recipe.yaml +``` + +### Deploying Recipes (GraphQL) + +Create ingestion sources using [DataHub's GraphQL API](./api/graphql/overview.md) using the **createIngestionSource** mutation endpoint. + +```graphql +mutation { + createIngestionSource( + input: { + name: "My Test Ingestion Source" + type: "mysql" + description: "My ingestion source description" + schedule: { interval: "*/5 * * * *", timezone: "UTC" } + config: { + recipe: "{\"source\":{\"type\":\"mysql\",\"config\":{\"include_tables\":true,\"database\":null,\"password\":\"${MYSQL_PASSWORD}\",\"profiling\":{\"enabled\":false},\"host_port\":null,\"include_views\":true,\"username\":\"${MYSQL_USERNAME}\"}},\"pipeline_name\":\"urn:li:dataHubIngestionSource:f38bd060-4ea8-459c-8f24-a773286a2927\"}" + version: "0.8.18" + executorId: "mytestexecutor" + } + } + ) +} +``` + +**Note**: Recipe must be double quotes escaped when using GraphQL + +## Frequently Asked Questions + +### Why does ingestion fail with 'Failed to Connect' errors in Docker environments? + +If you're running DataHub using `datahub docker quickstart` and experiencing connection failures, this may be due to network configuration issues. The ingestion executor might be unable to reach DataHub's backend services. + +Try updating your ingestion configuration to use the Docker internal DNS name: + +

+ +

+ +### What does a dash mark (-) status mean and how do I fix it? + +If your ingestion source shows a dash mark (-) status and never changes to 'Running', this could mean: + +1. **The source has never been triggered to run** - Try clicking the "Play" button to execute the source +2. **The DataHub actions executor is not running or healthy** (DataHub Core users only) + +If clicking "Play" doesn't resolve the issue, DataHub Core users should diagnose their actions container: + +1. Check container status with `docker ps` +2. View executor logs with `docker logs ` +3. Restart the actions container if necessary + +### When should I use CLI/YAML instead of UI ingestion? + +Consider using CLI-based ingestion when: + +- Your data sources aren't reachable from DataHub's network (use [remote executors](managed-datahub/operator-guide/setting-up-remote-ingestion-executor.md) for DataHub Cloud) +- You need custom ingestion logic not available in UI templates +- Your ingestion requires local file system access +- You want to distribute ingestion across multiple environments +- You need complex transformations or custom metadata processing + +## Additional Resources + +- **Demo Video**: [Watch a complete UI ingestion walkthrough](https://www.youtube.com/watch?v=EyMyLcaw_74) +- **Quick Start Guides**: Step-by-step setup instructions for popular data sources +- **Recipe Documentation**: [Comprehensive YAML configuration reference](metadata-ingestion/recipe_overview.md) +- **Integration Catalog**: [Browse all supported data sources and their features](/integrations) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what-is-datahub/datahub-concepts.md b/docs-archive/versioned_docs/version-1.5.0/docs/what-is-datahub/datahub-concepts.md new file mode 100644 index 00000000..b352dcab --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what-is-datahub/datahub-concepts.md @@ -0,0 +1,213 @@ +--- +title: DataHub Concepts +sidebar_label: Concepts +slug: /what-is-datahub/datahub-concepts +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/what-is-datahub/datahub-concepts.md +--- +# DataHub Concepts + +Explore key concepts of DataHub to take full advantage of its capabilities in managing your data. + +## General Concepts + +### URN (Uniform Resource Name) + +URN (Uniform Resource Name) is the chosen scheme of URI to uniquely define any resource in DataHub. It has the following form. + +``` +urn::: +``` + +Examples include `urn:li:dataset:(urn:li:dataPlatform:hive,fct_users_created,PROD)`, `urn:li:corpuser:jdoe`. + +> - [What is URN?](/docs/what/urn.md) + +### Policy + +Access policies in DataHub define who can do what to which resources. + +> - [Authorization: Policies Guide](/docs/authorization/policies.md) +> - [Developer Guides: DataHubPolicy](/docs/generated/metamodel/entities/dataHubPolicy.md) +> - [Feature Guides: About DataHub Access Policies](/docs/authorization/access-policies-guide.md) + +### Role + +DataHub provides the ability to use Roles to manage permissions. + +> - [Authorization: About DataHub Roles](/docs/authorization/roles.md) +> - [Developer Guides: DataHubRole](/docs/generated/metamodel/entities/dataHubRole.md) + +### Access Token (Personal Access Token) + +Personal Access Tokens, or PATs for short, allow users to represent themselves in code and programmatically use DataHub's APIs in deployments where security is a concern. +Used along-side with [authentication-enabled metadata service](/docs/authentication/introducing-metadata-service-authentication.md), PATs add a layer of protection to DataHub where only authorized users are able to perform actions in an automated way. + +> - [Authentication: About DataHub Personal Access Tokens](/docs/authentication/personal-access-tokens.md) +> - [Developer Guides: DataHubAccessToken](/docs/generated/metamodel/entities/dataHubAccessToken.md) + +### View + +Views allow you to save and share sets of filters for reuse when browsing DataHub. A view can either be public or personal. + +> - [DataHubView](/docs/generated/metamodel/entities/dataHubView.md) + +### Deprecation + +Deprecation is an aspect that indicates the deprecation status of an entity. Typically it is expressed as a Boolean value. + +> - [Deprecation of a dataset](/docs/generated/metamodel/entities/dataset.md#deprecation) + +### Ingestion Source + +Ingestion sources refer to the data systems that we are extracting metadata from. For example, we have sources for BigQuery, Looker, Tableau and many others. + +> - [Sources](/metadata-ingestion/README.md#sources) +> - [DataHub Integrations](/integrations) + +### Container + +A container of related data assets. + +> - [Developer Guides: Container](/docs/generated/metamodel/entities/container.md) + +### Data Platform + +Data Platforms are systems or tools that contain Datasets, Dashboards, Charts, and all other kinds of data assets modeled in the metadata graph. + +
+List of Data Platforms + + +- Azure Data Lake (Gen 1) +- Azure Data Lake (Gen 2) +- Airflow +- Ambry +- ClickHouse +- Couchbase +- External Source +- HDFS +- SAP HANA +- Hive +- Iceberg +- AWS S3 +- Kafka +- Kafka Connect +- Kusto +- Mode +- MongoDB +- MySQL +- MariaDB +- OpenAPI +- Oracle +- Pinot +- PostgreSQL +- Presto +- Tableau +- Vertica + +Reference : [data_platforms.yaml](https://github.com/datahub-project/datahub/blob/master/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml) + +
+ +> - [Developer Guides: Data Platform](/docs/generated/metamodel/entities/dataPlatform.md) + +### Dataset + +Datasets represent collections of data that are typically represented as Tables or Views in a database (e.g. BigQuery, Snowflake, Redshift etc.), Streams in a stream-processing environment (Kafka, Pulsar etc.), bundles of data found as Files or Folders in data lake systems (S3, ADLS, etc.). + +> - [Developer Guides: Dataset](/docs/generated/metamodel/entities/dataset.md) + +### Chart + +A single data vizualization derived from a Dataset. A single Chart can be a part of multiple Dashboards. Charts can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include a Superset or Looker Chart. + +> - [Developer Guides: Chart](/docs/generated/metamodel/entities/chart.md) + +### Dashboard + +A collection of Charts for visualization. Dashboards can have tags, owners, links, glossary terms, and descriptions attached to them. Examples include a Superset or Mode Dashboard. + +> - [Developer Guides: Dashboard](/docs/generated/metamodel/entities/dashboard.md) + +### Data Job + +An executable job that processes data assets, where "processing" implies consuming data, producing data, or both. +In orchestration systems, this is sometimes referred to as an individual "Task" within a "DAG". Examples include an Airflow Task. + +> - [Developer Guides: Data Job](/docs/generated/metamodel/entities/dataJob.md) + +### Data Flow + +An executable collection of Data Jobs with dependencies among them, or a DAG. +Sometimes referred to as a "Pipeline". Examples include an Airflow DAG. + +> - [Developer Guides: Data Flow](/docs/generated/metamodel/entities/dataFlow.md) + +### Glossary Term + +Shared vocabulary within the data ecosystem. + +> - [Feature Guides: Glossary](/docs/glossary/business-glossary.md) +> - [Developer Guides: GlossaryTerm](/docs/generated/metamodel/entities/glossaryTerm.md) + +### Glossary Term Group + +Glossary Term Group is similar to a folder, containing Terms and even other Term Groups to allow for a nested structure. + +> - [Feature Guides: Term & Term Group](/docs/glossary/business-glossary.md#terms--term-groups) + +### Tag + +Tags are informal, loosely controlled labels that help in search & discovery. They can be added to datasets, dataset schemas, or containers, for an easy way to label or categorize entities – without having to associate them to a broader business glossary or vocabulary. + +> - [Feature Guides: About DataHub Tags](/docs/tags.md) +> - [Developer Guides: Tags](/docs/generated/metamodel/entities/tag.md) + +### Domain + +Domains are curated, top-level folders or categories where related assets can be explicitly grouped. + +> - [Feature Guides: About DataHub Domains](/docs/domains.md) +> - [Developer Guides: Domain](/docs/generated/metamodel/entities/domain.md) + +### Owner + +Owner refers to the users or groups that has ownership rights over entities. For example, owner can be acceessed to dataset or a column or a dataset. + +> - [Getting Started : Adding Owners On Datasets/Columns](/docs/api/tutorials/owners.md#add-owners) + +### Users (CorpUser) + +CorpUser represents an identity of a person (or an account) in the enterprise. + +> - [Developer Guides: CorpUser](/docs/generated/metamodel/entities/corpuser.md) + +### Groups (CorpGroup) + +CorpGroup represents an identity of a group of users in the enterprise. + +> - [Developer Guides: CorpGroup](/docs/generated/metamodel/entities/corpGroup.md) + +## Metadata Model + +### Entity + +An entity is the primary node in the metadata graph. For example, an instance of a Dataset or a CorpUser is an Entity. + +> - [How does DataHub model metadata?](/docs/modeling/metadata-model.md) + +### Aspect + +An aspect is a collection of attributes that describes a particular facet of an entity. +Aspects can be shared across entities, for example "Ownership" is an aspect that is re-used across all the Entities that have owners. + +> - [What is a metadata aspect?](/docs/what/aspect.md) +> - [How does DataHub model metadata?](/docs/modeling/metadata-model.md) + +### Relationships + +A relationship represents a named edge between 2 entities. They are declared via foreign key attributes within Aspects along with a custom annotation (@Relationship). + +> - [What is a relationship?](/docs/what/relationship.md) +> - [How does DataHub model metadata?](/docs/modeling/metadata-model.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/aspect.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/aspect.md new file mode 100644 index 00000000..da15c6dc --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/aspect.md @@ -0,0 +1,51 @@ +--- +title: What is a metadata aspect? +slug: /what/aspect +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/aspect.md' +--- +# What is a metadata aspect? + +A metadata aspect is a structured document, or more precisely a `record` in [PDL](https://linkedin.github.io/rest.li/pdl_schema), +that represents a specific kind of metadata (e.g. ownership, schema, statistics, upstreams). +A metadata aspect on its own has no meaning (e.g. ownership for what?) and must be associated with a particular entity (e.g. ownership for PageViewEvent). +We purposely not to impose any model requirement on metadata aspects, as each aspect is expected to differ significantly. + +Metadata aspects are immutable by design, i.e. every change to a particular aspect results in a [new version](../advanced/aspect-versioning.md) created. +An optional retention policy can be applied such that X number of most recent versions will be retained after each update. +Setting X to 1 effectively means the metadata aspect is non-versioned. +It is also possible to apply the retention based on time, e.g. only keeps the metadata changes from the past 30 days. + +While a metadata aspect can be arbitrary complex document with multiple levels of nesting, it is sometimes desirable to break a monolithic aspect into smaller independent aspects. +This will provide the benefits of: + +1. **Faster read/write**: As metadata aspects are immutable, every "update" will lead to the writing the entire large aspect back to the underlying data store. + Likewise, readers will need to retrieve the entire aspect even if it’s only interested in a small part of it. +2. **Ability to independently version different aspects**: For example, one may like to get the change history of all the "ownership metadata" independent of the changes made to "schema metadata" for a dataset. +3. **Help with rest.li endpoint modeling**: While it’s not required to have 1:1 mapping between rest.li endpoints and metadata aspects, + it’d follow this pattern naturally, which means one will end up with smaller, more modular, endpoints instead of giant ones. + +Here’s an example metadata aspect. Note that the `admin` and `members` fields are implicitly conveying a relationship between `Group` entity & `User` entity. +It’s very natural to save such relationships as URNs in a metadata aspect. +The [relationship](relationship.md) section explains how this relationship can be explicitly extracted and modelled. + +``` +namespace com.linkedin.group + +import com.linkedin.common.AuditStamp +import com.linkedin.common.CorpuserUrn + +/** + * The membership metadata for a group + */ +record Membership { + + /** Audit stamp for the last change */ + auditStamp: AuditStamp + + /** Admin of the group */ + admin: CorpuserUrn + + /** Members of the group, ordered in descending importance */ + members: array[CorpuserUrn] +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/delta.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/delta.md new file mode 100644 index 00000000..cc0652e3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/delta.md @@ -0,0 +1,66 @@ +--- +title: What is a metadata delta? +slug: /what/delta +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/delta.md' +--- +# What is a metadata delta? + +Rest.li supports [partial update](https://linkedin.github.io/rest.li/user_guide/restli_server#partial_update) natively without needing explicitly defined models. +However, the granularity of update is always limited to each field in a PDL model. +There are cases where the update need to happen at an even finer grain, e.g. adding or removing items from an array. + +To this end, we’re proposing the following entity-specific metadata delta model that allows atomic partial updates at any desired granularity. +Note that: + +1. Just like metadata [aspects](aspect.md), we’re not imposing any limit on the partial update model, as long as it’s a valid PDL record. + This is because the rest.li endpoint will have the logic that performs the corresponding partial update based on the information in the model. + That said, it’s common to have fields that denote the list of items to be added or removed (e.g. `membersToAdd` & `membersToRemove` from below) +2. Similar to metadata [snapshots](snapshot.md), entity that supports metadata delta will add an entity-specific metadata delta + (e.g. `GroupDelta` from below) that unions all supported partial update models. +3. The entity-specific metadata delta is then added to the global `Delta` typeref, which is added as part of [Metadata Change Event](mxe.md#metadata-change-event-mce) and used during [Metadata Ingestion](../architecture/metadata-ingestion.md). + +``` +namespace com.linkedin.group + +import com.linkedin.common.CorpuserUrn + +/** + * A metadata delta for a specific group entity + */ +record MembershipPartialUpdate { + + /** List of members to be added to the group */ + membersToAdd: array[CorpuserUrn] + + /** List of members to be removed from the group */ + membersToRemove: array[CorpuserUrn] +} +``` + +``` +namespace com.linkedin.metadata.delta + +import com.linkedin.common.CorpGroupUrn +import com.linkedin.group.MembershipPartialUpdate + +/** + * A metadata delta for a specific group entity + */ +record GroupDelta { + + /** URN for the entity the metadata delta is associated with */ + urn: CorpGroupUrn + + /** The specific type of metadata delta to apply */ + delta: union[MembershipPartialUpdate] +} +``` + +``` +namespace com.linkedin.metadata.delta + +/** + * A union of all supported metadata delta types. + */ +typeref Delta = union[GroupDelta] +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/entity.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/entity.md new file mode 100644 index 00000000..ca427026 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/entity.md @@ -0,0 +1,9 @@ +--- +title: Entities +slug: /what/entity +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/entity.md' +--- +# Entities + +This page has been moved. Please refer to [The Metadata Model](../modeling/extending-the-metadata-model.md) for details on +the metadata model. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/gma.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/gma.md new file mode 100644 index 00000000..8df672e7 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/gma.md @@ -0,0 +1,18 @@ +--- +title: What is Generalized Metadata Architecture (GMA)? +slug: /what/gma +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/gma.md' +--- +# What is Generalized Metadata Architecture (GMA)? + +GMA is the backend infrastructure for DataHub. Unlike existing architectures, GMA leverages multiple storage technologies to efficiently service the four most commonly used query patterns + +- Document-oriented CRUD +- Complex queries (including joining distributed tables) +- Graph traversal +- Fulltext search and autocomplete + +GMA also embraces a distributed model, where each team owns, develops and operates their own metadata services (known as [GMS](gms.md)), while the metadata are automatically aggregated to populate the central [metadata graph](graph.md) and [search indexes](search-index.md). This is made possible by standardizing the metadata models and the access layer. + +We strongly believe that GMA can bring tremendous leverage to any team that has a need to store and access metadata. +Moreover, standardizing metadata modeling promotes a model-first approach to developments, resulting in a more concise, consistent, and highly connected metadata ecosystem that benefits all DataHub users. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/gms.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/gms.md new file mode 100644 index 00000000..aed13da1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/gms.md @@ -0,0 +1,10 @@ +--- +title: What is Generalized Metadata Service (GMS)? +slug: /what/gms +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/gms.md' +--- +# What is Generalized Metadata Service (GMS)? + +Metadata for [entities](entity.md) [onboarded](../modeling/metadata-model.md) to [GMA](gma.md) is served through microservices known as Generalized Metadata Service (GMS). GMS typically provides a [Rest.li](http://rest.li) API and must access the metadata using [GMA DAOs](../architecture/metadata-serving.md). + +GMA is designed to support a distributed fleet of GMS, each serving a subset of the [GMA graph](graph.md). However, for simplicity we include a single centralized GMS that serves all entities. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/graph.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/graph.md new file mode 100644 index 00000000..fd9f719d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/graph.md @@ -0,0 +1,21 @@ +--- +title: What is GMA graph? +slug: /what/graph +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/graph.md' +--- +# What is GMA graph? + +All the [entities](entity.md) and [relationships](relationship.md) are stored in a graph database, Neo4j. +The graph always represents the current state of the world and has no direct support for versioning or history. +However, as stated in the [Metadata Modeling](../modeling/metadata-model.md) section, +the graph is merely a derived view of all metadata [aspects](aspect.md) thus can always be rebuilt directly from historic [MAEs](mxe.md#metadata-audit-event-mae). +Consequently, it is possible to build a specific snapshot of the graph in time by replaying MAEs up to that point. + +In theory, the system can work with any generic [OLTP](https://en.wikipedia.org/wiki/Online_transaction_processing) graph DB that supports the following operations: + +- Dynamical creation, modification, and removal of nodes and edges +- Dynamical attachment of key-value properties to each node and edge +- Transactional partial updates of properties of a specific node or edge +- Fast ID-based retrieval of nodes & edges +- Efficient queries involving both graph traversal and properties value filtering +- Support efficient bidirectional graph traversal diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/mxe.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/mxe.md new file mode 100644 index 00000000..1a76c101 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/mxe.md @@ -0,0 +1,431 @@ +--- +title: Metadata Events +slug: /what/mxe +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/mxe.md' +--- +# Metadata Events + +DataHub makes use a few important Kafka events for operation. The most notable of these include + +1. Metadata Change Proposal +2. Metadata Change Log (Versioned + Timeseries) +3. Platform Event + +Each event is originally authored using [PDL](https://linkedin.github.io/rest.li/pdl_schema), a modeling language developed by LinkedIn, and +then converted into their Avro equivalents, which are used when writing and reading the events to Kafka. + +In the document, we'll describe each of these events in detail - including notes about their structure & semantics. + +## Metadata Change Proposal (MCP) + +A Metadata Change Proposal represents a request to change to a specific [aspect](aspect.md) on an enterprise's Metadata +Graph. Each MCP provides a new value for a given aspect. For example, a single MCP can +be emitted to change ownership or documentation or domains or deprecation status for a data asset. + +### Emission + +MCPs may be emitted by clients of DataHub's low-level ingestion APIs (e.g. ingestion sources) +during the process of metadata ingestion. The DataHub Python API exposes an interface for +easily sending MCPs into DataHub. + +The default Kafka topic name for MCPs is `MetadataChangeProposal_v1`. + +### Consumption + +DataHub's storage layer actively listens for new Metadata Change Proposals, attempts +to apply the requested change to the Metadata Graph. + +### Schema + +| Name | Type | Description | Optional | +| ------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| entityUrn | String | The unique identifier for the Entity being changed. For example, a Dataset's urn. | False | +| entityType | String | The type of the entity the new aspect is associated with. This corresponds to the entity name in the DataHub Entity Registry, for example 'dataset'. | False | +| entityKeyAspect | Object | The key struct of the entity that was changed. Only present if the Metadata Change Proposal contained the raw key struct. | True | +| changeType | String | The change type. CREATE, UPSERT and DELETE are currently supported. PATCH has limited support for specific aspects | False | +| aspectName | String | The entity aspect which was changed. | False | +| aspect | Object | The new aspect value. Null if the aspect was deleted. | True | +| aspect.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json`. | False | +| aspect.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| systemMetadata | Object | The new system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeProposal.pdl). + +### Examples + +An MCP representing a request to update the 'ownership' aspect for a particular Dataset: + +```json +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\":[{\"type\":\"DATAOWNER\",\"owner\":\"urn:li:corpuser:datahub\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:datahub\",\"time\":1651516640488}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1651516640493, + "runId": "no-run-id-provided", + "registryName": "unknownRegistry", + "registryVersion": "0.0.0.0-dev", + "properties": null + } +} +``` + +Note how the aspect payload is serialized as JSON inside the "value" field. The exact structure +of the aspect is determined by its PDL schema. (For example, the [ownership](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl) schema) + +## Metadata Change Log (MCL) + +A Metadata Change Log represents _any_ change which has been made to the Metadata Graph. +Metadata Change Log events are emitted to Kafka immediately after writing the change +the durable storage. + +There are 2 flavors of Metadata Change Log: _versioned_ and _timeseries_. These correspond to the type +of aspects which were updated for a given change. **Versioned** aspects are those +which represent the "latest" state of some attributes, for example the most recent owners of an asset +or its documentation. **Timeseries** aspects are those which represent events related to an asset +that occurred at a particular time, for example profiling of a Dataset. + +### Emission + +MCLs are emitted when _any_ change is made to an entity on the DataHub Metadata Graph, this includes +writing to any aspect of an entity. + +Two distinct topics are maintained for Metadata Change Log. The default Kafka topic name for **versioned** aspects is `MetadataChangeLog_Versioned_v1` and for +**timeseries** aspects is `MetadataChangeLog_Timeseries_v1`. + +### Consumption + +DataHub ships with a Kafka Consumer Job (mae-consumer-job) which listens for MCLs and uses them to update DataHub's search and graph indices, +as well as to generate derived Platform Events (described below). + +In addition, the [Actions Framework](../actions/README.md) consumes Metadata Change Logs to power its [Metadata Change Log](../actions/events/metadata-change-log-event.md) event API. + +### Schema + +| Name | Type | Description | Optional | +| ------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | +| entityUrn | String | The unique identifier for the Entity being changed. For example, a Dataset's urn. | False | +| entityType | String | The type of the entity the new aspect is associated with. This corresponds to the entity name in the DataHub Entity Registry, for example 'dataset'. | False | +| entityKeyAspect | Object | The key struct of the entity that was changed. Only present if the Metadata Change Proposal contained the raw key struct. | True | +| changeType | String | The change type. CREATE, UPSERT and DELETE are currently supported. | False | +| aspectName | String | The entity aspect which was changed. | False | +| aspect | Object | The new aspect value. Null if the aspect was deleted. | True | +| aspect.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json`. | False | +| aspect.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| previousAspectValue | Object | The previous aspect value. Null if the aspect did not exist previously. | True | +| previousAspectValue.contentType | String | The serialization type of the aspect itself. The only supported value is `application/json` | False | +| previousAspectValue.value | String | The serialized aspect. This is a JSON-serialized representing the aspect document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | +| systemMetadata | Object | The new system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | +| previousSystemMetadata | Object | The previous system metadata. This includes the the ingestion run-id, model registry and more. For the full structure, see https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/SystemMetadata.pdl | True | +| created | Object | Audit stamp about who triggered the Metadata Change and when. | False | +| created.time | Number | The timestamp in milliseconds when the aspect change occurred. | False | +| created.actor | String | The URN of the actor (e.g. corpuser) that triggered the change. | + +The PDL schema for can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeLog.pdl). + +### Examples + +An MCL corresponding to a change in the 'ownership' aspect for a particular Dataset: + +```json +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:hdfs,SampleHdfsDataset,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "value": "{\"owners\":[{\"type\":\"DATAOWNER\",\"owner\":\"urn:li:corpuser:datahub\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:datahub\",\"time\":1651516640488}}", + "contentType": "application/json" + }, + "previousAspectValue": { + "value": "{\"owners\":[{\"owner\":\"urn:li:corpuser:jdoe\",\"type\":\"DATAOWNER\"},{\"owner\":\"urn:li:corpuser:datahub\",\"type\":\"DATAOWNER\"}],\"lastModified\":{\"actor\":\"urn:li:corpuser:jdoe\",\"time\":1581407189000}}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1651516640493, + "runId": "no-run-id-provided", + "registryName": "unknownRegistry", + "registryVersion": "0.0.0.0-dev", + "properties": null + }, + "previousSystemMetadata": { + "lastObserved": 1651516415088, + "runId": "file-2022_05_02-11_33_35", + "registryName": null, + "registryVersion": null, + "properties": null + }, + "created": { + "time": 1651516640490, + "actor": "urn:li:corpuser:datahub", + "impersonator": null + } +} +``` + +Note how the aspect payload is serialized as JSON inside the "value" field. The exact structure +of the aspect is determined by its PDL schema. (For example, the [ownership](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/common/Ownership.pdl) schema) + +## Platform Event (PE) + +A Platform Event represents an arbitrary business-logic event emitted by DataHub. Each +Platform Event has a `name` which determines its contents. + +### Types + +- **Entity Change Event** (entityChangeEvent): The most important Platform Event is named **Entity Change Event**, and represents a log of semantic changes + (tag addition, removal, deprecation change, etc) that have occurred on DataHub. It is used an important + component of the DataHub Actions Framework. + +All registered Platform Event types are declared inside the DataHub Entity Registry (`entity-registry.yml`). + +### Emission + +All Platform Events are generated by DataHub itself during normal operation. + +PEs are extremely dynamic - they can contain arbitrary payloads depending on the `name`. Thus, +can be emitted in a variety of circumstances. + +The default Kafka topic name for all Platform Events is `PlatformEvent_v1`. + +### Consumption + +The [Actions Framework](../actions/README.md) consumes Platform Events to power its [Entity Change Event](../actions/events/entity-change-event.md) API. + +### Schema + +| Name | Type | Description | Optional | +| ---------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | +| header | Object | Header fields | False | +| header.timestampMillis | Long | The time at which the event was generated. | False | +| name | String | The name / type of the event. | False | +| payload | Object | The event itself. | False | +| payload.contentType | String | The serialization type of the event payload. The only supported value is `application/json`. | False | +| payload.value | String | The serialized payload. This is a JSON-serialized representing the payload document originally defined in PDL. See https://github.com/datahub-project/datahub/tree/master/metadata-models/src/main/pegasus/com/linkedin for more. | False | + +The full PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/PlatformEvent.pdl). + +### Examples + +An example of an 'Entity Change Event' Platform Event that is emitted when a new owner is added to a Dataset: + +```json +{ + "header": { + "timestampMillis": 1655390732551 + }, + "name": "entityChangeEvent", + "payload": { + "value": "{\"entityUrn\":\"urn:li:dataset:abc\",\"entityType\":\"dataset\",\"category\":\"OWNER\",\"operation\":\"ADD\",\"modifier\":\"urn:li:corpuser:jdoe\",\"parameters\":{\"ownerUrn\":\"urn:li:corpuser:jdoe\",\"ownerType\":\"BUSINESS_OWNER\"},\"auditStamp\":{\"actor\":\"urn:li:corpuser:jdoe\",\"time\":1649953100653}}", + "contentType": "application/json" +} +``` + +Note how the actual payload for the event is serialized as JSON inside the 'payload' field. The exact +structure of the Platform Event is determined by its PDL schema. (For example, the [Entity Change Event](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/platform/event/v1/EntityChangeEvent.pdl) schema) + +## Failed Metadata Change Proposal (FMCP) + +When a Metadata Change Proposal cannot be processed successfully, the event is written to a [dead letter queue](https://en.wikipedia.org/wiki/Dead_letter_queue) +in an event called Failed Metadata Change Proposal (FMCP). + +The event simply wraps the original Metadata Change Proposal and an error message, which contains the reason for rejection. +This event can be used for debugging any potential ingestion issues, as well as for re-playing any previous rejected proposal if necessary. + +### Emission + +FMCEs are emitted when MCEs cannot be successfully committed to DataHub's storage layer. + +The default Kafka topic name for FMCPs is `FailedMetadataChangeProposal_v1`. + +### Consumption + +No active consumers. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/FailedMetadataChangeProposal.pdl). + +# Deprecated Events + +DataHub ships with a set of deprecated events, which were historically used for proposing and logging +changes to the Metadata Graph. + +Each event in this category was deprecated due to its inflexibility - namely the fact that +the schemas had to be updated when a new aspect was introduced. These events have since been replaced +by the more flexible events described above (Metadata Change Proposal, Metadata Change Log). + +It is not recommended to build dependencies on deprecated events. + +## Metadata Change Event (MCE) + +A Metadata Change Event represents a request to change multiple aspects for the same entity. +It leverages a deprecated concept of `Snapshot`, which is a strongly-typed list of aspects for the same +entity. + +A MCE is a "proposal" for a set of metadata changes, as opposed to [MAE](#metadata-audit-event-mae), which is conveying a committed change. +Consequently, only successfully accepted and processed MCEs will lead to the emission of a corresponding MAE / MCLs. + +### Emission + +MCEs may be emitted by clients of DataHub's low-level ingestion APIs (e.g. ingestion sources) +during the process of metadata ingestion. + +The default Kafka topic name for MCEs is `MetadataChangeEvent_v4`. + +### Consumption + +DataHub's storage layer actively listens for new Metadata Change Events, attempts +to apply the requested changes to the Metadata Graph. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataChangeEvent.pdl). + +### Examples + +An example of an MCE emitted to change the 'ownership' aspect for an Entity: + +```json +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + } +} +``` + +## Metadata Audit Event (MAE) + +A Metadata Audit Event captures changes made to one or multiple metadata [aspects](aspect.md) associated with a particular [entity](entity.md), in the form of a metadata [snapshot](snapshot.md) (deprecated) before the change, and a metadata snapshot after the change. + +Every source-of-truth for a particular metadata aspect is expected to emit a MAE whenever a change is committed to that aspect. By ensuring that, any listener of MAE will be able to construct a complete view of the latest state for all aspects. +Furthermore, because each MAE contains the "after image", any mistake made in emitting the MAE can be easily mitigated by emitting a follow-up MAE with the correction. By the same token, the initial bootstrap problem for any newly added entity can also be solved by emitting a MAE containing all the latest metadata aspects associated with that entity. + +### Emission + +> Note: In recent versions of DataHub (mid 2022), MAEs are no longer actively emitted, and will soon be completely removed from DataHub. +> Use Metadata Change Log instead. + +MAEs are emitted once any metadata change has been successfully committed into DataHub's storage +layer. + +The default Kafka topic name for MAEs is `MetadataAuditEvent_v4`. + +### Consumption + +No active consumers. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/MetadataAuditEvent.pdl). + +### Examples + +An example of an MAE emitted representing a change made to the 'ownership' aspect for an Entity (owner removed): + +```json +{ + "oldSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:jdoe", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + }, + "newSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:datahub", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 1581407189000, + "actor": "urn:li:corpuser:jdoe", + "impersonator": null + } + } + } + ] + } + } +} +``` + +## Failed Metadata Change Event (FMCE) + +When a Metadata Change Event cannot be processed successfully, the event is written to a [dead letter queue](https://en.wikipedia.org/wiki/Dead_letter_queue) in an event called Failed Metadata Change Event (FMCE). + +The event simply wraps the original Metadata Change Event and an error message, which contains the reason for rejection. +This event can be used for debugging any potential ingestion issues, as well as for re-playing any previous rejected proposal if necessary. + +### Emission + +FMCEs are emitted when MCEs cannot be successfully committed to DataHub's storage layer. + +### Consumption + +No active consumers. + +### Schema + +The PDL schema can be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/mxe/FailedMetadataChangeEvent.pdl). + +The default Kafka topic name for FMCEs is `FailedMetadataChangeEvent_v4`. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/relationship.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/relationship.md new file mode 100644 index 00000000..f77d4c5b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/relationship.md @@ -0,0 +1,80 @@ +--- +title: What is a relationship? +slug: /what/relationship +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/what/relationship.md +--- +# What is a relationship? + +A relationship is a named associate between exactly two [entities](entity.md), a source and a destination. + +

+ +

+ +From the above graph, a `Group` entity can be linked to a `User` entity via a `HasMember` relationship. +Note that the name of the relationship reflects the direction, i.e. pointing from `Group` to `User`. +This is due to the fact that the actual metadata aspect holding this information is associated with `Group`, rather than +User. +Had the direction been reversed, the relationship would have been named `IsMemberOf` instead. +See [Direction of Relationships](#direction-of-relationships) for more discussions on relationship directionality. +A specific instance of a relationship, e.g. `urn:li:corpGroup:group1` has a member `urn:li:corpuser:user1`, +corresponds to an edge in the metadata graph. + +Relationships are meant to be "entity-neutral". In other words, one would expect to use the same `OwnedBy` relationship +to link a `Dataset` to a `User` and to link a `Dashboard` to a `User`. +As Pegasus doesn’t allow typing a field using multiple URNs (because they’re all essentially strings), we resort to +using generic URN type for the source and destination. +We also introduce a `@Relationship` [annotation](../modeling/extending-the-metadata-model.md/#relationship) to +limit the allowed source and destination URN types. + +While it’s possible to model relationships in rest.li +as [association resources](https://linkedin.github.io/rest.li/modeling/modeling#association), which often get stored as +mapping tables, it is far more common to model them as "foreign keys" field in a metadata aspect. For instance, +the `Ownership` aspect is likely to contain an array of owner’s corpUser URNs. + +Below is an example of how a relationship is modeled in PDL. Note that: + +1. This aspect, `nativeGroupMembership` would be associated with a `corpUser` +2. The `corpUser`'s aspect points to one or more parent entities of type `corpGroup` + +``` +namespace com.linkedin.identity + +import com.linkedin.common.Urn + +/** + * Carries information about the native CorpGroups a user is in. + */ +@Aspect = { + "name": "nativeGroupMembership" +} +record NativeGroupMembership { + @Relationship = { + "/*": { + "name": "IsMemberOfNativeGroup", + "entityTypes": [ "corpGroup" ] + } + } + nativeGroups: array[Urn] +} +``` + +## Direction of Relationships + +As relationships are modeled as directed edges between nodes, it’s natural to ask which way should it be pointing, +or should there be edges going both ways? The answer is, "doesn’t really matter." It’s rather an aesthetic choice than +technical one. + +For one, the actual direction doesn’t really impact the execution of graph queries. Most graph DBs are fully capable of +traversing edges in reverse direction efficiently. + +That being said, generally there’s a more "natural way" to specify the direction of a relationship, which closely relate +to how the metadata is stored. For example, the membership information for an LDAP group is generally stored as a list +in group’s metadata. As a result, it’s more natural to model a `HasMember` relationship that points from a group to a +member, instead of a `IsMemberOf` relationship pointing from member to group. + +## High Cardinality Relationships + +See [this doc](../advanced/high-cardinality.md) for suggestions on how to best model relationships with high +cardinality. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/search-document.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/search-document.md new file mode 100644 index 00000000..96bae098 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/search-document.md @@ -0,0 +1,69 @@ +--- +title: What is a search document? +slug: /what/search-document +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/what/search-document.md +--- +# What is a search document? + +[Search documents](https://en.wikipedia.org/wiki/Search_engine_indexing) are also modeled using [PDL](https://linkedin.github.io/rest.li/pdl_schema) explicitly. +In many ways, the model for a Document is very similar to an [Entity](entity.md) and [Relationship](relationship.md) model, +where each attribute/field contains a value that’s derived from various metadata aspects. +However, a search document is also allowed to have array type of attribute that contains only primitives or enum items. +This is because most full-text search engines supports membership testing against an array field, e.g. an array field containing all the terms used in a document. + +One obvious use of the attributes is to perform search filtering, e.g. give me all the `User` whose first name or last name is similar to “Joe” and reports up to `userFoo`. +Since the document is also served as the main interface for the search API, the attributes can also be used to format the search snippet. +As a result, one may be tempted to add as many attributes as needed. This is acceptable as the underlying search engine is designed to index a large number of fields. + +Below shows an example schema for the `User` search document. Note that: + +1. Each search document is required to have a type-specific `urn` field, generally maps to an entity in the [graph](graph.md). +2. Similar to `Entity`, each document has an optional `removed` field for "soft deletion". +3. Similar to `Entity`, all remaining fields are made `optional` to support partial updates. +4. `management` shows an example of a string array field. +5. `ownedDataset` shows an example on how a field can be derived from metadata [aspects](aspect.md) associated with other types of entity (in this case, `Dataset`). + +``` +namespace com.linkedin.metadata.search + +/** + * Common fields that may apply to all documents + */ +record BaseDocument { + + /** Whether the entity has been removed or not */ + removed: optional boolean = false +} +``` + +``` +namespace com.linkedin.metadata.search + +import com.linkedin.common.CorpuserUrn +import com.linkedin.common.DatasetUrn + +/** + * Data model for user entity search + */ +record UserDocument includes BaseDocument { + + /** Urn for the user */ + urn: CorpuserUrn + + /** First name of the user */ + firstName: optional string + + /** Last name of the user */ + lastName: optional string + + /** The chain of management all the way to CEO */ + management: optional array[CorpuserUrn] = [] + + /** Code for the cost center */ + costCenter: optional int + + /** The list of dataset the user owns */ + ownedDatasets: optional array[DatasetUrn] = [] +} +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/search-index.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/search-index.md new file mode 100644 index 00000000..25165895 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/search-index.md @@ -0,0 +1,24 @@ +--- +title: What is GMA search index? +slug: /what/search-index +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/docs/what/search-index.md +--- +# What is GMA search index? + +Each [search document](search-document.md) type (or [entity](entity.md) type) will be mapped to an independent search index in Elasticsearch. +Beyond the standard search engine features (analyzer, tokenizer, filter queries, faceting, sharding, etc), +GMA also supports the following specific features: + +- Partial update of indexed documents +- Membership testing on multi-value fields +- Zero downtime switch between indices + +Check out [Search DAO](../architecture/metadata-serving.md#search-dao) for search query abstraction in GMA. + +## Search Automation (TBD) + +We aim to automate the index creation, schema evolution, and reindexing such that the team will only need to focus on the search document model and their custom [Index Builder](../architecture/metadata-ingestion.md#search-index-builders) logic. +As the logic changes, a new version of the index will be created and populated from historic MAEs. +Once it’s fully populated, the team can switch to the new version through a simple config change from their [GMS](gms.md). +They can also rollback to an older version of index whenever needed. diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/snapshot.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/snapshot.md new file mode 100644 index 00000000..dabdeb60 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/snapshot.md @@ -0,0 +1,54 @@ +--- +title: What is a snapshot? +slug: /what/snapshot +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/snapshot.md' +--- +# What is a snapshot? + +A metadata snapshot models the current state of one or multiple metadata [aspects](aspect.md) associated with a particular [entity](entity.md). +Each entity type is expected to have: + +1. An entity-specific aspect (e.g. `CorpGroupAspect` from below), which is a `typeref` containing a union of all possible metadata aspects for the entity. +2. An entity-specific snapshot (e.g. `CorpGroupSnapshot` from below), which contains an array (aspects) of entity-specific aspects. + +``` +namespace com.linkedin.metadata.aspect + +import com.linkedin.group.Membership +import com.linkedin.group.SomeOtherMetadata + +/** + * A union of all supported metadata aspects for a group + */ +typeref CorpGroupAspect = union[Membership, SomeOtherMetadata] +``` + +``` +namespace com.linkedin.metadata.snapshot + +import com.linkedin.common.CorpGroupUrn +import com.linkedin.metadata.aspect.CorpGroupAspect + +/** + * A metadata snapshot for a specific Group entity. + */ +record CorpGroupSnapshot { + + /** URN for the entity the metadata snapshot is associated with */ + urn: CorpGroupUrn + + /** The list of metadata aspects associated with the group */ + aspects: array[CorpGroupAspect] +} +``` + +The generic `Snapshot` typeref contains a union of all entity-specific snapshots and can therefore be used to represent the state of any metadata aspect for all supported entity types. + +``` +namespace com.linkedin.metadata.snapshot + +/** + * A union of all supported metadata snapshot types. + */ +typeref Snapshot = union[DatasetSnapshot, CorpGroupSnapshot, CorpUserSnapshot] +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/docs/what/urn.md b/docs-archive/versioned_docs/version-1.5.0/docs/what/urn.md new file mode 100644 index 00000000..7f7f8839 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/docs/what/urn.md @@ -0,0 +1,62 @@ +--- +title: What is URN? +slug: /what/urn +custom_edit_url: 'https://github.com/datahub-project/datahub/blob/master/docs/what/urn.md' +--- +# What is URN? + +URN ([Uniform Resource Name](https://en.wikipedia.org/wiki/Uniform_Resource_Name)) is the chosen scheme of [URI](https://en.wikipedia.org/wiki/Uniform_Resource_Identifier) to uniquely define any resource in DataHub. It has the following form + +``` +urn::: +``` + +[Onboarding a new entity](../modeling/metadata-model.md) to [GMA](gma.md) starts with modelling an URN specific to that entity. +You can use the existing [URN models](https://github.com/datahub-project/datahub/blob/master/li-utils/src/main/javaPegasus/com/linkedin/common/urn) for built-in entities as a reference. + +## Namespace + +All URNs available in DataHub are using `li` as their namespace. +This can be easily changed to a different namespace for your organization if you fork DataHub. + +## Entity Type + +Entity type for URN is different than [entity](entity.md) in GMA context. This can be thought of as the object type of +any resource for which you need unique identifier for its each instance. While you can create URNs for GMA entities such as +[DatasetUrn] with entity type `dataset`, you can also define URN for data platforms, [DataPlatformUrn]. + +## ID + +ID is the unique identifier part of a URN. It's unique for a specific entity type within a specific namespace. +ID could contain a single field, or multi fields in the case of complex URNs. A complex URN can even contain other URNs as ID fields. This type of URN is also referred to as nested URN. For non-URN ID fields, the value can be either a string, number, or [Pegasus Enum](https://linkedin.github.io/rest.li/pdl_schema#enum-type). + +Here are some example URNs with a single ID field: + +``` +urn:li:dataPlatform:kafka +urn:li:corpuser:jdoe +``` + +[DatasetUrn](https://github.com/datahub-project/datahub/blob/master/li-utils/src/main/javaPegasus/com/linkedin/common/urn/DatasetUrn.java) is an example of a complex nested URN. It contains 3 ID fields: `platform`, `name` and `fabric`, where `platform` is another [URN](https://github.com/datahub-project/datahub/blob/master/li-utils/src/main/javaPegasus/com/linkedin/common/urn/DataPlatformUrn.java). Here are some examples + +``` +urn:li:dataset:(urn:li:dataPlatform:kafka,PageViewEvent,PROD) +urn:li:dataset:(urn:li:dataPlatform:hdfs,PageViewEvent,EI) +``` + +## Restrictions + +There are a few restrictions when creating an URN: + +The following characters are not allowed anywhere in the URN + +1. Parentheses are reserved characters in URN fields: `(` or `)` +2. The "unit separator" unicode character `␟` (U+241F) + +The following characters are not allowed within an URN tuple. + +1. Commas are reserved characters in URN tuples: `,` + +Example: `urn:li:dashboard:(looker,dashboards.thelook)` is a valid urn, but `urn:li:dashboard:(looker,dashboards.the,look)` is invalid. + +Please do not use these characters when creating or generating urns. One approach is to use URL encoding for the characters. diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/enums.md b/docs-archive/versioned_docs/version-1.5.0/graphql/enums.md new file mode 100644 index 00000000..dcb1cfbd --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/enums.md @@ -0,0 +1,4254 @@ +--- +id: enums +title: Enums +slug: enums +sidebar_position: 5 +--- + +## AccessLevel + +The access level for a Metadata Entity, either public or private + +

Values

+ + + + + + + + + + + + + +
ValueDescription
PUBLIC +

Publicly available

+
PRIVATE +

Restricted to a subset of viewers

+
+ +## AccessTokenDuration + +The duration for which an Access Token is valid. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
ONE_HOUR +

1 hour

+
ONE_DAY +

1 day

+
ONE_WEEK +

1 week

+
ONE_MONTH +

1 month

+
THREE_MONTHS +

3 months

+
SIX_MONTHS +

6 months

+
ONE_YEAR +

1 year

+
NO_EXPIRY +

No expiry

+
+ +## AccessTokenType + +A type of DataHub Access Token. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
PERSONAL +

Generates a personal access token

+
SERVICE_ACCOUNT +

Generates a service account access token

+
+ +## AssertionActionType + +The type of the Action + +

Values

+ + + + + + + + + + + + + +
ValueDescription
RAISE_INCIDENT +

Raise an incident.

+
RESOLVE_INCIDENT +

Resolve open incidents related to the assertion.

+
+ +## AssertionResultErrorType + +The type of error encountered when evaluating an AssertionResult + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
SOURCE_CONNECTION_ERROR +

Source is unreachable

+
SOURCE_QUERY_FAILED +

Source query failed to execute

+
INVALID_PARAMETERS +

Invalid parameters were detected

+
INSUFFICIENT_DATA +

Insufficient data to evaluate assertion

+
INVALID_SOURCE_TYPE +

Event type not supported by the specified source

+
UNSUPPORTED_PLATFORM +

Platform not supported

+
CUSTOM_SQL_ERROR +

Error while executing a custom SQL assertion

+
FIELD_ASSERTION_ERROR +

Error while executing a field assertion

+
UNKNOWN_ERROR +

Unknown error

+
+ +## AssertionResultType + +The result type of an assertion, success or failure. + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
INIT +

The assertion has not yet been fully evaluated.

+
SUCCESS +

The assertion succeeded.

+
FAILURE +

The assertion failed.

+
ERROR +

The assertion errored.

+
+ +## AssertionRunStatus + +The state of an assertion run, as defined within an Assertion Run Event. + +

Values

+ + + + + + + + + +
ValueDescription
COMPLETE +

An assertion run has completed.

+
+ +## AssertionSourceType + +The source of an assertion + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
NATIVE +

The assertion was defined natively on DataHub by a user.

+
EXTERNAL +

The assertion was defined and managed externally of DataHub.

+
INFERRED +

The assertion was inferred, e.g. from offline AI / ML models.

+
+ +## AssertionStdAggregation + +An "aggregation" function that can be applied to column values of a Dataset to create the input to an Assertion Operator. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
IDENTITY +

Assertion is applied on individual column value

+
MEAN +

Assertion is applied on column mean

+
MEDIAN +

Assertion is applied on column median

+
UNIQUE_COUNT +

Assertion is applied on number of distinct values in column

+
UNIQUE_PROPOTION +

Assertion is applied on proportion of distinct values in column

+
NULL_COUNT +

Assertion is applied on number of null values in column

+
NULL_PROPORTION +

Assertion is applied on proportion of null values in column

+
STDDEV +

Assertion is applied on column std deviation

+
MIN +

Assertion is applied on column min

+
MAX +

Assertion is applied on column std deviation

+
SUM +

Assertion is applied on column sum

+
COLUMNS +

Assertion is applied on all columns

+
COLUMN_COUNT +

Assertion is applied on number of columns

+
ROW_COUNT +

Assertion is applied on number of rows

+
_NATIVE_ +

Other

+
+ +## AssertionStdOperator + +A standard operator or condition that constitutes an assertion definition + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
BETWEEN +

Value being asserted is between min_value and max_value

+
LESS_THAN +

Value being asserted is less than max_value

+
LESS_THAN_OR_EQUAL_TO +

Value being asserted is less than or equal to max_value

+
GREATER_THAN +

Value being asserted is greater than min_value

+
GREATER_THAN_OR_EQUAL_TO +

Value being asserted is greater than or equal to min_value

+
EQUAL_TO +

Value being asserted is equal to value

+
NOT_EQUAL_TO +

Value being asserted is not equal to value

+
NULL +

Value being asserted is null

+
NOT_NULL +

Value being asserted is not null

+
CONTAIN +

Value being asserted contains value

+
END_WITH +

Value being asserted ends with value

+
START_WITH +

Value being asserted starts with value

+
REGEX_MATCH +

Value being asserted matches the regex value.

+
IN +

Value being asserted is one of the array values

+
NOT_IN +

Value being asserted is not in one of the array values.

+
IS_TRUE +

Value being asserted is true

+
IS_FALSE +

Value being asserted is false

+
_NATIVE_ +

Other

+
+ +## AssertionStdParameterType + +The type of an AssertionStdParameter + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
STRING +

A string value

+
NUMBER +

A numeric value

+
LIST +

A list of values. When used, the value should be formatted as a serialized JSON array.

+
SET +

A set of values. When used, the value should be formatted as a serialized JSON array.

+
UNKNOWN +

A value of unknown type

+
+ +## AssertionType + +The top-level assertion type. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
DATASET +

A single-dataset assertion.

+
FRESHNESS +

An assertion which indicates when a particular operation should occur to an asset.

+
VOLUME +

An assertion which indicates how much data should be available for a particular asset.

+
SQL +

A raw SQL-statement based assertion.

+
FIELD +

A structured assertion targeting a specific column or field of the Dataset.

+
DATA_SCHEMA +

A schema or structural assertion.

+
CUSTOM +

A custom assertion.

+
+ +## AssertionValueChangeType + +An enum to represent a type of change in an assertion value, metric, or measurement. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
ABSOLUTE +

A change that is defined in absolute terms.

+
PERCENTAGE +

A change that is defined in relative terms using percentage change +from the original value.

+
+ +## ChangeCategoryType + +Enum of CategoryTypes + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
DOCUMENTATION +

When documentation has been edited

+
GLOSSARY_TERM +

When glossary terms have been added or removed

+
OWNERSHIP +

When ownership has been modified

+
TECHNICAL_SCHEMA +

When technical schemas have been added or removed

+
TAG +

When tags have been added or removed

+
PARENT +

When parent relationship has been modified

+
RELATED_ENTITIES +

When related entities have been added or removed

+
+ +## ChangeOperationType + +Enum of types of changes + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
ADD +

When an element is added

+
MODIFY +

When an element is modified

+
REMOVE +

When an element is removed

+
+ +## ChartQueryType + +The type of the Chart Query + +

Values

+ + + + + + + + + + + + + +
ValueDescription
SQL +

Standard ANSI SQL

+
LOOKML +

LookML

+
+ +## ChartType + +The type of a Chart Entity + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
BAR +

Bar graph

+
PIE +

Pie chart

+
SCATTER +

Scatter plot

+
TABLE +

Table

+
TEXT +

Markdown formatted text

+
LINE +

A line chart

+
AREA +

An area chart

+
HISTOGRAM +

A histogram chart

+
BOX_PLOT +

A box plot chart

+
WORD_CLOUD +

A word cloud chart

+
COHORT +

A Cohort Analysis chart

+
+ +## CorpUserStatus + +The state of a CorpUser + +

Values

+ + + + + + + + + + + + + +
ValueDescription
ACTIVE +

A User that has been provisioned and logged in

+
SUSPENDED +

A user that has been suspended

+
+ +## CostType + + + +

Values

+ + + + + + + + + +
ValueDescription
ORG_COST_TYPE +

Org Cost Type to which the Cost of this entity should be attributed to

+
+ +## DataContractState + +The state of the data contract + +

Values

+ + + + + + + + + + + + + +
ValueDescription
ACTIVE +

The data contract is active.

+
PENDING +

The data contract is pending. Note that this symbol is currently experimental.

+
+ +## DataHubConnectionDetailsType + +The type of a DataHub connection + +

Values

+ + + + + + + + + +
ValueDescription
JSON +

A json-encoded set of connection details.

+
+ +## DataHubPageModuleType + +Enum containing the types of page modules that there are + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
LINK +

Link type module

+
RICH_TEXT +

Module containing rich text to be rendered

+
ASSET_COLLECTION +

A module with a collection of assets

+
HIERARCHY +

A module displaying a hierarchy to navigate

+
OWNED_ASSETS +

Module displaying assets owned by a user

+
DOMAINS +

Module displaying the top domains

+
ASSETS +

Module displaying the assets of parent entities

+
CHILD_HIERARCHY +

Module displaying the hierarchy of the children of a given entity. Glossary or Domains.

+
DATA_PRODUCTS +

Module displaying child data products of a given domain

+
RELATED_TERMS +

Module displaying the related terms of a given glossary term

+
PLATFORMS +

Module displaying the platforms in the instance

+
LINEAGE +

Module displaying the lineage of an asset

+
COLUMNS +

Module displaying the columns of a dataset

+
UNKNOWN +

Unknown module type - this can occur with corrupted data or rolling back to versions without new modules

+
+ +## DataHubViewType + +The type of a DataHub View + +

Values

+ + + + + + + + + + + + + +
ValueDescription
PERSONAL +

A personal view - e.g. saved filters

+
GLOBAL +

A global view, e.g. role view

+
+ +## DataProcessInstanceRunResultType + +The result of the data process run + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
SUCCESS +

The run finished successfully

+
FAILURE +

The run finished in failure

+
SKIPPED +

The run was skipped

+
UP_FOR_RETRY +

The run failed and is up for retry

+
+ +## DataProcessRunStatus + +The status of the data process instance + +

Values

+ + + + + + + + + + + + + +
ValueDescription
STARTED +

The data process instance has started but not completed

+
COMPLETE +

The data process instance has completed

+
+ +## DatasetAssertionScope + +The scope that a Dataset-level assertion applies to. + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
DATASET_COLUMN +

Assertion applies to columns of a dataset.

+
DATASET_ROWS +

Assertion applies to rows of a dataset.

+
DATASET_SCHEMA +

Assertion applies to schema of a dataset.

+
UNKNOWN +

The scope of an assertion is unknown.

+
+ +## DatasetFilterType + +Type of partition + +

Values

+ + + + + + + + + +
ValueDescription
SQL +

Use a SQL string to apply the filter

+
+ +## DatasetLineageType + +Deprecated +The type of an edge between two Datasets + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
COPY +

Direct copy without modification

+
TRANSFORMED +

Transformed dataset

+
VIEW +

Represents a view defined on the sources

+
+ +## DateInterval + +For consumption by UI only + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
SECOND + +
MINUTE + +
HOUR + +
DAY + +
WEEK + +
MONTH + +
YEAR + +
+ +## DocumentChangeType + +Types of changes that can occur to a document + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
CREATED +

Document was created

+
TITLE_CHANGED +

Document title was modified

+
TEXT_CHANGED +

Document text content was modified

+
PARENT_CHANGED +

Document was moved to a different parent

+
RELATED_DOCUMENTS_CHANGED +

Relationships to other documents were added or removed

+
RELATED_ASSETS_CHANGED +

Relationships to assets (datasets, dashboards, etc.) were added or removed

+
STATE_CHANGED +

Document state changed (e.g., published <-> unpublished)

+
DELETED +

Document was deleted

+
+ +## DocumentSourceType + +The type of source for a document + +

Values

+ + + + + + + + + + + + + +
ValueDescription
NATIVE +

Created via the DataHub UI or API

+
EXTERNAL +

The document was ingested from an external source

+
+ +## DocumentState + +The state of a Document + +

Values

+ + + + + + + + + + + + + +
ValueDescription
PUBLISHED +

Document is published and visible to users

+
UNPUBLISHED +

Document is not published publically

+
+ +## EntityType + +A top level Metadata Entity Type + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
DOMAIN +

A Domain containing Metadata Entities

+
DATASET +

The Dataset Entity

+
CORP_USER +

The CorpUser Entity

+
CORP_GROUP +

The CorpGroup Entity

+
DATA_PLATFORM +

The DataPlatform Entity

+
ER_MODEL_RELATIONSHIP +

The ERModelRelationship Entity

+
DASHBOARD +

The Dashboard Entity

+
NOTEBOOK +

The Notebook Entity

+
CHART +

The Chart Entity

+
DATA_FLOW +

The Data Flow (or Data Pipeline) Entity,

+
DATA_JOB +

The Data Job (or Data Task) Entity

+
TAG +

The Tag Entity

+
GLOSSARY_TERM +

The Glossary Term Entity

+
GLOSSARY_NODE +

The Glossary Node Entity

+
CONTAINER +

A container of Metadata Entities

+
MLMODEL +

The ML Model Entity

+
MLMODEL_GROUP +

The MLModelGroup Entity

+
MLFEATURE_TABLE +

ML Feature Table Entity

+
MLFEATURE +

The ML Feature Entity

+
MLPRIMARY_KEY +

The ML Primary Key Entity

+
INGESTION_SOURCE +

A DataHub Managed Ingestion Source

+
EXECUTION_REQUEST +

A DataHub ExecutionRequest

+
ASSERTION +

A DataHub Assertion

+
DATA_PROCESS_INSTANCE +

An instance of an individual run of a data job or data flow

+
DATA_PLATFORM_INSTANCE +

Data Platform Instance Entity

+
ACCESS_TOKEN +

A DataHub Access Token

+
TEST +

A DataHub Test

+
DATAHUB_POLICY +

A DataHub Policy

+
DATAHUB_ROLE +

A DataHub Role

+
POST +

A DataHub Post

+
SCHEMA_FIELD +

A Schema Field

+
DATAHUB_VIEW +

A DataHub View

+
QUERY +

A dataset query

+
DATA_PRODUCT +

A Data Product

+
CUSTOM_OWNERSHIP_TYPE +

A Custom Ownership Type

+
DATAHUB_CONNECTION +

A connection to an external source.

+
INCIDENT +

A DataHub incident - SaaS only

+
ROLE +

" +A Role from an organisation

+
DATA_CONTRACT +

A data contract

+
STRUCTURED_PROPERTY +

" +An structured property on entities

+
FORM +

" +A form entity on entities

+
DATA_TYPE +

" +A data type registered to DataHub

+
ENTITY_TYPE +

" +A type of entity registered to DataHub

+
RESTRICTED +

" +A type of entity that is restricted to the user

+
OTHER +

Another entity type - refer to a provided entity type urn.

+
BUSINESS_ATTRIBUTE +

A Business Attribute

+
VERSION_SET +

A set of versioned entities, representing a single source / logical entity over time

+
APPLICATION +

An application

+
DOCUMENT +

A Knowledge Article

+
DATAHUB_PAGE_TEMPLATE +

An DataHub Page Template

+
DATAHUB_PAGE_MODULE +

An DataHub Page Module

+
DATAHUB_FILE +

An DataHub File

+
+ +## ERModelRelationshipCardinality + +The Cardinality of the ERModelRelationship + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
ONE_ONE +

One to One

+
ONE_N +

One to Many

+
N_ONE +

Many to One

+
N_N +

Many to Many

+
+ +## FabricType + +An environment identifier for a particular Entity, ie staging or production +Note that this model will soon be deprecated in favor of a more general purpose of notion +of data environment + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
DEV +

Designates development fabrics

+
TEST +

Designates testing fabrics

+
QA +

Designates quality assurance fabrics

+
UAT +

Designates user acceptance testing fabrics

+
EI +

Designates early integration fabrics

+
PRE +

Designates pre-production fabrics

+
STG +

Designates staging fabrics

+
NON_PROD +

Designates non-production fabrics

+
PROD +

Designates production fabrics

+
CORP +

Designates corporation fabrics

+
RVW +

Designates review fabrics

+
SANDBOX +

Designates sandbox fabrics

+
PRD +

Designates alternative spelling PROD

+
TST +

Designates alternative spelling TEST

+
+ +## FieldAssertionType + +The type of a Field assertion + +

Values

+ + + + + + + + + + + + + +
ValueDescription
FIELD_VALUES +

An assertion used to validate the values contained with a field / column given a set of rows.

+
FIELD_METRIC +

An assertion used to validate the value of a common field / column metric (e.g. aggregation) +such as null count + percentage, min, max, median, and more.

+
+ +## FieldMetricType + +A standard metric that can be derived from the set of values +for a specific field / column of a dataset / table. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
UNIQUE_COUNT +

The number of unique values found in the column value set

+
UNIQUE_PERCENTAGE +

The percentage of unique values to total rows for the dataset

+
NULL_COUNT +

The number of null values found in the column value set

+
NULL_PERCENTAGE +

The percentage of null values to total rows for the dataset

+
MIN +

The minimum value in the column set (applies to numeric columns)

+
MAX +

The maximum value in the column set (applies to numeric columns)

+
MEAN +

The mean length found in the column set (applies to numeric columns)

+
MEDIAN +

The median length found in the column set (applies to numeric columns)

+
STDDEV +

The stddev length found in the column set (applies to numeric columns)

+
NEGATIVE_COUNT +

The number of negative values found in the value set (applies to numeric columns)

+
NEGATIVE_PERCENTAGE +

The percentage of negative values to total rows for the dataset (applies to numeric columns)

+
ZERO_COUNT +

The number of zero values found in the value set (applies to numeric columns)

+
ZERO_PERCENTAGE +

The percentage of zero values to total rows for the dataset (applies to numeric columns)

+
MIN_LENGTH +

The minimum length found in the column set (applies to string columns)

+
MAX_LENGTH +

The maximum length found in the column set (applies to string columns)

+
EMPTY_COUNT +

The number of empty string values found in the value set (applies to string columns). +Note: This is a completely different metric different from NULL_COUNT!

+
EMPTY_PERCENTAGE +

The percentage of empty string values to total rows for the dataset (applies to string columns). +Note: This is a completely different metric different from NULL_PERCENTAGE!

+
+ +## FieldTransformType + +The type of the Field Transform + +

Values

+ + + + + + + + + +
ValueDescription
LENGTH +

Obtain the length of a string field / column (applicable to string types)

+
+ +## FieldValuesFailThresholdType + +The type of failure threshold. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
COUNT +

The maximum number of column values (i.e. rows) that are allowed +to fail the defined expectations before the assertion officially fails.

+
PERCENTAGE +

The maximum percentage of rows that are allowed +to fail the defined column expectations before the assertion officially fails.

+
+ +## FilterOperator + + + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
CONTAIN +

Represent the relation: String field contains value, e.g. name contains Profile

+
EQUAL +

Represent the relation: field = value, e.g. platform = hdfs

+
IEQUAL +

Represent the relation: field = value (case-insensitive), e.g. platform = HDFS

+
IN +
    +
  • Represent the relation: String field is one of the array values to, e.g. name in ["Profile", "Event"]
    • +
+
EXISTS +

Represents the relation: The field exists. If the field is an array, the field is either not present or empty.

+
GREATER_THAN +

Represent the relation greater than, e.g. ownerCount > 5

+
GREATER_THAN_OR_EQUAL_TO +

Represent the relation greater than or equal to, e.g. ownerCount >= 5

+
LESS_THAN +

Represent the relation less than, e.g. ownerCount < 3

+
LESS_THAN_OR_EQUAL_TO +

Represent the relation less than or equal to, e.g. ownerCount <= 3

+
START_WITH +

Represent the relation: String field starts with value, e.g. name starts with PageView

+
END_WITH +

Represent the relation: String field ends with value, e.g. name ends with Event

+
DESCENDANTS_INCL +

Represent the relation: URN field any nested children in addition to the given URN

+
ANCESTORS_INCL +

Represent the relation: URN field matches any nested parent in addition to the given URN

+
RELATED_INCL +

Represent the relation: URN field matches any nested child or parent in addition to the given URN

+
+ +## FormPromptType + +Enum of all form prompt types + +

Values

+ + + + + + + + + + + + + +
ValueDescription
STRUCTURED_PROPERTY +

A structured property form prompt type.

+
FIELDS_STRUCTURED_PROPERTY +

A schema field-level structured property form prompt type.

+
+ +## FormType + +The type of a form. This is optional on a form entity + +

Values

+ + + + + + + + + + + + + +
ValueDescription
VERIFICATION +

This form is used for "verifying" entities as a state for governance and compliance

+
COMPLETION +

This form is used to help with filling out metadata on entities

+
+ +## FreshnessAssertionScheduleType + +The type of an Freshness assertion + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
CRON +

An schedule based on a CRON schedule representing the expected event times.

+
FIXED_INTERVAL +

A scheduled based on a recurring fixed schedule which is used to compute the expected operation window. E.g. "every 24 hours".

+
SINCE_THE_LAST_CHECK +

A schedule computed based on when the assertion was last evaluated, to the current moment in time.

+
+ +## FreshnessAssertionType + +The type of an Freshness assertion + +

Values

+ + + + + + + + + + + + + +
ValueDescription
DATASET_CHANGE +

An assertion defined against a Dataset Change Operation - insert, update, delete, etc

+
DATA_JOB_RUN +

An assertion defined against a Data Job run

+
+ +## HealthStatus + + + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
PASS +

The Asset is in a healthy state

+
WARN +

The Asset is in a warning state

+
FAIL +

The Asset is in a failing (unhealthy) state

+
+ +## HealthStatusType + +The type of the health status + +

Values

+ + + + + + + + + + + + + +
ValueDescription
ASSERTIONS +

Assertions status

+
INCIDENTS +

Incidents status

+
+ +## IconLibrary + + + +

Values

+ + + + + + + + + +
ValueDescription
MATERIAL +

Icons from the Material UI icon library

+
+ +## IncidentPriority + +The priority of the incident + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
LOW +

A low priority incident (P3)

+
MEDIUM +

A medium priority incident (P2)

+
HIGH +

A high priority incident (P1)

+
CRITICAL +

A critical priority incident (P0)

+
+ +## IncidentSourceType + +The source type of an incident, implying how it was created. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
MANUAL +

The incident was created manually, from either the API or the UI.

+
ASSERTION_FAILURE +

An assertion has failed, triggering the incident.

+
+ +## IncidentStage + +The lifecycle stage of the incident. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
TRIAGE +

The impact and priority of the incident is being actively assessed.

+
INVESTIGATION +

The incident root cause is being investigated.

+
WORK_IN_PROGRESS +

The incident is in the remediation stage.

+
FIXED +

The incident is in the resolved as completed stage.

+
NO_ACTION_REQUIRED +

The incident is in the resolved with no action required state, e.g., the +incident was a false positive, or was expected.

+
+ +## IncidentState + +The state of an incident. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
ACTIVE +

The incident is ongoing, or active.

+
RESOLVED +

The incident is resolved.

+
+ +## IncidentType + +A specific type of incident + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
FRESHNESS +

A Freshness Assertion has failed, triggering the incident. +Raised on assets where assertions are configured to generate incidents.

+
VOLUME +

A Volume Assertion has failed, triggering the incident. +Raised on assets where assertions are configured to generate incidents.

+
FIELD +

A Field Assertion has failed, triggering the incident. +Raised on assets where assertions are configured to generate incidents.

+
SQL +

A SQL Assertion has failed, triggering the incident. +Raised on assets where assertions are configured to generate incidents.

+
DATA_SCHEMA +

A Schema has failed, triggering the incident. +Raised on assets where assertions are configured to generate incidents.

+
OPERATIONAL +

An operational incident, e.g. failure to materialize a dataset, or failure to execute a task / pipeline.

+
CUSTOM +

A custom type of incident

+
+ +## IncrementingSegmentFieldTransformerType + +The 'standard' transformer type. Note that not all source systems will support all operators. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
TIMESTAMP_MS_TO_MINUTE +

Rounds a timestamp (in seconds) down to the start of the month.

+
TIMESTAMP_MS_TO_HOUR +

Rounds a timestamp (in milliseconds) down to the nearest hour.

+
TIMESTAMP_MS_TO_DATE +

Rounds a timestamp (in milliseconds) down to the start of the day.

+
TIMESTAMP_MS_TO_MONTH +

Rounds a timestamp (in milliseconds) down to the start of the month

+
TIMESTAMP_MS_TO_YEAR +

Rounds a timestamp (in milliseconds) down to the start of the year

+
FLOOR +

Rounds a numeric value down to the nearest integer.

+
CEILING +

Rounds a numeric value up to the nearest integer.

+
NATIVE +

A backdoor to provide a native operator type specific to a given source system like +Snowflake, Redshift, BQ, etc.

+
+ +## IngestionSourceSourceType + +The type of ingestion source source + +

Values

+ + + + + + + + + +
ValueDescription
SYSTEM +

A system internal source, e.g. for running search indexing operations, feature computation, etc.

+
+ +## IntendedUserType + + + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
ENTERPRISE +

Developed for Enterprise Users

+
HOBBY +

Developed for Hobbyists

+
ENTERTAINMENT +

Developed for Entertainment Purposes

+
+ +## LineageDirection + +Direction between two nodes in the lineage graph + +

Values

+ + + + + + + + + + + + + +
ValueDescription
UPSTREAM +

Upstream, or left-to-right in the lineage visualization

+
DOWNSTREAM +

Downstream, or right-to-left in the lineage visualization

+
+ +## LineageSearchPath + +The path taken when doing search across lineage + +

Values

+ + + + + + + + + + + + + +
ValueDescription
TORTOISE +

Designates the tortoise lineage code path

+
LIGHTNING +

Designates the lightning lineage code path

+
+ +## LogicalOperator + +A Logical Operator, AND or OR. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
AND +

An AND operator.

+
OR +

An OR operator.

+
+ +## MediaType + +The type of media + +

Values

+ + + + + + + + + +
ValueDescription
IMAGE +

An image

+
+ +## MLFeatureDataType + +The data type associated with an individual Machine Learning Feature + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
USELESS + +
NOMINAL + +
ORDINAL + +
BINARY + +
COUNT + +
TIME + +
INTERVAL + +
IMAGE + +
VIDEO + +
AUDIO + +
TEXT + +
MAP + +
SEQUENCE + +
SET + +
CONTINUOUS + +
BYTE + +
UNKNOWN + +
+ +## NotebookCellType + +The type for a NotebookCell + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
TEXT_CELL +

TEXT Notebook cell type. The cell context is text only.

+
QUERY_CELL +

QUERY Notebook cell type. The cell context is query only.

+
CHART_CELL +

CHART Notebook cell type. The cell content is chart only.

+
+ +## OperationSourceType + +Enum to define the source/reporter type for an Operation. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
DATA_PROCESS +

A data process reported the operation.

+
DATA_PLATFORM +

A data platform reported the operation.

+
+ +## OperationType + +Enum to define the operation type when an entity changes. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
INSERT +

When data is inserted.

+
UPDATE +

When data is updated.

+
DELETE +

When data is deleted.

+
CREATE +

When table is created.

+
ALTER +

When table is altered

+
DROP +

When table is dropped

+
UNKNOWN +

Unknown operation

+
CUSTOM +

Custom

+
+ +## OriginType + +Enum to define where an entity originated from. + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
NATIVE +

The entity is native to DataHub.

+
EXTERNAL +

The entity is external to DataHub.

+
UNKNOWN +

The entity is of unknown origin.

+
+ +## OwnerEntityType + +Entities that are able to own other entities + +

Values

+ + + + + + + + + + + + + +
ValueDescription
CORP_USER +

A corp user owner

+
CORP_GROUP +

A corp group owner

+
+ +## OwnershipSourceType + +The origin of Ownership metadata associated with a Metadata Entity + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
AUDIT +

Auditing system or audit logs

+
DATABASE +

Database, eg GRANTS table

+
FILE_SYSTEM +

File system, eg file or directory owner

+
ISSUE_TRACKING_SYSTEM +

Issue tracking system, eg Jira

+
MANUAL +

Manually provided by a user

+
SERVICE +

Other ownership like service, eg Nuage, ACL service etc

+
SOURCE_CONTROL +

SCM system, eg GIT, SVN

+
OTHER +

Other sources

+
+ +## OwnershipType + +The type of the ownership relationship between a Person and a Metadata Entity +Note that this field will soon become deprecated due to low usage + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
TECHNICAL_OWNER +

A person or group who is responsible for technical aspects of the asset.

+
BUSINESS_OWNER +

A person or group who is responsible for logical, or business related, aspects of the asset.

+
DATA_STEWARD +

A steward, expert, or delegate responsible for the asset.

+
NONE +

No specific type associated with the owner.

+
CUSTOM +

Associated ownership type is a custom ownership type. Please check OwnershipTypeEntity urn for custom value.

+
DATAOWNER +

A person or group that owns the data. +Deprecated! This ownership type is no longer supported. Use TECHNICAL_OWNER instead.

+
DEVELOPER +

A person or group that is in charge of developing the code +Deprecated! This ownership type is no longer supported. Use TECHNICAL_OWNER instead.

+
DELEGATE +

A person or a group that overseas the operation, eg a DBA or SRE +Deprecated! This ownership type is no longer supported. Use TECHNICAL_OWNER instead.

+
PRODUCER +

A person, group, or service that produces or generates the data +Deprecated! This ownership type is no longer supported. Use TECHNICAL_OWNER instead.

+
STAKEHOLDER +

A person or a group that has direct business interest +Deprecated! Use BUSINESS_OWNER instead.

+
CONSUMER +

A person, group, or service that consumes the data +Deprecated! This ownership type is no longer supported.

+
+ +## PageModuleScope + +Different scopes for where this module is relevant + +

Values

+ + + + + + + + + + + + + +
ValueDescription
PERSONAL +

This module is used for individual use only

+
GLOBAL +

This module is used across users

+
+ +## PageTemplateScope + +Different scopes for where this template is relevant + +

Values

+ + + + + + + + + + + + + +
ValueDescription
PERSONAL +

This template is used for individual use only

+
GLOBAL +

This template is used across users

+
+ +## PageTemplateSurfaceType + +Different surface areas for a page template + +

Values

+ + + + + + + + + + + + + +
ValueDescription
HOME_PAGE +

This template applies to what to display on the home page for users.

+
ASSET_SUMMARY +

This template applies to what to display on asset summary pages

+
+ +## PartitionType + + + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
FULL_TABLE + +
QUERY + +
PARTITION + +
+ +## PatchOperationType + + + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
ADD + +
REMOVE + +
REPLACE + +
MOVE + +
COPY + +
TEST + +
+ +## PersonalSidebarSection + +Variants of APIs used in the Search bar to get data + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
YOUR_GROUPS +

The section containing groups you are in

+
YOUR_ASSETS +

The section containing assets you own

+
YOUR_DOMAINS +

The section containing domains you own

+
YOUR_GLOSSARY_NODES +

The section containing glossary nodes you own

+
YOUR_TAGS +

The section containing tags you own

+
+ +## PlatformNativeType + +Deprecated, do not use this type +The logical type associated with an individual Dataset + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
TABLE +

Table

+
VIEW +

View

+
DIRECTORY +

Directory in file system

+
STREAM +

Stream

+
BUCKET +

Bucket in key value store

+
+ +## PlatformType + +The category of a specific Data Platform + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
FILE_SYSTEM +

Value for a file system

+
KEY_VALUE_STORE +

Value for a key value store

+
MESSAGE_BROKER +

Value for a message broker

+
OBJECT_STORE +

Value for an object store

+
OLAP_DATASTORE +

Value for an OLAP datastore

+
QUERY_ENGINE +

Value for a query engine

+
RELATIONAL_DB +

Value for a relational database

+
SEARCH_ENGINE +

Value for a search engine

+
OTHERS +

Value for other platforms

+
+ +## PolicyMatchCondition + +Match condition + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
EQUALS +

Whether the field matches the value

+
STARTS_WITH +

Whether the field value starts with the value

+
NOT_EQUALS +

Whether the field does not match the value

+
+ +## PolicyState + +The state of an Access Policy + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
DRAFT +

A Policy that has not been officially created, but in progress +Currently unused

+
ACTIVE +

A Policy that is active and being enforced

+
INACTIVE +

A Policy that is not active or being enforced

+
+ +## PolicyType + +The type of the Access Policy + +

Values

+ + + + + + + + + + + + + +
ValueDescription
METADATA +

An access policy that grants privileges pertaining to Metadata Entities

+
PLATFORM +

An access policy that grants top level administrative privileges pertaining to the DataHub Platform itself

+
+ +## PostContentType + +The type of post + +

Values

+ + + + + + + + + + + + + +
ValueDescription
TEXT +

Text content

+
LINK +

Link content

+
+ +## PostType + +The type of post + +

Values

+ + + + + + + + + + + + + +
ValueDescription
HOME_PAGE_ANNOUNCEMENT +

Posts on the home page

+
ENTITY_ANNOUNCEMENT +

Posts on an entity page

+
+ +## PropertyCardinality + +The cardinality of a Structured Property determining whether one or multiple values +can be applied to the entity from this property. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
SINGLE +

Only one value of this property can applied to an entity

+
MULTIPLE +

Multiple values of this property can applied to an entity

+
+ +## QueryLanguage + +A query language / dialect. + +

Values

+ + + + + + + + + +
ValueDescription
SQL +

Standard ANSI SQL

+
+ +## QuerySource + +The source of the query + +

Values

+ + + + + + + + + + + + + +
ValueDescription
MANUAL +

The query was provided manually, e.g. from the UI.

+
SYSTEM +

The query was extracted by the system, e.g. from a dashboard.

+
+ +## RecommendationRenderType + +Enum that defines how the modules should be rendered. +There should be two frontend implementation of large and small modules per type. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
ENTITY_NAME_LIST +

Simple list of entities

+
PLATFORM_SEARCH_LIST +

List of platforms

+
TAG_SEARCH_LIST +

Tag search list

+
SEARCH_QUERY_LIST +

A list of recommended search queries

+
GLOSSARY_TERM_SEARCH_LIST +

Glossary Term search list

+
DOMAIN_SEARCH_LIST +

Domain Search List

+
+ +## RelationshipDirection + +Direction between a source and destination node + +

Values

+ + + + + + + + + + + + + +
ValueDescription
INCOMING +

A directed edge pointing at the source Entity

+
OUTGOING +

A directed edge pointing at the destination Entity

+
+ +## ScenarioType + +Type of the scenario requesting recommendation + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
HOME +

Recommendations to show on the users home page

+
SEARCH_RESULTS +

Recommendations to show on the search results page

+
ENTITY_PROFILE +

Recommendations to show on an Entity Profile page

+
SEARCH_BAR +

Recommendations to show on the search bar when clicked

+
+ +## SchemaAssertionCompatibility + +Defines the required compatibility level for the schema assertion to pass. + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
EXACT_MATCH +

The schema must be exactly the same as the expected schema.

+
SUPERSET +

The schema must be a superset of the expected schema.

+
SUBSET +

The schema must be a subset of the expected schema.

+
+ +## SchemaFieldDataType + +The type associated with a single Dataset schema field + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
BOOLEAN +

A boolean type

+
FIXED +

A fixed bytestring type

+
STRING +

A string type

+
BYTES +

A string of bytes

+
NUMBER +

A number, including integers, floats, and doubles

+
DATE +

A datestrings type

+
TIME +

A timestamp type

+
ENUM +

An enum type

+
NULL +

A NULL type

+
MAP +

A map collection type

+
ARRAY +

An array collection type

+
UNION +

An union type

+
STRUCT +

An complex struct type

+
+ +## SearchBarAPI + +Variants of APIs used in the Search bar to get data + +

Values

+ + + + + + + + + + + + + +
ValueDescription
AUTOCOMPLETE_FOR_MULTIPLE + +
SEARCH_ACROSS_ENTITIES + +
+ +## SortOrder + +Order for sorting + +

Values

+ + + + + + + + + + + + + +
ValueDescription
ASCENDING + +
DESCENDING + +
+ +## SourceCodeUrlType + + + +

Values

+ + + + + + + + + + + + + + + + + +
ValueDescription
ML_MODEL_SOURCE_CODE +

MLModel Source Code

+
TRAINING_PIPELINE_SOURCE_CODE +

Training Pipeline Source Code

+
EVALUATION_PIPELINE_SOURCE_CODE +

Evaluation Pipeline Source Code

+
+ +## SqlAssertionType + +The type of the SQL assertion being monitored. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
METRIC +

A SQL Metric Assertion, e.g. one based on a numeric value returned by an arbitrary SQL query.

+
METRIC_CHANGE +

A SQL assertion that is evaluated against the CHANGE in a metric assertion over time.

+
+ +## StdDataType + +A well-supported, standard DataHub Data Type. + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
STRING +

String data type

+
NUMBER +

Number data type

+
URN +

Urn data type

+
RICH_TEXT +

Rich text data type. Right now this is markdown only.

+
DATE +

Date data type in format YYYY-MM-DD

+
OTHER +

Any other data type - refer to a provided data type urn.

+
+ +## SubResourceType + +A type of Metadata Entity sub resource + +

Values

+ + + + + + + + + +
ValueDescription
DATASET_FIELD +

A Dataset field or column

+
+ +## SummaryElementType + +Different types of elements in asset summaries + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
CREATED + +
LAST_MODIFIED + +
TAGS + +
GLOSSARY_TERMS + +
OWNERS + +
DOMAIN + +
STRUCTURED_PROPERTY + +
DOCUMENT_STATUS + +
DOCUMENT_TYPE + +
+ +## TermRelationshipType + +A type of Metadata Entity sub resource + +

Values

+ + + + + + + + + + + + + +
ValueDescription
isA +

When a Term inherits from, or has an 'Is A' relationship with another Term

+
hasA +

When a Term contains, or has a 'Has A' relationship with another Term

+
+ +## TestResultType + +The result type of a test that has been run + +

Values

+ + + + + + + + + + + + + +
ValueDescription
SUCCESS +

The test succeeded.

+
FAILURE +

The test failed.

+
+ +## TimeRange + +A time range used in fetching Usage statistics + +

Values

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ValueDescription
DAY +

Last day

+
WEEK +

Last week

+
MONTH +

Last month

+
QUARTER +

Last quarter

+
HALF_YEAR +

Last half year

+
YEAR +

Last year

+
ALL +

All time

+
+ +## UploadDownloadScenario + +Enum to specify the context of the upload. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
ASSET_DOCUMENTATION +

Upload for asset documentation.

+
ASSET_DOCUMENTATION_LINKS +

Upload for asset documentation links.

+
+ +## UserSetting + +An individual setting type for a Corp User. + +

Values

+ + + + + + + + + + + + + +
ValueDescription
SHOW_SIMPLIFIED_HOMEPAGE +

Show simplified homepage

+
SHOW_THEME_V2 +

Show theme v2

+
+ +## VolumeAssertionType + +A type of volume (row count) assertion + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
ROW_COUNT_TOTAL +

A volume assertion that is evaluated against the total row count of a dataset.

+
ROW_COUNT_CHANGE +

A volume assertion that is evaluated against an incremental row count of a dataset, +or a row count change.

+
INCREMENTING_SEGMENT_ROW_COUNT_TOTAL +

A volume assertion that checks the latest "segment" in a table based on an incrementing +column to check whether it's row count falls into a particular range. +This can be used to monitor the row count of an incrementing date-partition column segment.

+
INCREMENTING_SEGMENT_ROW_COUNT_CHANGE +

A volume assertion that compares the row counts in neighboring "segments" or "partitions" +of an incrementing column. This can be used to track changes between subsequent date partition +in a table, for example.

+
+ +## WindowDuration + +The duration of a fixed window of time + +

Values

+ + + + + + + + + + + + + + + + + + + + + +
ValueDescription
DAY +

A one day window

+
WEEK +

A one week window

+
MONTH +

A one month window

+
YEAR +

A one year window

+
+ diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/inputObjects.md b/docs-archive/versioned_docs/version-1.5.0/graphql/inputObjects.md new file mode 100644 index 00000000..c768b450 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/inputObjects.md @@ -0,0 +1,11450 @@ +--- +id: inputObjects +title: Input objects +slug: inputObjects +sidebar_position: 7 +--- + +## AcceptRoleInput + +Input provided when accepting a DataHub role using an invite token + +

Arguments

+ + + + + + + + + +
NameDescription
+inviteToken
+String! +		 +

The token needed to accept the role

+
+ +## ActorFilterInput + +Input required when creating or updating an Access Policies Determines which actors the Policy applies to + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+users
+[String!] +		 +

A disjunctive set of users to apply the policy to

+
+groups
+[String!] +		 +

A disjunctive set of groups to apply the policy to

+
+resourceOwners
+Boolean! +		 +

Whether the filter should return TRUE for owners of a particular resource +Only applies to policies of type METADATA, which have a resource associated with them

+
+resourceOwnersTypes
+[String!] +		 +

Set of OwnershipTypes to apply the policy to (if resourceOwners field is set to True)

+
+allUsers
+Boolean! +		 +

Whether the filter should apply to all users

+
+allGroups
+Boolean! +		 +

Whether the filter should apply to all groups

+
+ +## AddBusinessAttributeInput + +Input required to attach Business Attribute +If businessAttributeUrn is null, then it will remove the business attribute from the resource + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+businessAttributeUrn
+String! +		 +

The urn of the business attribute to add

+
+resourceUrn
+[ResourceRefInput!]! +		 +

resource urns to add the business attribute to

+
+ +## AddGroupMembersInput + +Input required to add members to an external DataHub group + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+groupUrn
+String! +		 +

The group to add members to

+
+userUrns
+[String!]! +		 +

The members to add to the group

+
+ +## AddLinkInput + +Input provided when adding the association between a Metadata Entity and a Link + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+linkUrl
+String! +		 +

The url of the link to add or remove

+
+label
+String! +		 +

A label to attach to the link

+
+resourceUrn
+String! +		 +

The urn of the resource or entity to attach the link to, for example a dataset urn

+
+settings
+LinkSettingsInput +		 +

Optional settings input for this link

+
+ +## AddNativeGroupMembersInput + +Input required to add members to a native DataHub group + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+groupUrn
+String! +		 +

The group to add members to

+
+userUrns
+[String!]! +		 +

The members to add to the group

+
+ +## AddOwnerInput + +Input provided when adding the association between a Metadata Entity and an user or group owner + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+ownerUrn
+String! +		 +

The primary key of the Owner to add or remove

+
+ownerEntityType
+OwnerEntityType! +		 +

The owner type, either a user or group

+
+type
+OwnershipType +		 +
Deprecated: No longer supported
+ +

The ownership type for the new owner. If none is provided, then a new NONE will be added. +Deprecated - Use ownershipTypeUrn field instead.

+
+ownershipTypeUrn
+String +		 +

The urn of the ownership type entity.

+
+resourceUrn
+String! +		 +

The urn of the resource or entity to attach or remove the owner from, for example a dataset urn

+
+ +## AddOwnersInput + +Input provided when adding multiple associations between a Metadata Entity and an user or group owner + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+owners
+[OwnerInput!]! +		 +

The primary key of the Owner to add or remove

+
+resourceUrn
+String! +		 +

The urn of the resource or entity to attach or remove the owner from, for example a dataset urn

+
+ +## AddTagsInput + +Input provided when adding tags to an asset + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+tagUrns
+[String!]! +		 +

The primary key of the Tags

+
+resourceUrn
+String! +		 +

The target Metadata Entity to add or remove the Tag to

+
+subResourceType
+SubResourceType +		 +

An optional type of a sub resource to attach the Tag to

+
+subResource
+String +		 +

An optional sub resource identifier to attach the Tag to

+
+ +## AddTermsInput + +Input provided when adding Terms to an asset + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+termUrns
+[String!]! +		 +

The primary key of the Glossary Term to add or remove

+
+resourceUrn
+String! +		 +

The target Metadata Entity to add or remove the Glossary Term from

+
+subResourceType
+SubResourceType +		 +

An optional type of a sub resource to attach the Glossary Term to

+
+subResource
+String +		 +

An optional sub resource identifier to attach the Glossary Term to

+
+ +## AggregateAcrossEntitiesInput + +Input arguments for a full text search query across entities to get aggregations + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+types
+[EntityType!] +		 +

Entity types to be searched. If this is not provided, all entities will be searched.

+
+query
+String! +		 +

The query string

+
+facets
+[String] +		 +

The list of facets to get aggregations for. If list is empty or null, get aggregations for all facets +Sub-aggregations can be specified with the unicode character ␞ (U+241E) as a delimiter between the subtypes. +e.g. _entityType␞owners

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+searchFlags
+SearchFlags +		 +

Flags controlling search options

+
+ +## AllowedValueInput + +An input entry for an allowed value for a structured property + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+stringValue
+String +		 +

The allowed string value if the value is of type string +Either this or numberValue is required.

+
+numberValue
+Float +		 +

The allowed number value if the value is of type number. +Either this or stringValue is required.

+
+description
+String +		 +

The description of this allowed value

+
+ +## AndFilterInput + +A list of disjunctive criterion for the filter. (or operation to combine filters) + +

Arguments

+ + + + + + + + + +
NameDescription
+and
+[FacetFilterInput!] +		 +

A list of and criteria the filter applies to the query

+
+ +## ApplicationEntitiesInput + +Input required to fetch the entities inside of a Application. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+query
+String +		 +

Optional query filter for particular entities inside the Application

+
+start
+Int +		 +

The offset of the result set

+
+count
+Int +		 +

The number of entities to include in result set

+
+filters
+[FacetFilterInput!] +		 +

Optional Facet filters to apply to the result set

+
+ +## ArrayPrimaryKeyInput + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+arrayField
+String! +		 + +
+keys
+[String!]! +		 + +
+ +## AspectParams + +Params to configure what list of aspects should be fetched by the aspects property + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+autoRenderOnly
+Boolean +		 +

Only fetch auto render aspects

+
+aspectNames
+[String!] +		 +

Fetch using aspect names +If absent, returns all aspects matching other inputs

+
+ +## AssertionResultErrorInput + +Input for reporting an Error during Assertion Run + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+type
+AssertionResultErrorType! +		 +

The type of error encountered

+
+message
+String! +		 +

The error message with details of error encountered

+
+ +## AssertionResultInput + +Input for reporting result of the assertion + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+timestampMillis
+Long +		 +

Optional: Provide a timestamp associated with the run event. If not provided, one will be generated for you based +on the current time.

+
+type
+AssertionResultType! +		 +

The final result of assertion, e.g. either SUCCESS or FAILURE.

+
+properties
+[StringMapEntryInput!] +		 +

Additional metadata representing about the native results of the assertion. +These will be displayed alongside the result. +It should be used to capture additional context that is useful for the user.

+
+externalUrl
+String +		 +

Native platform URL of the Assertion Run Event

+
+error
+AssertionResultErrorInput +		 +

Error details, if type is ERROR

+
+ +## AssetCollectionModuleParamsInput + +Input for the params required if the module is type ASSET_COLLECTION + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+assetUrns
+[String!]! +		 +

The list of asset urns for the asset collection module

+
+dynamicFilterJson
+String +		 +

Optional dynamic filters

+

The stringified json representing the logical predicate built in the UI to select assets. +This predicate is turned into orFilters to send through graphql since graphql doesn't support +arbitrary nesting. This string is used to restore the UI for this logical predicate.

+
+ +## AutoCompleteInput + +Input for performing an auto completion query against a single Metadata Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+EntityType +		 +

Entity type to be autocompleted against

+
+query
+String! +		 +

The raw query string

+
+field
+String +		 +

An optional entity field name to autocomplete on

+
+limit
+Int +		 +

The maximum number of autocomplete results to be returned

+
+filters
+[FacetFilterInput!] +		 +

Faceted filters applied to autocomplete results

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+ +## AutoCompleteMultipleInput + +Input for performing an auto completion query against a a set of Metadata Entities + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+types
+[EntityType!] +		 +

Entity types to be autocompleted against +Optional, if none supplied, all searchable types will be autocompleted against

+
+query
+String! +		 +

The raw query string

+
+field
+String +		 +

An optional field to autocomplete against

+
+limit
+Int +		 +

The maximum number of autocomplete results

+
+filters
+[FacetFilterInput!] +		 +

Faceted filters applied to autocomplete results

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+ +## BatchAddOwnersInput + +Input provided when adding owners to a batch of assets + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+owners
+[OwnerInput!]! +		 +

The primary key of the owners

+
+ownershipTypeUrn
+String +		 +

The ownership type to remove, optional. By default will remove regardless of ownership type.

+
+resources
+[ResourceRefInput]! +		 +

The target assets to attach the owners to

+
+ +## BatchAddTagsInput + +Input provided when adding tags to a batch of assets + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+tagUrns
+[String!]! +		 +

The primary key of the Tags

+
+resources
+[ResourceRefInput!]! +		 +

The target assets to attach the tags to

+
+ +## BatchAddTermsInput + +Input provided when adding glossary terms to a batch of assets + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+termUrns
+[String!]! +		 +

The primary key of the Glossary Terms

+
+resources
+[ResourceRefInput]! +		 +

The target assets to attach the glossary terms to

+
+ +## BatchAssignFormInput + +Input for batch assigning a form to different entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+formUrn
+String! +		 +

The urn of the form being assigned to entities

+
+entityUrns
+[String!]! +		 +

The entities that this form is being assigned to

+
+ +## BatchAssignRoleInput + +Input provided when batch assigning a role to a list of users + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+roleUrn
+String +		 +

The urn of the role to assign to the actors. If undefined, will remove the role.

+
+actors
+[String!]! +		 +

The urns of the actors to assign the role to

+
+ +## BatchDatasetUpdateInput + +Arguments provided to batch update Dataset entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Primary key of the Dataset to which the update will be applied

+
+update
+DatasetUpdateInput! +		 +

Arguments provided to update the Dataset

+
+ +## BatchGetStepStatesInput + +Input arguments required for fetching step states + +

Arguments

+ + + + + + + + + +
NameDescription
+ids
+[String!]! +		 +

The unique ids for the steps to retrieve

+
+ +## BatchRemoveFormInput + +Input for batch removing a form from different entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+formUrn
+String! +		 +

The urn of the form being removed from entities

+
+entityUrns
+[String!]! +		 +

The entities that this form is being removed from

+
+ +## BatchRemoveOwnersInput + +Input provided when removing owners from a batch of assets + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+ownerUrns
+[String!]! +		 +

The primary key of the owners

+
+ownershipTypeUrn
+String +		 +

The ownership type to remove, optional. By default will remove regardless of ownership type.

+
+resources
+[ResourceRefInput]! +		 +

The target assets to remove the owners from

+
+ +## BatchRemoveTagsInput + +Input provided when removing tags from a batch of assets + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+tagUrns
+[String!]! +		 +

The primary key of the Tags

+
+resources
+[ResourceRefInput]! +		 +

The target assets to remove the tags from

+
+ +## BatchRemoveTermsInput + +Input provided when removing glossary terms from a batch of assets + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+termUrns
+[String!]! +		 +

The primary key of the Glossary Terms

+
+resources
+[ResourceRefInput]! +		 +

The target assets to remove the glossary terms from

+
+ +## BatchSetApplicationInput + +Input properties required for batch setting a Application on other entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+applicationUrn
+String +		 +

The urn of the application you are setting on a group of resources. +If this is null, the Application will be unset for the given resources.

+
+resourceUrns
+[String!]! +		 +

The urns of the entities the given application should be set on

+
+ +## BatchSetDataProductInput + +Input properties required for batch setting a DataProduct on other entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+dataProductUrn
+String +		 +

The urn of the data product you are setting on a group of resources. +If this is null, the Data Product will be unset for the given resources.

+
+resourceUrns
+[String!]! +		 +

The urns of the entities the given data product should be set on

+
+ +## BatchSetDataProductsInput + +Input for batch adding/removing assets to/from multiple data products. +Available when multipleDataProductsPerAsset feature flag is enabled. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+dataProductUrns
+[String!]! +		 +

The urns of the data products to add/remove assets to/from

+
+resourceUrns
+[String!]! +		 +

The urns of the entities (assets) to add/remove

+
+ +## BatchSetDomainInput + +Input provided when adding tags to a batch of assets + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+domainUrn
+String +		 +

The primary key of the Domain, or null if the domain will be unset

+
+resources
+[ResourceRefInput!]! +		 +

The target assets to attach the Domain

+
+ +## BatchUnsetApplicationInput + +Input properties required for batch unsetting a specific Application from entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+applicationUrn
+String! +		 +

The urn of the application to be removed from the resources

+
+resourceUrns
+[String!]! +		 +

The urns of the entities the given application should be removed from

+
+ +## BatchUpdateDeprecationInput + +Input provided when updating the deprecation status for a batch of assets. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+deprecated
+Boolean! +		 +

Whether the Entity is marked as deprecated.

+
+decommissionTime
+Long +		 +

Optional - The time user plan to decommission this entity

+
+note
+String +		 +

Optional - Additional information about the entity deprecation plan

+
+resources
+[ResourceRefInput]! +		 +

The target assets to attach the tags to

+
+replacement
+String +		 +

Optional - URN to replace the entity with

+
+ +## BatchUpdateSoftDeletedInput + +Input provided when updating the soft-deleted status for a batch of assets + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urns
+[String!]! +		 +

The urns of the assets to soft delete

+
+deleted
+Boolean! +		 +

Whether to mark the asset as soft-deleted (hidden)

+
+ +## BatchUpdateStepStatesInput + +Input arguments required for updating step states + +

Arguments

+ + + + + + + + + +
NameDescription
+states
+[StepStateInput!]! +		 +

Set of step states. If the id does not exist, it will be created.

+
+ +## BrowseInput + +Input required for browse queries + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+EntityType! +		 +

The browse entity type

+
+path
+[String!] +		 +

The browse path

+
+start
+Int +		 +

The starting point of paginated results

+
+count
+Int +		 +

The number of elements included in the results

+
+filters
+[FacetFilterInput!] +		 +
Deprecated: Use `orFilters`- they are more expressive
+ +

Deprecated in favor of the more expressive orFilters field +Facet filters to apply to search results. These will be 'AND'-ed together.

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+ +## BrowsePathsInput + +Inputs for fetching the browse paths for a Metadata Entity + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+type
+EntityType! +		 +

The browse entity type

+
+urn
+String! +		 +

The entity urn

+
+ +## BrowseV2Input + +Input required for browse queries + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+EntityType +		 +

The browse entity type - deprecated use types instead

+
+types
+[EntityType!] +		 +

The browse entity type - deprecated use types instead. If not provided, all types will be used.

+
+path
+[String!] +		 +

The browse path V2 - a list with each entry being part of the browse path V2

+
+start
+Int +		 +

The starting point of paginated results

+
+count
+Int +		 +

The number of elements included in the results

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+query
+String +		 +

The search query string

+
+searchFlags
+SearchFlags +		 +

Flags controlling search options

+
+ +## BusinessAttributeInfoInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

name of the business attribute

+
+description
+String +		 +

description of business attribute

+
+type
+SchemaFieldDataType +		 +

Platform independent field type of the field

+
+ +## CancelIngestionExecutionRequestInput + +Input for cancelling an execution request input + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+ingestionSourceUrn
+String! +		 +

Urn of the ingestion source

+
+executionRequestUrn
+String! +		 +

Urn of the specific execution request to cancel

+
+ +## ChartEditablePropertiesUpdate + +Update to writable Chart fields + +

Arguments

+ + + + + + + + + +
NameDescription
+description
+String! +		 +

Writable description aka documentation for a Chart

+
+ +## ChartUpdateInput + +Arguments provided to update a Chart Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+ownership
+OwnershipUpdate +		 +

Update to ownership

+
+globalTags
+GlobalTagsUpdate +		 +

Deprecated, use tags field instead +Update to global tags

+
+tags
+GlobalTagsUpdate +		 +

Update to tags

+
+editableProperties
+ChartEditablePropertiesUpdate +		 +

Update to editable properties

+
+ +## ContainerEntitiesInput + +Input required to fetch the entities inside of a container. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+query
+String +		 +

Optional query filter for particular entities inside the container

+
+start
+Int +		 +

The offset of the result set

+
+count
+Int +		 +

The number of entities to include in result set

+
+filters
+[FacetFilterInput!] +		 +

Optional Facet filters to apply to the result set

+
+ +## CorpGroupUpdateInput + +Arguments provided to update a CorpGroup Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+description
+String +		 +

DataHub description of the group

+
+slack
+String +		 +

Slack handle for the group

+
+email
+String +		 +

Email address for the group

+
+pictureLink
+String +		 +

A URL which points to a picture which user wants to set as a profile photo

+
+ +## CorpUserUpdateInput + +Arguments provided to update a CorpUser Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+displayName
+String +		 +

Display name to show on DataHub

+
+title
+String +		 +

Title to show on DataHub

+
+aboutMe
+String +		 +

About me section of the user

+
+teams
+[String!] +		 +

Teams that the user belongs to

+
+skills
+[String!] +		 +

Skills that the user possesses

+
+pictureLink
+String +		 +

A URL which points to a picture which user wants to set as a profile photo

+
+slack
+String +		 +

The slack handle of the user

+
+phone
+String +		 +

Phone number for the user

+
+email
+String +		 +

Email address for the user

+
+platformUrns
+[String!] +		 +

The platforms that the user frequently works with

+
+personaUrn
+String +		 +

The user's persona urn"

+
+ +## CreateAccessTokenInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+AccessTokenType! +		 +

The type of the Access Token.

+
+actorUrn
+String! +		 +

The actor associated with the Access Token.

+
+duration
+AccessTokenDuration! +		 +

The duration for which the Access Token is valid.

+
+name
+String! +		 +

The name of the token to be generated.

+
+description
+String +		 +

Description of the token if defined.

+
+ +## CreateApplicationInput + +Input required for creating a Application. + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+properties
+CreateApplicationPropertiesInput! +		 +

Properties about the Application

+
+domainUrn
+String +		 +

An Optional Domain

+
+id
+String +		 +

An optional id for the new application

+
+ +## CreateApplicationPropertiesInput + +Input properties required for creating a Application + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

A display name for the Application

+
+description
+String +		 +

An optional description for the Application

+
+ +## CreateBusinessAttributeInput + +Input required for creating a BusinessAttribute. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Optional! A custom id to use as the primary key identifier. If not provided, a random UUID will be generated as the id.

+
+name
+String! +		 +

name of the business attribute

+
+description
+String +		 +

description of business attribute

+
+type
+SchemaFieldDataType +		 +

Platform independent field type of the field

+
+ +## CreateDataHubFileInput + +Input for creating a DataHub file + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String! +		 +

Unique id for the file (will be used to create the URN)

+
+storageKey
+String! +		 +

The key for where this file is stored inside of the given bucket

+
+originalFileName
+String! +		 +

The original filename as uploaded by the user

+
+mimeType
+String! +		 +

MIME type of the file (e.g., image/png, application/pdf)

+
+sizeInBytes
+Long! +		 +

Size of the file in bytes

+
+scenario
+UploadDownloadScenario! +		 +

The scenario/context in which this file was uploaded

+
+referencedByAsset
+String +		 +

Optional URN of the entity this file is associated with

+
+schemaField
+String +		 +

The dataset schema field urn this file is referenced by

+
+contentHash
+String +		 +

SHA-256 hash of file contents

+
+ +## CreateDataProductInput + +Input required for creating a DataProduct. + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+properties
+CreateDataProductPropertiesInput! +		 +

Properties about the Query

+
+domainUrn
+String! +		 +

The primary key of the Domain

+
+id
+String +		 +

An optional id for the new data product

+
+ +## CreateDataProductPropertiesInput + +Input properties required for creating a DataProduct + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

A display name for the DataProduct

+
+description
+String +		 +

An optional description for the DataProduct

+
+ +## CreateDocumentInput + +Input required to create a new Document + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Optional! A custom id to use as the primary key identifier for the document. +If not provided, a random UUID will be generated as the id.

+
+subType
+String +		 +

Optional sub-type of the Document (e.g., "FAQ", "Tutorial", "Reference"). +If not provided, the document will have no type set.

+
+title
+String +		 +

Optional title for the document

+
+state
+DocumentState +		 +

Optional initial state of the document. Defaults to UNPUBLISHED if not provided.

+
+contents
+DocumentContentInput! +		 +

Content of the Document

+
+owners
+[OwnerInput!] +		 +

Optional owners for the document. If not provided, the creator is automatically added as an owner.

+
+parentDocument
+String +		 +

Optional URN of the parent document

+
+relatedAssets
+[String!] +		 +

Optional URNs of related assets

+
+relatedDocuments
+[String!] +		 +

Optional URNs of related documents

+
+settings
+DocumentSettingsInput +		 +

Optional settings for the document. If not provided, defaults will be used. +Documents created from the UI sidebar will always have showInGlobalContext=true. +API/programmatic clients can set showInGlobalContext=false to create private context documents.

+
+ +## CreateDomainInput + +Input required to create a new Domain. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Optional! A custom id to use as the primary key identifier for the domain. If not provided, a random UUID will be generated as the id.

+
+name
+String! +		 +

Display name for the Domain

+
+description
+String +		 +

Optional description for the Domain

+
+parentDomain
+String +		 +

Optional parent domain urn for the domain

+
+owners
+[OwnerInput!] +		 +

Optional - Add owners to the domain. +If not provided, we'll automatically assign the current actor as the owner.

+
+ +## CreateDynamicFormAssignmentInput + +Input for batch assigning a form to different entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+formUrn
+String! +		 +

The urn of the form being assigned to entities that match some criteria

+
+orFilters
+[AndFilterInput!]! +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters). +Entities that match this filter will have this form applied to them. +Currently, we only support a set of fields to filter on and they are: +(1) platform (2) subType (3) container (4) _entityType (5) domain

+
+ +## CreateFormInput + +Input for batch removing a form from different entities + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Advanced: Optionally provide an ID to create a form urn from

+
+name
+String! +		 +

The name of the form being created

+
+description
+String +		 +

The optional description of the form being created

+
+type
+FormType +		 +

The type of this form, whether it's verification or completion. Default is completion.

+
+prompts
+[CreatePromptInput!] +		 +

The type of this form, whether it's verification or completion. Default is completion.

+
+actors
+FormActorAssignmentInput +		 +

Information on how this form should be assigned to users/groups

+
+ +## CreateGlossaryEntityInput + +Input required to create a new Glossary Entity - a Node or a Term. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Optional! A custom id to use as the primary key identifier for the domain. If not provided, a random UUID will be generated as the id.

+
+name
+String! +		 +

Display name for the Node or Term

+
+description
+String +		 +

Description for the Node or Term

+
+parentNode
+String +		 +

Optional parent node urn for the Glossary Node or Term

+
+ +## CreateGroupInput + +Input for creating a new group + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Optional! A custom id to use as the primary key identifier for the group. If not provided, a random UUID will be generated as the id.

+
+name
+String! +		 +

The display name of the group

+
+description
+String +		 +

The description of the group

+
+ +## CreateIngestionExecutionRequestInput + +Input for creating an execution request input + +

Arguments

+ + + + + + + + + +
NameDescription
+ingestionSourceUrn
+String! +		 +

Urn of the ingestion source to execute

+
+ +## CreateInviteTokenInput + +Input provided when creating an invite token + +

Arguments

+ + + + + + + + + +
NameDescription
+roleUrn
+String +		 +

The urn of the role to create the invite token for

+
+ +## CreateNativeUserResetTokenInput + +Input required to generate a password reset token for a native user. + +

Arguments

+ + + + + + + + + +
NameDescription
+userUrn
+String! +		 +

The urn of the user to reset the password of

+
+ +## CreateOwnershipTypeInput + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the Custom Ownership Type

+
+description
+String! +		 +

The description of the Custom Ownership Type

+
+ +## CreatePostInput + +Input provided when creating a Post + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+postType
+PostType! +		 +

The type of post

+
+content
+UpdatePostContentInput! +		 +

The content of the post

+
+resourceUrn
+String +		 +

Optional target URN for the post

+
+subResourceType
+SubResourceType +		 +

An optional type of a sub resource to attach the Tag to

+
+subResource
+String +		 +

Optional target subresource for the post

+
+ +## CreatePromptInput + +Input for creating form prompts + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Advanced: Optionally provide an ID to this prompt. All prompt IDs must be globally unique.

+
+title
+String! +		 +

The title of the prompt

+
+description
+String +		 +

The optional description of the prompt

+
+type
+FormPromptType! +		 +

The type of the prompt.

+
+structuredPropertyParams
+StructuredPropertyParamsInput +		 +

The params required if this prompt type is STRUCTURED_PROPERTY or FIELDS_STRUCTURED_PROPERTY

+
+required
+Boolean +		 +

Whether this prompt will be required or not. Default is false.

+
+ +## CreateQueryInput + +Input required for creating a Query. Requires the 'Edit Queries' privilege for all query subjects. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+properties
+CreateQueryPropertiesInput! +		 +

Properties about the Query

+
+subjects
+[CreateQuerySubjectInput!]! +		 +

Subjects for the query

+
+ +## CreateQueryPropertiesInput + +Input properties required for creating a Query + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

An optional display name for the Query

+
+description
+String +		 +

An optional description for the Query

+
+statement
+QueryStatementInput! +		 +

The Query contents

+
+ +## CreateQuerySubjectInput + +Input required for creating a Query. For now, only datasets are supported. + +

Arguments

+ + + + + + + + + +
NameDescription
+datasetUrn
+String! +		 +

The urn of the dataset that is the subject of the query

+
+ +## CreateSecretInput + +Input arguments for creating a new Secret + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the secret for reference in ingestion recipes

+
+value
+String! +		 +

The value of the secret, to be encrypted and stored

+
+description
+String +		 +

An optional description for the secret

+
+ +## CreateServiceAccountInput + +Input arguments for creating a service account + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+displayName
+String +		 +

An optional display name for the service account.

+
+description
+String +		 +

An optional description for the service account.

+
+ +## CreateStructuredPropertyInput + +Input for creating a new structured property entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

(Advanced) An optional unique ID to use when creating the urn of this entity

+
+qualifiedName
+String +		 +

The unique fully qualified name of this structured property, dot delimited. +This will be required to match the ID of this structured property.

+
+displayName
+String +		 +

The optional display name for this property

+
+description
+String +		 +

The optional description for this property

+
+immutable
+Boolean +		 +

Whether the property will be mutable once it is applied or not. Default is false.

+
+valueType
+String! +		 +

The urn of the value type that this structured property accepts. +For example: `urn:li:dataType:datahub.string` or `urn:li:dataType:datahub.date`

+
+typeQualifier
+TypeQualifierInput +		 +

The optional input for specifying specific entity types as values

+
+allowedValues
+[AllowedValueInput!] +		 +

The optional input for specifying a list of allowed values

+
+cardinality
+PropertyCardinality +		 +

The optional input for specifying if one or multiple values can be applied. +Default is one value (single cardinality)

+
+entityTypes
+[String!]! +		 +

The list of entity types that this property can be applied to. +For example: ["`urn:li:entityType:datahub.dataset`"]

+
+settings
+StructuredPropertySettingsInput +		 +

Settings for this structured property

+
+ +## CreateTagInput + +Input required to create a new Tag + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Optional! A custom id to use as the primary key identifier for the Tag. If not provided, a random UUID will be generated as the id.

+
+name
+String! +		 +

Display name for the Tag

+
+description
+String +		 +

Optional description for the Tag

+
+ +## CreateTestConnectionRequestInput + +Input for creating a test connection request + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+recipe
+String! +		 +

A JSON-encoded recipe

+
+version
+String +		 +

Advanced: The version of the ingestion framework to use

+
+ +## CreateTestInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Advanced: a custom id for the test.

+
+name
+String! +		 +

The name of the Test

+
+category
+String! +		 +

The category of the Test (user defined)

+
+description
+String +		 +

Description of the test

+
+definition
+TestDefinitionInput! +		 +

The test definition

+
+ +## CreateViewInput + +Input provided when creating a DataHub View + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+viewType
+DataHubViewType! +		 +

The type of View

+
+name
+String! +		 +

The name of the View

+
+description
+String +		 +

An optional description of the View

+
+definition
+DataHubViewDefinitionInput! +		 +

The view definition itself

+
+ +## DashboardEditablePropertiesUpdate + +Update to writable Dashboard fields + +

Arguments

+ + + + + + + + + +
NameDescription
+description
+String! +		 +

Writable description aka documentation for a Dashboard

+
+ +## DashboardUpdateInput + +Arguments provided to update a Dashboard Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+ownership
+OwnershipUpdate +		 +

Update to ownership

+
+globalTags
+GlobalTagsUpdate +		 +

Deprecated, use tags field instead +Update to global tags

+
+tags
+GlobalTagsUpdate +		 +

Update to tags

+
+editableProperties
+DashboardEditablePropertiesUpdate +		 +

Update to editable properties

+
+ +## DataFlowEditablePropertiesUpdate + +Update to writable Data Flow fields + +

Arguments

+ + + + + + + + + +
NameDescription
+description
+String! +		 +

Writable description aka documentation for a Data Flow

+
+ +## DataFlowUpdateInput + +Arguments provided to update a Data Flow aka Pipeline Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+ownership
+OwnershipUpdate +		 +

Update to ownership

+
+globalTags
+GlobalTagsUpdate +		 +

Deprecated, use tags field instead +Update to global tags

+
+tags
+GlobalTagsUpdate +		 +

Update to tags

+
+editableProperties
+DataFlowEditablePropertiesUpdate +		 +

Update to editable properties

+
+ +## DataHubJsonConnectionInput + +The details of a JSON Connection + +

Arguments

+ + + + + + + + + +
NameDescription
+blob
+String! +		 +

The JSON blob containing the specific connection details.

+
+ +## DataHubViewDefinitionInput + +Input required for creating a DataHub View Definition + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+entityTypes
+[EntityType!]! +		 +

A set of entity types that the view applies for. If left empty, then ALL entities will be in scope.

+
+filter
+DataHubViewFilterInput! +		 +

A set of filters to apply.

+
+ +## DataHubViewFilterInput + +Input required for creating a DataHub View Definition + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+operator
+LogicalOperator! +		 +

The operator used to combine the filters.

+
+filters
+[FacetFilterInput!]! +		 +

A set of filters combined via an operator. If left empty, then no filters will be applied.

+
+ +## DataJobEditablePropertiesUpdate + +Update to writable Data Job fields + +

Arguments

+ + + + + + + + + +
NameDescription
+description
+String! +		 +

Writable description aka documentation for a Data Job

+
+ +## DataJobUpdateInput + +Arguments provided to update a Data Job aka Task Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+ownership
+OwnershipUpdate +		 +

Update to ownership

+
+globalTags
+GlobalTagsUpdate +		 +

Deprecated, use tags field instead +Update to global tags

+
+tags
+GlobalTagsUpdate +		 +

Update to tags

+
+editableProperties
+DataJobEditablePropertiesUpdate +		 +

Update to editable properties

+
+ +## DataProductEntitiesInput + +Input required to fetch the entities inside of a Data Product. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+query
+String +		 +

Optional query filter for particular entities inside the Data Product

+
+start
+Int +		 +

The offset of the result set

+
+count
+Int +		 +

The number of entities to include in result set

+
+filters
+[FacetFilterInput!] +		 +

Optional Facet filters to apply to the result set

+
+ +## DataQualityContractInput + +Input required to create a data quality contract + +

Arguments

+ + + + + + + + + +
NameDescription
+assertionUrn
+String! +		 +

The assertion monitoring this part of the data contract. Assertion must be of type Dataset, Volume, Field / Column, or Custom SQL.

+
+ +## DatasetDeprecationUpdate + +An update for the deprecation information for a Metadata Entity + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+deprecated
+Boolean! +		 +

Whether the dataset is deprecated

+
+decommissionTime
+Long +		 +

The time user plan to decommission this dataset

+
+note
+String! +		 +

Additional information about the dataset deprecation plan

+
+ +## DatasetEditablePropertiesUpdate + +Update to writable Dataset fields + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+description
+String! +		 +

Writable description aka documentation for a Dataset

+
+name
+String +		 +

Editable name of the Dataset

+
+ +## DatasetFilterInput + +Input required to create or update a DatasetFilter + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+type
+DatasetFilterType! +		 +

Type of partition

+
+sql
+String +		 +

The raw query if using a SQL FilterType

+
+ +## DatasetUpdateInput + +Arguments provided to update a Dataset Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+ownership
+OwnershipUpdate +		 +

Update to ownership

+
+deprecation
+DatasetDeprecationUpdate +		 +

Update to deprecation status

+
+institutionalMemory
+InstitutionalMemoryUpdate +		 +

Update to institutional memory, ie documentation

+
+globalTags
+GlobalTagsUpdate +		 +

Deprecated, use tags field instead +Update to global tags

+
+tags
+GlobalTagsUpdate +		 +

Update to tags

+
+editableSchemaMetadata
+EditableSchemaMetadataUpdate +		 +

Update to editable schema metadata of the dataset

+
+editableProperties
+DatasetEditablePropertiesUpdate +		 +

Update to editable properties

+
+ +## DeleteFormInput + +Input for deleting a form + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the form that is being deleted

+
+ +## DeletePageModuleInput + +Input for deleting a DataHub page module + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the page module to delete

+
+ +## DeletePageTemplateInput + +Input for deleting a DataHub page template + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the page template to delete

+
+ +## DeleteStructuredPropertyInput + +Input for deleting a form + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the structured properties that is being deleted

+
+ +## DescriptionUpdateInput + +Incubating. Updates the description of a resource. Currently supports DatasetField descriptions only + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+description
+String! +		 +

The new description

+
+resourceUrn
+String! +		 +

The primary key of the resource to attach the description to, eg dataset urn

+
+subResourceType
+SubResourceType +		 +

An optional sub resource type

+
+subResource
+String +		 +

A sub resource identifier, eg dataset field path

+
+ +## DisplayPropertiesUpdateInput + +Update a particular entity's display properties + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+colorHex
+String +		 +

Color associated with the entity in Hex. For example #FFFFFF

+
+icon
+IconPropertiesInput +		 +

The icon associated with the entity

+
+ +## DocumentContentInput + +Input for Document content + +

Arguments

+ + + + + + + + + +
NameDescription
+text
+String! +		 +

The text contents of the Document

+
+ +## DocumentSettingsInput + +Input for Document settings + +

Arguments

+ + + + + + + + + +
NameDescription
+showInGlobalContext
+Boolean +		 +

Whether or not this document should be visible in the global context (e.g., global navigation, knowledge base search). +Defaults to true if not specified.

+
+ +## DomainEntitiesInput + +Input required to fetch the entities inside of a Domain. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+query
+String +		 +

Optional query filter for particular entities inside the domain

+
+start
+Int +		 +

The offset of the result set

+
+count
+Int +		 +

The number of entities to include in result set

+
+filters
+[FacetFilterInput!] +		 +

Optional Facet filters to apply to the result set

+
+ +## EditableSchemaFieldInfoUpdate + +Update to writable schema field metadata + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+fieldPath
+String! +		 +

Flattened name of a field identifying the field the editable info is applied to

+
+description
+String +		 +

Edited description of the field

+
+globalTags
+GlobalTagsUpdate +		 +

Tags associated with the field

+
+ +## EditableSchemaMetadataUpdate + +Update to editable schema metadata of the dataset + +

Arguments

+ + + + + + + + + +
NameDescription
+editableSchemaFieldInfo
+[EditableSchemaFieldInfoUpdate!]! +		 +

Update to writable schema field metadata

+
+ +## EntityCountInput + +Input for the get entity counts endpoint + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+types
+[EntityType!] +		 + +
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+ +## EntityRequestContext + +Context that defines an entity page requesting recommendations + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+type
+EntityType! +		 +

Type of the enity being displayed

+
+urn
+String! +		 +

Urn of the entity being displayed

+
+ +## EntityTypeToPlatforms + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+entityType
+EntityType! +		 +

Entity type to ignore as hops, if no platform is applied applies to all entities of this type.

+
+platforms
+[String!] +		 +

List of platforms to ignore as hops, empty implies all. Must be a valid platform urn

+
+ +## ERModelRelationshipEditablePropertiesUpdate + +Update to writable Dataset fields + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

Display name of the ERModelRelationship

+
+description
+String! +		 +

Writable description for ERModelRelationship

+
+ +## ERModelRelationshipPropertiesInput + +Details about the ERModelRelationship + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Details about the ERModelRelationship

+
+source
+String! +		 +

Details about the ERModelRelationship

+
+destination
+String! +		 +

Details about the ERModelRelationship

+
+relationshipFieldmappings
+[RelationshipFieldMappingInput!] +		 +

Details about the ERModelRelationship

+
+erModelRelationshipCardinality
+ERModelRelationshipCardinality +		 +

optional field to store cardinality of the ERModelRelationship

+
+created
+Boolean +		 +

optional flag about the ERModelRelationship is getting create

+
+createdAt
+Long +		 +

optional field to prevent created time while the ERModelRelationship is getting update

+
+createdBy
+String +		 +

optional field to prevent create actor while the ERModelRelationship is getting update

+
+ +## ERModelRelationshipUpdateInput + +Input required to create/update a new ERModelRelationship + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+properties
+ERModelRelationshipPropertiesInput +		 +

Details about the ERModelRelationship

+
+editableProperties
+ERModelRelationshipEditablePropertiesUpdate +		 +

Update to editable properties

+
+ +## FacetFilterInput + +Facet filters to apply to search results + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+field
+String! +		 +

Name of field to filter by

+
+value
+String +		 +
Deprecated: Prefer `values` for single elements
+ +

Value of the field to filter by. Deprecated in favor of values, which should accept a single element array for a +value

+
+values
+[String!] +		 +

Values, one of which the intended field should match.

+
+negated
+Boolean +		 +

If the filter should or should not be matched

+
+condition
+FilterOperator +		 +

Condition for the values. How to If unset, assumed to be equality

+
+ +## FilterInput + +A set of filter criteria + +

Arguments

+ + + + + + + + + +
NameDescription
+and
+[FacetFilterInput!]! +		 +

A list of conjunctive filters

+
+ +## FormActorAssignmentInput + +Input for assigning a form to actors + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+owners
+Boolean +		 +

Whether this form will be applied to owners of associated entities or not. Default is true.

+
+users
+[String!] +		 +

The optional list of user urns to assign this form to

+
+groups
+[String!] +		 +

The optional list of group urns to assign this form to

+
+ +## FormActorAssignmentUpdateInput + +Update input for assigning a form to actors + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+owners
+Boolean +		 +

Whether this form will be applied to owners of associated entities or not. Default is true.

+
+usersToAdd
+[String!] +		 +

The optional list of user urns to assign this form to

+
+usersToRemove
+[String!] +		 +

The users being removed from being assigned to this form

+
+groupsToAdd
+[String!] +		 +

The optional list of group urns to assign this form to

+
+groupsToRemove
+[String!] +		 +

The groups being removed from being assigned to this form

+
+ +## FreshnessContractInput + +Input required to create an Freshness contract + +

Arguments

+ + + + + + + + + +
NameDescription
+assertionUrn
+String! +		 +

The assertion monitoring this part of the data contract. Assertion must be of type Freshness.

+
+ +## GetAccessTokenInput + +Input required to fetch a new Access Token. + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+AccessTokenType! +		 +

The type of the Access Token.

+
+actorUrn
+String! +		 +

The actor associated with the Access Token.

+
+duration
+AccessTokenDuration! +		 +

The duration for which the Access Token is valid.

+
+ +## GetGrantedPrivilegesInput + +Input for getting granted privileges + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+actorUrn
+String! +		 +

Urn of the actor

+
+resourceSpec
+ResourceSpec +		 +

Spec to identify resource. If empty, gets privileges granted to the actor

+
+includeEvaluationDetails
+Boolean +		 +

Whether to include policy evaluation details. +This will only return result if user has Manage Policies Privilege.

+
+ +## GetInviteTokenInput + +Input provided when getting an invite token + +

Arguments

+ + + + + + + + + +
NameDescription
+roleUrn
+String +		 +

The urn of the role to get the invite token for

+
+ +## GetPresignedUploadUrlInput + +Input for the getUploadPresignedUrl query. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+fileName
+String! +		 +

Original name of the file to upload

+
+scenario
+UploadDownloadScenario! +		 +

The scenario for the upload (e.g., ASSET_DOCUMENTATION, PROFILE_IMAGE).

+
+assetUrn
+String +		 +

The URN of the asset associated with the upload (required when scenario is ASSET_DOCUMENTATION).

+
+schemaFieldUrn
+String +		 +

The dataset schema field this file is referenced by

+
+contentType
+String +		 +

Optional content type of file e.g. "application/pdf"

+
+ +## GetQuickFiltersInput + +Input for getting Quick Filters + +

Arguments

+ + + + + + + + + +
NameDescription
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+ +## GetRootGlossaryEntitiesInput + +Input required when getting Business Glossary entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Glossary Entities in the returned result set

+
+ +## GetSchemaBlameInput + +Input for getting schema changes computed at a specific version. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+datasetUrn
+String! +		 +

The dataset urn

+
+version
+String +		 +

Changes after this version are not shown. If not provided, this is the latestVersion.

+
+ +## GetSchemaVersionListInput + +Input for getting list of schema versions. + +

Arguments

+ + + + + + + + + +
NameDescription
+datasetUrn
+String! +		 +

The dataset urn

+
+ +## GetSecretValuesInput + +Input arguments for retrieving the plaintext values of a set of secrets + +

Arguments

+ + + + + + + + + +
NameDescription
+secrets
+[String!]! +		 +

A list of secret names

+
+ +## GetTimelineInput + +Input for getting timeline from a specific version. +Todo: this is where additional filtering would go such as start & end times/versions, change types, etc + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn to fetch timeline for

+
+changeCategories
+[ChangeCategoryType!] +		 +

The change category types to filter by. If left empty, will fetch all.

+
+ +## GlobalTagsUpdate + +Deprecated, use addTag or removeTag mutation instead +Update to the Tags associated with a Metadata Entity + +

Arguments

+ + + + + + + + + +
NameDescription
+tags
+[TagAssociationUpdate!] +		 +

The new set of tags

+
+ +## GroupingCriterion + +A single grouping criterion for grouping search results + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+baseEntityType
+EntityType +		 +

The base entity type that needs to be grouped +e.g. schemaField +Omitting this field will result in all base entities being grouped into the groupingEntityType.

+
+groupingEntityType
+EntityType! +		 +

The type of entity being grouped into +e.g. dataset, domain, etc.

+
+ +## GroupingSpec + +A grouping specification for search results. + +

Arguments

+ + + + + + + + + +
NameDescription
+groupingCriteria
+[GroupingCriterion!] +		 +

A list of grouping criteria for grouping search results. +There is no implied order in the grouping criteria.

+
+ +## HierarchyViewModuleParamsInput + +Input for the params required if the module is type HIERARCHY_VIEW + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+assetUrns
+[String!]! +		 +

The list of assets to show in the module

+
+showRelatedEntities
+Boolean! +		 +

Whether to show related entities in the module

+
+relatedEntitiesFilterJson
+String +		 +

Optional filters to filter relatedEntities (assetUrns) out

+

The stringified json representing the logical predicate built in the UI to select assets. +This predicate is turned into orFilters to send through graphql since graphql doesn't support +arbitrary nesting. This string is used to restore the UI for this logical predicate.

+
+ +## IconPropertiesInput + +Input for Properties describing an icon associated with an entity + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+iconLibrary
+IconLibrary +		 +

The source of the icon: e.g. Antd, Material, etc

+
+name
+String +		 +

The name of the icon

+
+style
+String +		 +

Any modifier for the icon, this will be library-specific, e.g. filled/outlined, etc

+
+ +## IncidentSourceInput + +Input required to create an incident source + +

Arguments

+ + + + + + + + + +
NameDescription
+type
+IncidentSourceType! +		 +

The type of the incident source

+
+ +## IncidentStatusInput + +Input required to create an incident status + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState! +		 +

The state of the incident

+
+stage
+IncidentStage +		 +

The lifecycle stage ofthe incident

+
+message
+String +		 +

An optional message associated with the status

+
+ +## IngestionSourceSourceInput + +Input for specifying ingestion source source information + +

Arguments

+ + + + + + + + + +
NameDescription
+type
+IngestionSourceSourceType! +		 +

The source type of the ingestion source

+
+ +## InstitutionalMemoryMetadataUpdate + +An institutional memory to add to a Metadata Entity +TODO Add a USER or GROUP actor enum + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+url
+String! +		 +

Link to a document or wiki page or another internal resource

+
+description
+String +		 +

Description of the resource

+
+author
+String! +		 +

The corp user urn of the author of the metadata

+
+createdAt
+Long +		 +

The time at which this metadata was created

+
+ +## InstitutionalMemoryUpdate + +An update for the institutional memory information for a Metadata Entity + +

Arguments

+ + + + + + + + + +
NameDescription
+elements
+[InstitutionalMemoryMetadataUpdate!]! +		 +

The individual references in the institutional memory

+
+ +## LineageEdge + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+downstreamUrn
+String! +		 +

Urn of the source entity. This urn is downstream of the destinationUrn.

+
+upstreamUrn
+String! +		 +

Urn of the destination entity. This urn is upstream of the destinationUrn

+
+ +## LineageFlags + +Flags to control lineage behavior + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entitiesExploredPerHopLimit
+Int +		 +

Limits the number of results explored per hop, still gets all edges each time a hop happens

+
+startTimeMillis
+Long +		 +

An optional starting time to filter on

+
+endTimeMillis
+Long +		 +

An optional ending time to filter on

+
+ignoreAsHops
+[EntityTypeToPlatforms!] +		 +

Map of entity types to platforms to ignore when counting hops during graph walk. Note: this can potentially cause +a large amount of additional hops to occur and should be used with caution.

+
+ +## LineageInput + +Input for the list lineage property of an Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+direction
+LineageDirection! +		 +

The direction of the relationship, either incoming or outgoing from the source entity

+
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+separateSiblings
+Boolean +		 +

Optional flag to not merge siblings in the response. They are merged by default.

+
+startTimeMillis
+Long +		 +

An optional starting time to filter on

+
+endTimeMillis
+Long +		 +

An optional ending time to filter on

+
+includeGhostEntities
+Boolean +		 +

If enabled, include entities that do not exist or are soft deleted.

+
+ +## LinkModuleParamsInput + +Input for the params required if the module is type LINK + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+linkUrl
+String! +		 +

The URL of the link

+
+imageUrl
+String +		 +

The image URL of the link

+
+description
+String +		 +

The description of the link

+
+ +## LinkSettingsInput + + + +

Arguments

+ + + + + + + + + +
NameDescription
+showInAssetPreview
+Boolean +		 +

Determines whether this link should be shown in the asset preview like entity header and search result cards

+
+ +## LinkVersionInput + +Input for linking a versioned entity to a Version Set + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+versionSet
+String! +		 +

The target version set

+
+linkedEntity
+String! +		 +

The target versioned entity to link

+
+version
+String! +		 +

Version Tag label for the version, should be unique within a version set (not enforced)

+
+sourceTimestamp
+Long +		 +

Optional timestamp from the source system

+
+sourceCreator
+String +		 +

Optional creator from the source system, will be converted to an Urn

+
+comment
+String +		 +

Optional comment about the version

+
+ +## ListAccessTokenInput + +Input arguments for listing access tokens + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set, default is 0

+
+count
+Int +		 +

The number of results to be returned, default is 20

+
+filters
+[FacetFilterInput!] +		 +

Facet filters to apply to search results, default is no filters

+
+ +## ListBusinessAttributesInput + +Input provided when listing Business Attribute + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Business Attributes to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+ +## ListDomainsInput + +Input required when listing DataHub Domains + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Domains to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+parentDomain
+String +		 +

Optional parent domain

+
+ +## ListExecutionRequestsInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+query
+String +		 +

An optional search query

+
+filters
+[FacetFilterInput!] +		 +

Optional Facet filters to apply to the result set

+
+sort
+SortCriterion +		 +

Optional sort order. Defaults to use systemCreated.

+
+systemSources
+Boolean +		 +

When defined it adds filter to show executions with only system/non system sources

+
+ +## ListGlobalViewsInput + +Input provided when listing DataHub Global Views + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Views to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+ +## ListGroupsInput + +Input required when listing DataHub Groups + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Policies to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+ +## ListIngestionSourcesInput + +Input arguments for listing Ingestion Sources + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+query
+String +		 +

An optional search query

+
+filters
+[FacetFilterInput!] +		 +

Optional Facet filters to apply to the result set

+
+sort
+SortCriterion +		 +

Optional sort order. Defaults to use systemCreated.

+
+ +## ListMyViewsInput + +Input provided when listing DataHub Views + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Views to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+viewType
+DataHubViewType +		 +

Optional - List the type of View to filter for.

+
+ +## ListOwnershipTypesInput + +Input required for listing custom ownership types entities + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned, default is 0

+
+count
+Int +		 +

The maximum number of Custom Ownership Types to be returned in the result set, default is 20

+
+query
+String +		 +

Optional search query

+
+filters
+[FacetFilterInput!] +		 +

Optional Facet filters to apply to the result set

+
+ +## ListPoliciesInput + +Input required when listing DataHub Access Policies + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Policies to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+ +## ListPostsInput + +Input provided when listing existing posts + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Roles to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+resourceUrn
+String +		 +

Optional resource urn

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+ +## ListQueriesInput + +Input required for listing query entities + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Queries to be returned in the result set

+
+query
+String +		 +

A raw search query

+
+source
+QuerySource +		 +

An optional source for the query

+
+datasetUrn
+String +		 +

An optional Urn for the parent dataset that the query is associated with.

+
+sortInput
+SortQueriesInput +		 +

Optional - Information on how to sort the list queries result

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+ +## ListRecommendationsInput + +Input arguments for fetching UI recommendations + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+userUrn
+String! +		 +

Urn of the actor requesting recommendations

+
+requestContext
+RecommendationRequestContext +		 +

Context provider by the caller requesting recommendations

+
+limit
+Int +		 +

Max number of modules to return

+
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+ +## ListRolesInput + +Input provided when listing existing roles + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Roles to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+ +## ListSecretsInput + +Input for listing DataHub Secrets + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+query
+String +		 +

An optional search query

+
+ +## ListServiceAccountsInput + +Input arguments for listing service accounts + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+query
+String +		 +

Optional search query to filter service accounts

+
+ +## ListTestsInput + +Input required when listing DataHub Tests + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Domains to be returned in the result set

+
+query
+String +		 +

Optional query string to match on

+
+ +## ListUsersInput + +Input required when listing DataHub Users + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Policies to be returned in the result set

+
+query
+String +		 +

Optional search query

+
+ +## MetadataAnalyticsInput + +Input to fetch metadata analytics charts + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+entityType
+EntityType +		 +

Entity type to fetch analytics for (If empty, queries across all entities)

+
+domain
+String +		 +

Urn of the domain to fetch analytics for (If empty or GLOBAL, queries across all domains)

+
+query
+String +		 +

Search query to filter down result (If empty, does not apply any search query)

+
+ +## MoveDocumentInput + +Input required to move a Document to a different parent + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the Document to move

+
+parentDocument
+String +		 +

Optional URN of the new parent document. If null, moves to root level.

+
+ +## MoveDomainInput + +Input for updating the parent domain of a domain. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+parentDomain
+String +		 +

The new parent domain urn. If parentDomain is null, this will remove the parent from this entity

+
+resourceUrn
+String! +		 +

The primary key of the resource to update the parent domain for

+
+ +## NotebookEditablePropertiesUpdate + +Update to writable Notebook fields + +

Arguments

+ + + + + + + + + +
NameDescription
+description
+String! +		 +

Writable description aka documentation for a Notebook

+
+ +## NotebookUpdateInput + +Arguments provided to update a Notebook Entity + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+ownership
+OwnershipUpdate +		 +

Update to ownership

+
+tags
+GlobalTagsUpdate +		 +

Update to tags

+
+editableProperties
+NotebookEditablePropertiesUpdate +		 +

Update to editable properties

+
+ +## OperationsStatsInput + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+range
+TimeRange +		 +

The time range you want to get operations stats for

+
+timeZone
+String +		 +

The optional time zone for aggregating stats. Accepts standard IANA time zone identifier ie. America/New_York

+
+ +## OwnerInput + +Input provided when adding an owner to an asset + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+ownerUrn
+String! +		 +

The primary key of the Owner to add or remove

+
+ownerEntityType
+OwnerEntityType! +		 +

The owner type, either a user or group

+
+type
+OwnershipType +		 +
Deprecated: No longer supported
+ +

The ownership type for the new owner. If none is provided, then a new NONE will be added. +Deprecated - Use ownershipTypeUrn field instead.

+
+ownershipTypeUrn
+String +		 +

The urn of the ownership type entity.

+
+ +## OwnershipUpdate + +An update for the ownership information for a Metadata Entity + +

Arguments

+ + + + + + + + + +
NameDescription
+owners
+[OwnerUpdate!]! +		 +

The updated list of owners

+
+ +## OwnerUpdate + +An owner to add to a Metadata Entity +TODO Add a USER or GROUP actor enum + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+owner
+String! +		 +

The owner URN, either a corpGroup or corpuser

+
+type
+OwnershipType +		 +
Deprecated: No longer supported
+ +

The owner type. Deprecated - Use ownershipTypeUrn field instead.

+
+ownershipTypeUrn
+String +		 +

The urn of the ownership type entity.

+
+ +## PageModuleParamsInput + +Input for the specific parameters stored for a module + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+linkParams
+LinkModuleParamsInput +		 +

The params required if the module is type LINK

+
+richTextParams
+RichTextModuleParamsInput +		 +

The params required if the module is type RICH_TEXT

+
+assetCollectionParams
+AssetCollectionModuleParamsInput +		 +

The params required if the module is type ASSET_COLLECTION

+
+hierarchyViewParams
+HierarchyViewModuleParamsInput +		 +

The params required if the module is type HIERARCHY_VIEW

+
+ +## PageTemplateAssetSummaryInput + +The optional info for asset summaries on this template + +

Arguments

+ + + + + + + + + +
NameDescription
+summaryElements
+[SummaryElementInput!]! +		 +

A list of summary element objects for what to store on asset summaries

+
+ +## PageTemplateRowInput + +Input a row of a page template + +

Arguments

+ + + + + + + + + +
NameDescription
+modules
+[String!]! +		 +

A list of Page Module urns that comprise this row

+
+ +## PatchEntityInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String +		 + +
+entityType
+String +		 + +
+aspectName
+String! +		 + +
+patch
+[PatchOperationInput!]! +		 + +
+arrayPrimaryKeys
+[ArrayPrimaryKeyInput!] +		 + +
+forceGenericPatch
+Boolean +		 + +
+systemMetadata
+SystemMetadataInput +		 + +
+headers
+[StringMapEntryInput!] +		 + +
+ +## PatchOperationInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+op
+PatchOperationType! +		 + +
+path
+String! +		 + +
+value
+String +		 + +
+from
+String +		 + +
+ +## PlatformInput + +Input representing A Data Platform + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String +		 +

Urn of platform

+
+name
+String +		 +

Name of platform

+
+ +## PolicyMatchCriterionInput + +Criterion to define relationship between field and values + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+field
+String! +		 +

The name of the field that the criterion refers to +e.g. entity_type, entity_urn, domain

+
+values
+[String!]! +		 +

Values. Matches criterion if any one of the values matches condition (OR-relationship)

+
+condition
+PolicyMatchCondition! +		 +

The name of the field that the criterion refers to

+
+ +## PolicyMatchFilterInput + +Filter object that encodes a complex filter logic with OR + AND + +

Arguments

+ + + + + + + + + +
NameDescription
+criteria
+[PolicyMatchCriterionInput!] +		 +

List of criteria to apply

+
+ +## PolicyUpdateInput + +Input provided when creating or updating an Access Policy + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+PolicyType! +		 +

The Policy Type

+
+name
+String! +		 +

The Policy name

+
+state
+PolicyState! +		 +

The Policy state

+
+description
+String +		 +

A Policy description

+
+resources
+ResourceFilterInput +		 +

The set of resources that the Policy privileges apply to

+
+privileges
+[String!]! +		 +

The set of privileges that the Policy grants

+
+actors
+ActorFilterInput! +		 +

The set of actors that the Policy privileges are granted to

+
+ +## PropertyValueInput + +Input for collecting structured property values to apply to entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+stringValue
+String +		 +

The string value for this structured property

+
+numberValue
+Float +		 +

The number value for this structured property

+
+ +## QueryStatementInput + +Input required for creating a Query Statement + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+value
+String! +		 +

The query text

+
+language
+QueryLanguage! +		 +

The query language

+
+ +## RaiseIncidentInput + +Input required to create a new incident in the 'Active' state. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+IncidentType! +		 +

The type of incident

+
+customType
+String +		 +

A custom type of incident. Present only if type is 'CUSTOM'

+
+title
+String +		 +

An optional title associated with the incident

+
+description
+String +		 +

An optional description associated with the incident

+
+resourceUrn
+String +		 +

The resource (dataset, dashboard, chart, dataFlow, etc) that the incident is associated with. +This must be present if resourceUrns are not defined.

+
+resourceUrns
+[String!] +		 +

The resources (dataset, dashboard, chart, dataFlow, etc) that the incident is associated with. +This must be present and not empty if resourceUrn is not defined.

+
+startedAt
+Long +		 +

The time at which the incident actually started (may be before the date it was raised).

+
+source
+IncidentSourceInput +		 +

The source of the incident, i.e. how it was generated

+
+status
+IncidentStatusInput +		 +

The status of the incident

+
+priority
+IncidentPriority +		 +

An optional priority for the incident.

+
+assigneeUrns
+[String!] +		 +

An optional set of user or group assignee urns

+
+ +## RecommendationRequestContext + +Context that defines the page requesting recommendations +i.e. for search pages, the query/filters. for entity pages, the entity urn and tab + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+scenario
+ScenarioType! +		 +

Scenario in which the recommendations will be displayed

+
+searchRequestContext
+SearchRequestContext +		 +

Additional context for defining the search page requesting recommendations

+
+entityRequestContext
+EntityRequestContext +		 +

Additional context for defining the entity page requesting recommendations

+
+ +## RelatedDocumentsInput + +Input for querying context documents related to an entity. +The relatedAssets filter is automatically set to the parent entity's URN. +This returns all documents related to the asset, including those hidden from global context. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The maximum number of Documents to return

+
+parentDocuments
+[String!] +		 +

Optional list of parent document URNs to filter by (for batch child lookups)

+
+rootOnly
+Boolean +		 +

If true, only returns documents with no parent (root-level documents). +If false or not provided, returns all documents regardless of parent.

+
+types
+[String!] +		 +

Optional list of document types to filter by (ANDed with other filters)

+
+domains
+[String!] +		 +

Optional list of domain URNs to filter by (ANDed with other filters)

+
+sourceType
+DocumentSourceType +		 +

Optional document source type to filter by.

+
+ +## RelatedTermsInput + +Input provided when adding Terms to an asset + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The Glossary Term urn to add or remove this relationship to/from

+
+termUrns
+[String!]! +		 +

The primary key of the Glossary Term to add or remove

+
+relationshipType
+TermRelationshipType! +		 +

The type of relationship we're adding or removing to/from for a Glossary Term

+
+ +## RelationshipFieldMappingInput + +Details about the ERModelRelationship + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+sourceField
+String +		 +

Details about the ERModelRelationship

+
+destinationField
+String +		 +

Details about the ERModelRelationship

+
+ +## RelationshipsInput + +Input for the list relationships field of an Entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+types
+[String!]! +		 +

The types of relationships to query, representing an OR

+
+direction
+RelationshipDirection! +		 +

The direction of the relationship, either incoming or outgoing from the source entity

+
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+includeSoftDelete
+Boolean +		 +

Whether to include soft-deleted, related, entities

+
+ +## RemoveGroupMembersInput + +Input required to remove members from an external DataHub group + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+groupUrn
+String! +		 +

The group to remove members from

+
+userUrns
+[String!]! +		 +

The members to remove from the group

+
+ +## RemoveLinkInput + +Input provided when removing the association between a Metadata Entity and a Link + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+linkUrl
+String! +		 +

The url of the link to remove, which uniquely identifies the Link

+
+label
+String +		 +

The label of the link to remove, which uniquely identifies the Link

+
+resourceUrn
+String! +		 +

The urn of the resource or entity to attach the link to, for example a dataset urn

+
+ +## RemoveNativeGroupMembersInput + +Input required to remove members from a native DataHub group + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+groupUrn
+String! +		 +

The group to remove members from

+
+userUrns
+[String!]! +		 +

The members to remove from the group

+
+ +## RemoveOwnerInput + +Input provided when removing the association between a Metadata Entity and an user or group owner + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+ownerUrn
+String! +		 +

The primary key of the Owner to add or remove

+
+ownershipTypeUrn
+String +		 +

The ownership type to remove, optional. By default will remove regardless of ownership type.

+
+resourceUrn
+String! +		 +

The urn of the resource or entity to attach or remove the owner from, for example a dataset urn

+
+ +## RemoveStructuredPropertiesInput + +Input for removing structured properties on a given asset + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+assetUrn
+String! +		 +

The urn of the asset that we are removing properties from

+
+structuredPropertyUrns
+[String!]! +		 +

The list of structured properties you want to remove from this asset

+
+ +## ReportOperationInput + +Input provided to report an asset operation + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the asset (e.g. dataset) to report the operation for

+
+operationType
+OperationType! +		 +

The type of operation that was performed. Required

+
+customOperationType
+String +		 +

A custom type of operation. Required if operation type is CUSTOM.

+
+sourceType
+OperationSourceType! +		 +

The source or reporter of the operation

+
+customProperties
+[StringMapEntryInput!] +		 +

A list of key-value parameters to include

+
+partition
+String +		 +

An optional partition identifier

+
+numAffectedRows
+Long +		 +

Optional: The number of affected rows

+
+timestampMillis
+Long +		 +

Optional: Provide a timestamp associated with the operation. If not provided, one will be generated for you based +on the current time.

+
+ +## ResourceFilterInput + +Input required when creating or updating an Access Policies Determines which resources the Policy applies to + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+String +		 +
Deprecated: No longer supported
+ +

Deprecated, use filter field instead +The type of the resource the policy should apply to +Not required because in the future we want to support filtering by type OR by domain

+
+resources
+[String!] +		 +

Deprecated, use filter instead +A list of specific resource urns to apply the filter to

+
+allResources
+Boolean +		 +

Deprecated, use empty filter instead +Whether of not to apply the filter to all resources of the type

+
+filter
+PolicyMatchFilterInput +		 +

Whether of not to apply the filter to all resources of the type

+
+privilegeConstraints
+PolicyMatchFilterInput +		 +

Constraints on what subresources can be acted upon

+
+ +## ResourceRefInput + +Reference to a resource to apply an action to + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+resourceUrn
+String! +		 +

The urn of the resource being referenced

+
+subResourceType
+SubResourceType +		 +

An optional type of a sub resource to attach the Tag to

+
+subResource
+String +		 +

An optional sub resource identifier to attach the Tag to

+
+ +## ResourceSpec + +Spec to identify resource + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+resourceType
+EntityType! +		 +

Resource type

+
+resourceUrn
+String! +		 +

Resource urn

+
+ +## RichTextModuleParamsInput + +Input for the params required if the module is type RICH_TEXT + +

Arguments

+ + + + + + + + + +
NameDescription
+content
+String! +		 +

The content of the rich text module

+
+ +## RollbackIngestionInput + +Input for rolling back an ingestion execution + +

Arguments

+ + + + + + + + + +
NameDescription
+runId
+String! +		 +

An ingestion run ID

+
+ +## SchemaContractInput + +Input required to create a schema contract + +

Arguments

+ + + + + + + + + +
NameDescription
+assertionUrn
+String! +		 +

The assertion monitoring this part of the data contract. Assertion must be of type Data Schema.

+
+ +## ScrollAcrossEntitiesInput + +Input arguments for a full text search query across entities, specifying a starting pointer. Allows paging beyond 10k results + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+types
+[EntityType!] +		 +

Entity types to be searched. If this is not provided, all entities will be searched.

+
+query
+String! +		 +

The query string

+
+scrollId
+String +		 +

The starting point of paginated results, an opaque ID the backend understands as a pointer

+
+keepAlive
+String +		 +

The amount of time to keep the point in time snapshot alive, takes a time unit based string ex: 5m or 30s

+
+count
+Int +		 +

The number of elements included in the results

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+searchFlags
+SearchFlags +		 +

Flags controlling search options

+
+sortInput
+SearchSortInput +		 +

Optional - Information on how to sort this search result

+
+ +## ScrollAcrossLineageInput + +Input arguments for a search query over the results of a multi-hop graph query, uses scroll API + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String +		 +

Urn of the source node

+
+direction
+LineageDirection! +		 +

The direction of the relationship, either incoming or outgoing from the source entity

+
+types
+[EntityType!] +		 +

Entity types to be searched. If this is not provided, all entities will be searched.

+
+query
+String +		 +

The query string

+
+scrollId
+String +		 +

The starting point of paginated results, an opaque ID the backend understands as a pointer

+
+keepAlive
+String +		 +

The amount of time to keep the point in time snapshot alive, takes a time unit based string ex: 5m or 30s

+
+count
+Int +		 +

The number of elements included in the results

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+startTimeMillis
+Long +		 +

An optional starting time to filter on

+
+endTimeMillis
+Long +		 +

An optional ending time to filter on

+
+searchFlags
+SearchFlags +		 +

Flags controlling search options

+
+lineageFlags
+LineageFlags +		 +

Flags controlling the lineage query

+
+ +## SearchAcrossEntitiesInput + +Input arguments for a full text search query across entities + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+types
+[EntityType!] +		 +

Entity types to be searched. If this is not provided, all entities will be searched.

+
+query
+String! +		 +

The query string

+
+start
+Int +		 +

The starting point of paginated results

+
+count
+Int +		 +

The number of elements included in the results

+
+filters
+[FacetFilterInput!] +		 +
Deprecated: Use `orFilters`- they are more expressive
+ +

Deprecated in favor of the more expressive orFilters field +Facet filters to apply to search results. These will be 'AND'-ed together.

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+searchFlags
+SearchFlags +		 +

Flags controlling search options

+
+sortInput
+SearchSortInput +		 +

Optional - Information on how to sort this search result

+
+ +## SearchAcrossLineageInput + +Input arguments for a search query over the results of a multi-hop graph query + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String +		 +

Urn of the source node

+
+direction
+LineageDirection! +		 +

The direction of the relationship, either incoming or outgoing from the source entity

+
+types
+[EntityType!] +		 +

Entity types to be searched. If this is not provided, all entities will be searched.

+
+query
+String +		 +

The query string

+
+start
+Int +		 +

The starting point of paginated results

+
+count
+Int +		 +

The number of elements included in the results

+
+filters
+[FacetFilterInput!] +		 +
Deprecated: Use `orFilters`- they are more expressive
+ +

Deprecated in favor of the more expressive orFilters field +Facet filters to apply to search results. These will be 'AND'-ed together.

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+startTimeMillis
+Long +		 +
Deprecated: Use LineageFlags instead
+ +

An optional starting time to filter on

+
+endTimeMillis
+Long +		 +
Deprecated: Use LineageFlags instead
+ +

An optional ending time to filter on

+
+searchFlags
+SearchFlags +		 +

Flags controlling search options

+
+lineageFlags
+LineageFlags +		 +

Flags controlling the lineage query

+
+viewUrn
+String +		 +

Optional - A View to apply when generating results

+
+sortInput
+SearchSortInput +		 +

Optional - Information on how to sort this search result

+
+ +## SearchDocumentsInput + +Input required when searching Documents. + +Search results are automatically filtered based on document visibility and ownership: +- PUBLISHED documents are returned for all users +- UNPUBLISHED documents are only returned if owned by the current user or a group they belong to + +This ensures private/unpublished documents remain visible only to their owners. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set returned

+
+count
+Int +		 +

The maximum number of Documents to be returned in the result set

+
+query
+String +		 +

Optional semantic search query to search across document contents and metadata

+
+parentDocuments
+[String!] +		 +

Optional list of parent document URNs to filter by (for batch child lookups)

+
+rootOnly
+Boolean +		 +

If true, only returns documents with no parent (root-level documents). +If false or not provided, returns all documents regardless of parent.

+
+types
+[String!] +		 +

Optional list of document types to filter by (ANDed with other filters)

+
+domains
+[String!] +		 +

Optional list of domain URNs to filter by (ANDed with other filters)

+
+relatedAssets
+[String!] +		 +

Optional list of related asset URNs to filter by (ANDed with other filters). +Returns documents that are related to ANY of the provided asset URNs.

+
+sourceType
+DocumentSourceType +		 +

Optional document source type to filter by. +If not provided, searches all documents regardless of source.

+
+ +## SearchFlags + +Set of flags to control search behavior + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+skipCache
+Boolean +		 +

Whether to skip cache

+
+maxAggValues
+Int +		 +

The maximum number of values in an facet aggregation

+
+fulltext
+Boolean +		 +

Structured or unstructured fulltext query

+
+skipHighlighting
+Boolean +		 +

Whether to skip highlighting

+
+skipAggregates
+Boolean +		 +

Whether to skip aggregates/facets

+
+getSuggestions
+Boolean +		 +

Whether to request for search suggestions on the _entityName virtualized field

+
+groupingSpec
+GroupingSpec +		 +

Additional grouping specifications to apply to the search results +Grouping specifications will control how search results are grouped together +in the response. This is currently being used to group schema fields (columns) +as datasets, and in the future will be used to group other entities as well. +Note: This is an experimental feature and is subject to change.

+
+includeSoftDeleted
+Boolean +		 +

Whether to include soft deleted entities

+
+includeRestricted
+Boolean +		 +

Whether to include restricted entities

+
+customHighlightingFields
+[String!] +		 +

fields to include for custom Highlighting

+
+includeStructuredPropertyFacets
+Boolean +		 +

Whether or not to fetch and request for structured property facets when doing a search

+
+filterNonLatestVersions
+Boolean +		 +

Determines whether to filter out any non-latest entity version if entity is part of a Version Set, default true

+
+ +## SearchInput + +Input arguments for a full text search query + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+EntityType! +		 +

The Metadata Entity type to be searched against

+
+query
+String! +		 +

The raw query string

+
+start
+Int +		 +

The offset of the result set

+
+count
+Int +		 +

The number of entities to include in result set

+
+filters
+[FacetFilterInput!] +		 +
Deprecated: Use `orFilters`- they are more expressive
+ +

Deprecated in favor of the more expressive orFilters field +Facet filters to apply to search results. These will be 'AND'-ed together.

+
+orFilters
+[AndFilterInput!] +		 +

A list of disjunctive criterion for the filter. (or operation to combine filters)

+
+searchFlags
+SearchFlags +		 +

Flags controlling search options

+
+ +## SearchRequestContext + +Context that defines a search page requesting recommendatinos + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+query
+String! +		 +

Search query

+
+filters
+[FacetFilterInput!] +		 +

Faceted filters applied to search results

+
+ +## SearchSortInput + +Input required in order to sort search results + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+sortCriterion
+SortCriterion +		 +
Deprecated: Use sortCriteria instead
+ +

A criterion to sort search results on

+
+sortCriteria
+[SortCriterion!] +		 +

A list of values to sort search results on

+
+ +## SetLogicalParentInput + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+resourceUrn
+String! +		 + +
+parentUrn
+String +		 + +
+ +## SortCriterion + +A single sorting criterion for sorting search. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+field
+String! +		 +

A field upon which we'll do sorting on.

+
+sortOrder
+SortOrder! +		 +

The order in which we will be sorting

+
+ +## SortQueriesInput + + + +

Arguments

+ + + + + + + + + +
NameDescription
+sortCriterion
+SortCriterion! +		 +

A criterion to sort query results on

+
+ +## StepStateInput + +The input required to update the state of a step + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+id
+String! +		 +

The globally unique id for the step

+
+properties
+[StringMapEntryInput]! +		 +

The new properties for the step

+
+ +## StringMapEntryInput + +String map entry input + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+key
+String! +		 +

The key of the map entry

+
+value
+String +		 +

The value fo the map entry

+
+ +## StructuredPropertyInputParams + +A prompt shown to the user to collect metadata about an entity + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+structuredPropertyUrn
+String! +		 +

The urn of the structured property being applied to an entity

+
+values
+[PropertyValueInput!]! +		 +

The list of values you want to apply on this structured property to an entity

+
+ +## StructuredPropertyParamsInput + +Input for a structured property type prompt + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the structured property for a given form prompt

+
+ +## StructuredPropertySettingsInput + +Settings for a structured property + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+isHidden
+Boolean +		 +

Whether or not this asset should be hidden in the main application

+
+showInSearchFilters
+Boolean +		 +

Whether or not this asset should be displayed as a search filter

+
+showInAssetSummary
+Boolean +		 +

Whether or not this asset should be displayed in the asset sidebar

+
+hideInAssetSummaryWhenEmpty
+Boolean +		 +

Whether or not this asset should be hidden in the asset sidebar (showInAssetSummary should be enabled) +when its value is empty

+
+showAsAssetBadge
+Boolean +		 +

Whether or not this asset should be displayed as an asset badge on other asset's headers

+
+showInColumnsTable
+Boolean +		 +

Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab.

+
+ +## SubmitFormPromptInput + +Input for responding to a singular prompt in a form + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+promptId
+String! +		 +

The unique ID of the prompt this input is responding to

+
+formUrn
+String! +		 +

The urn of the form that this prompt is a part of

+
+type
+FormPromptType! +		 +

The type of prompt that this input is responding to

+
+fieldPath
+String +		 +

The fieldPath on a schema field that this prompt submission is association with. +This should be provided when the prompt is type FIELDS_STRUCTURED_PROPERTY

+
+structuredPropertyParams
+StructuredPropertyInputParams +		 +

The structured property required for the prompt on this entity

+
+ +## SummaryElementInput + +A summary element object for what to store on asset summaries + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+elementType
+SummaryElementType! +		 +

The summary element type

+
+structuredPropertyUrn
+String +		 +

The optional urn of the structured property for this element if elementType is STRUCTURED_PROPERTY

+
+ +## SystemMetadataInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+lastObserved
+Long +		 + +
+runId
+String +		 + +
+properties
+[StringMapEntryInput!] +		 + +
+ +## TagAssociationInput + +Input provided when updating the association between a Metadata Entity and a Tag + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+tagUrn
+String! +		 +

The primary key of the Tag to add or remove

+
+resourceUrn
+String! +		 +

The target Metadata Entity to add or remove the Tag to

+
+subResourceType
+SubResourceType +		 +

An optional type of a sub resource to attach the Tag to

+
+subResource
+String +		 +

An optional sub resource identifier to attach the Tag to

+
+ +## TagAssociationUpdate + +Deprecated, use addTag or removeTag mutation instead +A tag update to be applied + +

Arguments

+ + + + + + + + + +
NameDescription
+tag
+TagUpdateInput! +		 +

The tag being applied

+
+ +## TagUpdateInput + +Deprecated, use addTag or removeTag mutations instead +An update for a particular Tag entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Tag

+
+name
+String! +		 +

The display name of a Tag

+
+description
+String +		 +

Description of the tag

+
+ownership
+OwnershipUpdate +		 +

Ownership metadata of the tag

+
+ +## TermAssociationInput + +Input provided when updating the association between a Metadata Entity and a Glossary Term + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+termUrn
+String! +		 +

The primary key of the Glossary Term to add or remove

+
+resourceUrn
+String! +		 +

The target Metadata Entity to add or remove the Glossary Term from

+
+subResourceType
+SubResourceType +		 +

An optional type of a sub resource to attach the Glossary Term to

+
+subResource
+String +		 +

An optional sub resource identifier to attach the Glossary Term to

+
+ +## TestDefinitionInput + + + +

Arguments

+ + + + + + + + + +
NameDescription
+json
+String +		 +

The string representation of the Test +Deprecated! JSON representation is no longer supported.

+
+ +## TypeQualifierInput + +Input for specifying specific entity types as values + +

Arguments

+ + + + + + + + + +
NameDescription
+allowedTypes
+[String!] +		 +

The list of allowed entity types as urns (ie. ["`urn:li:entityType:datahub.corpuser`"])

+
+ +## UnlinkVersionInput + +Input for unlinking a versioned entity from a Version Set + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+versionSet
+String +		 +

The target version set

+
+unlinkedEntity
+String +		 +

The target versioned entity to unlink

+
+ +## UpdateApplicationInput + +Input properties required for update a Application + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

A display name for the Application

+
+description
+String +		 +

An optional description for the Application

+
+ +## UpdateApplicationsSettingsInput + +Input required to update global applications settings. + +

Arguments

+ + + + + + + + + +
NameDescription
+enabled
+Boolean +		 +

Whether the Applications feature is enabled

+
+ +## UpdateAssetSettingsInput + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the asset you are updating

+
+summary
+UpdateAssetSummaryInput +		 +

Input related to the summary page of this asset

+
+ +## UpdateAssetSummaryInput + +Input related to the summary page of this asset + +

Arguments

+ + + + + + + + + +
NameDescription
+template
+String! +		 +

The urn of the template you want to set for this asset summary page

+
+ +## UpdateBusinessAttributeInput + +Input required to update Business Attribute + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

name of the business attribute

+
+description
+String +		 +

business attribute description

+
+type
+SchemaFieldDataType +		 +

type

+
+ +## UpdateCorpUserViewsSettingsInput + +Input required to update a users settings. + +

Arguments

+ + + + + + + + + +
NameDescription
+defaultView
+String +		 +

The URN of the View that serves as this user's personal default. +If not provided, any existing default view will be removed.

+
+ +## UpdateDataProductInput + +Input properties required for update a DataProduct + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

A display name for the DataProduct

+
+description
+String +		 +

An optional description for the DataProduct

+
+ +## UpdateDeprecationInput + +Input provided when setting the Deprecation status for an Entity. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the Entity to set deprecation for.

+
+subResourceType
+SubResourceType +		 +

An optional type of a sub resource to set the deprecation for

+
+subResource
+String +		 +

An optional sub resource identifier to set the deprecation for

+
+deprecated
+Boolean! +		 +

Whether the Entity is marked as deprecated.

+
+decommissionTime
+Long +		 +

Optional - The time user plan to decommission this entity

+
+note
+String +		 +

Optional - Additional information about the entity deprecation plan

+
+replacement
+String +		 +

Optional - URN to replace the entity with

+
+ +## UpdateDocPropagationSettingsInput + +Input required to update doc propagation settings. + +

Arguments

+ + + + + + + + + +
NameDescription
+docColumnPropagation
+Boolean +		 +

The default doc propagation setting for the platform.

+
+ +## UpdateDocumentContentsInput + +Input required to update the contents of a Document + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the Document to update

+
+title
+String +		 +

Optional updated title for the document. If not provided, the existing title will not be updated.

+
+contents
+DocumentContentInput +		 +

The new text contents for the Document. If not provided, the existing contents will not be updated.

+
+subType
+String +		 +

Optional updated sub-type for the document (e.g., "FAQ", "Tutorial", "Reference")

+
+ +## UpdateDocumentRelatedEntitiesInput + +Input required to update the related entities of a Document + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the Document to update

+
+relatedAssets
+[String!] +		 +

Optional URNs of related assets (will replace existing)

+
+relatedDocuments
+[String!] +		 +

Optional URNs of related documents (will replace existing)

+
+ +## UpdateDocumentSettingsInput + +Input required to update the settings of a Document + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the Document to update

+
+showInGlobalContext
+Boolean! +		 +

Whether or not this document should be visible in the global context

+
+ +## UpdateDocumentStatusInput + +Input required to update the status of a Document + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the Document to update

+
+state
+DocumentState! +		 +

The new state for the document

+
+ +## UpdateDocumentSubTypeInput + +Input required to update the sub-type of a Document + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN of the Document to update

+
+subType
+String +		 +

The new sub-type for the document (e.g., "FAQ", "Tutorial", "Runbook"). Set to null to clear the sub-type.

+
+ +## UpdateEmbedInput + +Input required to set or clear information related to rendering a Data Asset inside of DataHub. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The URN associated with the Data Asset to update. Only dataset, dashboard, and chart urns are currently supported.

+
+renderUrl
+String +		 +

Set or clear a URL used to render an embedded asset.

+
+ +## UpdateFormInput + +Input for updating a form + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the form being updated

+
+name
+String +		 +

The new name of the form

+
+description
+String +		 +

The new description of the form

+
+type
+FormType +		 +

The new type of the form

+
+promptsToAdd
+[CreatePromptInput!] +		 +

The new prompts being added to this form

+
+promptsToRemove
+[String!] +		 +

The IDs of the prompts to remove from this form

+
+actors
+FormActorAssignmentUpdateInput +		 +

Information on how this form should be assigned to users/groups

+
+ +## UpdateGlobalViewsSettingsInput + +Input required to update Global View Settings. + +

Arguments

+ + + + + + + + + +
NameDescription
+defaultView
+String +		 +

The URN of the View that serves as the Global, or organization-wide, default. +If this field is not provided, the existing Global Default will be cleared.

+
+ +## UpdateIncidentInput + +Input required to update an existing incident. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+title
+String +		 +

An optional title associated with the incident

+
+description
+String +		 +

An optional description associated with the incident

+
+startedAt
+Long +		 +

An optional time at which the incident actually started (may be before the date it was raised).

+
+status
+IncidentStatusInput +		 +

The status of the incident

+
+priority
+IncidentPriority +		 +

An optional priority for the incident.

+
+resourceUrns
+[String!] +		 +

An optional set of resources that the incident is assigned to. +If defined, there must be at least one in the list.

+
+assigneeUrns
+[String!] +		 +

An optional set of user or group assignee urns

+
+ +## UpdateIncidentStatusInput + +Input required to update status of an existing incident + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState! +		 +

The new state of the incident

+
+stage
+IncidentStage +		 +

Optional - The new lifecycle stage of the incident

+
+message
+String +		 +

An optional message associated with the new state

+
+ +## UpdateIngestionSourceConfigInput + +Input parameters for creating / updating an Ingestion Source + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+recipe
+String! +		 +

A JSON-encoded recipe

+
+version
+String +		 +

The version of DataHub Ingestion Framework to use when executing the recipe.

+
+executorId
+String! +		 +

The id of the executor to use for executing the recipe

+
+debugMode
+Boolean +		 +

Whether or not to run ingestion in debug mode

+
+extraArgs
+[StringMapEntryInput!] +		 +

Extra arguments for the ingestion run.

+
+ +## UpdateIngestionSourceInput + +Input arguments for creating / updating an Ingestion Source + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

A name associated with the ingestion source

+
+type
+String! +		 +

The type of the source itself, e.g. mysql, bigquery, bigquery-usage. Should match the recipe.

+
+description
+String +		 +

An optional description associated with the ingestion source

+
+schedule
+UpdateIngestionSourceScheduleInput +		 +

An optional schedule for the ingestion source. If not provided, the source is only available for run on-demand.

+
+config
+UpdateIngestionSourceConfigInput! +		 +

A set of type-specific ingestion source configurations

+
+source
+IngestionSourceSourceInput +		 +

Optionally specify source

+
+ +## UpdateIngestionSourceScheduleInput + +Input arguments for creating / updating the schedule of an Ingestion Source + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+interval
+String! +		 +

The cron-formatted interval describing when the job should be executed

+
+timezone
+String! +		 +

The name of the timezone in which the cron interval should be scheduled (e.g. America/Los Angeles)

+
+ +## UpdateLineageInput + +Input required in order to upsert lineage edges + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+edgesToAdd
+[LineageEdge]! +		 +

New lineage edges to upsert

+
+edgesToRemove
+[LineageEdge]! +		 +

Lineage edges to remove. Takes precedence over edgesToAdd - so edges existing both edgesToAdd +and edgesToRemove will be removed.

+
+ +## UpdateLinkInput + +Input provided when updating the association between a Metadata Entity and a Link + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+currentUrl
+String! +		 +

Current url of the link

+
+currentLabel
+String! +		 +

Current label of the link

+
+linkUrl
+String! +		 +

The new url of the link

+
+label
+String! +		 +

The new label of the link

+
+settings
+LinkSettingsInput +		 +

The new optional settings input for the link

+
+resourceUrn
+String! +		 +

The urn of the resource or entity to attach the link to, for example a dataset urn

+
+ +## UpdateMediaInput + +Input provided for filling in a post content + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+type
+MediaType! +		 +

The type of media

+
+location
+String! +		 +

The location of the media (a URL)

+
+ +## UpdateNameInput + +Input for updating the name of an entity + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The new name

+
+urn
+String! +		 +

The primary key of the resource to update the name for

+
+ +## UpdateOwnershipTypeInput + + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

The name of the Custom Ownership Type

+
+description
+String +		 +

The description of the Custom Ownership Type

+
+ +## UpdateParentNodeInput + +Input for updating the parent node of a resource. Currently only GlossaryNodes and GlossaryTerms have parentNodes. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+parentNode
+String +		 +

The new parent node urn. If parentNode is null, this will remove the parent from this entity

+
+resourceUrn
+String! +		 +

The primary key of the resource to update the parent node for

+
+ +## UpdatePostContentInput + +Input provided for filling in a post content + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+contentType
+PostContentType! +		 +

The type of post content

+
+title
+String! +		 +

The title of the post

+
+description
+String +		 +

Optional content of the post

+
+link
+String +		 +

Optional link that the post is associated with

+
+media
+UpdateMediaInput +		 +

Optional media contained in the post

+
+ +## UpdatePostInput + +Input provided when creating a Post + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the post to edit or update

+
+postType
+PostType! +		 +

The type of post

+
+content
+UpdatePostContentInput! +		 +

The content of the post

+
+ +## UpdateQueryInput + +Input required for updating an existing Query. Requires the 'Edit Queries' privilege for all query subjects. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+properties
+UpdateQueryPropertiesInput +		 +

Properties about the Query

+
+subjects
+[UpdateQuerySubjectInput!] +		 +

Subjects for the query

+
+ +## UpdateQueryPropertiesInput + +Input properties required for creating a Query. Any non-null fields will be updated if provided. + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

An optional display name for the Query

+
+description
+String +		 +

An optional description for the Query

+
+statement
+QueryStatementInput +		 +

The Query contents

+
+ +## UpdateQuerySubjectInput + +Input required for creating a Query. For now, only datasets are supported. + +

Arguments

+ + + + + + + + + +
NameDescription
+datasetUrn
+String! +		 +

The urn of the dataset that is the subject of the query

+
+ +## UpdateSecretInput + +Input arguments for updating a Secret + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Secret to update

+
+name
+String! +		 +

The name of the secret for reference in ingestion recipes

+
+value
+String! +		 +

The value of the secret, to be encrypted and stored

+
+description
+String +		 +

An optional description for the secret

+
+ +## UpdateStructuredPropertyInput + +Input for updating an existing structured property entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the structured property being updated

+
+displayName
+String +		 +

The optional display name for this property

+
+description
+String +		 +

The optional description for this property

+
+immutable
+Boolean +		 +

Whether the property will be mutable once it is applied or not. Default is false.

+
+typeQualifier
+UpdateTypeQualifierInput +		 +

The optional input for specifying specific entity types as values

+
+newAllowedValues
+[AllowedValueInput!] +		 +

Append to the list of allowed values for this property. +For backwards compatibility, this is append only.

+
+setCardinalityAsMultiple
+Boolean +		 +

Set to true if you want to change the cardinality of this structured property +to multiple. Cannot change from multiple to single for backwards compatibility reasons.

+
+newEntityTypes
+[String!] +		 +

Append to the list of entity types that this property can be applied to. +For backwards compatibility, this is append only.

+
+settings
+StructuredPropertySettingsInput +		 +

Settings for this structured property

+
+ +## UpdateTestInput + + + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the Test

+
+category
+String! +		 +

The category of the Test (user defined)

+
+description
+String +		 +

Description of the test

+
+definition
+TestDefinitionInput! +		 +

The test definition

+
+ +## UpdateTypeQualifierInput + +Input for updating specifying specific entity types as values + +

Arguments

+ + + + + + + + + +
NameDescription
+newAllowedTypes
+[String!] +		 +

Append to the list of allowed entity types as urns for this property (ie. ["`urn:li:entityType:datahub.corpuser`"]) +For backwards compatibility, this is append only.

+
+ +## UpdateUserHomePageSettingsInput + +Input required to update a user's home page settings. + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+pageTemplate
+String +		 +

The URN of the page template to be rendered on the home page for the user.

+
+newDismissedAnnouncements
+[String] +		 +

The list of urns of announcement posts dismissed by the user.

+
+removePageTemplate
+Boolean +		 +

Whether to remove the page template for the user.

+
+ +## UpdateUserSettingInput + +Input for updating a user setting + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+name
+UserSetting! +		 +

The name of the setting

+
+value
+Boolean! +		 +

The new value of the setting

+
+ +## UpdateViewInput + +Input provided when updating a DataHub View + +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

The name of the View

+
+description
+String +		 +

An optional description of the View

+
+definition
+DataHubViewDefinitionInput +		 +

The view definition itself

+
+ +## UpsertCustomAssertionInput + +Input for upserting a Custom Assertion. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entityUrn
+String! +		 +

The entity targeted by this assertion.

+
+type
+String! +		 +

The type of the custom assertion.

+
+description
+String! +		 +

The description of this assertion.

+
+fieldPath
+String +		 +

The dataset field targeted by this assertion, if any.

+
+platform
+PlatformInput! +		 +

The external Platform associated with the assertion

+
+externalUrl
+String +		 +

Native platform URL of the Assertion

+
+logic
+String +		 +

Logic comprising a raw, unstructured assertion. for example - custom SQL query for the assertion.

+
+ +## UpsertDataContractInput + +Input required to upsert a Data Contract entity for an asset + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entityUrn
+String! +		 +

The urn of the related entity. Dataset is the only entity type supported today.

+
+freshness
+[FreshnessContractInput!] +		 +

The Freshness / Freshness portion of the contract. If not provided, this will be set to none. +For Dataset Contracts, it is expected that there will not be more than 1 Freshness contract. If there are, only the first will be displayed.

+
+schema
+[SchemaContractInput!] +		 +

The schema / structural portion of the contract. If not provided, this will be set to none. +For Dataset Contracts, it is expected that there will not be more than 1 Schema contract. If there are, only the first will be displayed.

+
+dataQuality
+[DataQualityContractInput!] +		 +

The data quality portion of the contract. If not provided, this will be set to none.

+
+state
+DataContractState +		 +

The state of the data contract. If not provided, it will be in ACTIVE mode by default.

+
+id
+String +		 +

Optional ID of the contract you want to create. Only applicable if this is a create operation. If not provided, a random +id will be generated for you.

+
+ +## UpsertDataHubConnectionInput + +Input required to upsert a new DataHub connection. + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

An optional ID to use when creating the URN of the connection. If none is provided, +a random UUID will be generated automatically.

+
+type
+DataHubConnectionDetailsType! +		 +

The type or format of connection

+
+platformUrn
+String! +		 +

Urn of the associated platform

+
+json
+DataHubJsonConnectionInput +		 +

A JSON-encoded connection. This must be present when type is JSON.

+
+name
+String +		 +

An optional name for this connection entity

+
+ +## UpsertLinkInput + +Input provided when upsert the association between a Metadata Entity and a Link + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+linkUrl
+String! +		 +

The url of the link to add or update

+
+label
+String! +		 +

The url of the link to add or update

+
+resourceUrn
+String! +		 +

The urn of the resource or entity to attach the link to, for example a dataset urn

+
+settings
+LinkSettingsInput +		 +

Optional settings input for this link

+
+ +## UpsertPageModuleInput + +Input for creating or updating a DataHub page module + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String +		 +

The URN of the page module to update. If not provided, a new module will be created.

+
+name
+String! +		 +

The display name of this module

+
+type
+DataHubPageModuleType! +		 +

The type of this module

+
+scope
+PageModuleScope! +		 +

The scope of this module and who can use/see it

+
+params
+PageModuleParamsInput! +		 +

The specific parameters stored for this module

+
+ +## UpsertPageTemplateInput + +Input for adding or updating a DataHubPageTemplate entity + +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String +		 +

The urn of the page template if updating, empty if creating a new page template

+
+rows
+[PageTemplateRowInput!]! +		 +

The rows of a page template

+
+assetSummary
+PageTemplateAssetSummaryInput +		 +

The optional info for asset summaries on this template

+
+scope
+PageTemplateScope! +		 +

The scope of the template ie. is it personal or global

+
+surfaceType
+PageTemplateSurfaceType! +		 +

The area that this template is used in

+
+ +## UpsertStructuredPropertiesInput + +Input for upserting structured properties on a given asset + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+assetUrn
+String! +		 +

The urn of the asset that we are updating

+
+structuredPropertyInputParams
+[StructuredPropertyInputParams!]! +		 +

The list of structured properties you want to upsert on this asset

+
+ +## VerifyFormInput + +Input for verifying forms on entities + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+formUrn
+String! +		 +

The urn of the form being verified on an entity

+
+entityUrn
+String! +		 +

The urn of the entity that is having a form verified on it

+
+ diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/interfaces.md b/docs-archive/versioned_docs/version-1.5.0/graphql/interfaces.md new file mode 100644 index 00000000..b29c058b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/interfaces.md @@ -0,0 +1,418 @@ +--- +id: interfaces +title: Interfaces +slug: interfaces +sidebar_position: 4 +--- + +## Aspect + +A versioned aspect, or single group of related metadata, associated with an Entity and having a unique version + +

Implemented by

+ +- [SchemaMetadata](/docs/graphql/objects#schemametadata) + +

Fields

+ + + + + + + + + +
NameDescription
+version
+Long +		 +

The version of the aspect, where zero represents the latest version

+
+ +## BrowsableEntity + +A Metadata Entity which is browsable, or has browse paths. + +

Implemented by

+ +- [Dataset](/docs/graphql/objects#dataset) +- [Notebook](/docs/graphql/objects#notebook) +- [Dashboard](/docs/graphql/objects#dashboard) +- [Chart](/docs/graphql/objects#chart) +- [DataFlow](/docs/graphql/objects#dataflow) +- [DataJob](/docs/graphql/objects#datajob) +- [MLModel](/docs/graphql/objects#mlmodel) +- [MLModelGroup](/docs/graphql/objects#mlmodelgroup) +- [MLFeatureTable](/docs/graphql/objects#mlfeaturetable) + +

Fields

+ + + + + + + + + +
NameDescription
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to an entity. If no Browse Paths have been generated before, this will be null.

+
+ +## Entity + +A top level Metadata Entity + +

Implemented by

+ +- [Assertion](/docs/graphql/objects#assertion) +- [AccessTokenMetadata](/docs/graphql/objects#accesstokenmetadata) +- [ServiceAccount](/docs/graphql/objects#serviceaccount) +- [DataHubConnection](/docs/graphql/objects#datahubconnection) +- [Dataset](/docs/graphql/objects#dataset) +- [DataContract](/docs/graphql/objects#datacontract) +- [Document](/docs/graphql/objects#document) +- [ERModelRelationship](/docs/graphql/objects#ermodelrelationship) +- [Role](/docs/graphql/objects#role) +- [VersionedDataset](/docs/graphql/objects#versioneddataset) +- [GlossaryTerm](/docs/graphql/objects#glossaryterm) +- [GlossaryNode](/docs/graphql/objects#glossarynode) +- [DataPlatform](/docs/graphql/objects#dataplatform) +- [DataPlatformInstance](/docs/graphql/objects#dataplatforminstance) +- [Container](/docs/graphql/objects#container) +- [SchemaFieldEntity](/docs/graphql/objects#schemafieldentity) +- [CorpUser](/docs/graphql/objects#corpuser) +- [CorpGroup](/docs/graphql/objects#corpgroup) +- [Tag](/docs/graphql/objects#tag) +- [Notebook](/docs/graphql/objects#notebook) +- [Dashboard](/docs/graphql/objects#dashboard) +- [Chart](/docs/graphql/objects#chart) +- [DataFlow](/docs/graphql/objects#dataflow) +- [DataJob](/docs/graphql/objects#datajob) +- [DataProcessInstance](/docs/graphql/objects#dataprocessinstance) +- [DataHubPolicy](/docs/graphql/objects#datahubpolicy) +- [MLModel](/docs/graphql/objects#mlmodel) +- [MLModelGroup](/docs/graphql/objects#mlmodelgroup) +- [MLFeature](/docs/graphql/objects#mlfeature) +- [MLPrimaryKey](/docs/graphql/objects#mlprimarykey) +- [MLFeatureTable](/docs/graphql/objects#mlfeaturetable) +- [Domain](/docs/graphql/objects#domain) +- [DataHubRole](/docs/graphql/objects#datahubrole) +- [Post](/docs/graphql/objects#post) +- [DataHubView](/docs/graphql/objects#datahubview) +- [QueryEntity](/docs/graphql/objects#queryentity) +- [DataProduct](/docs/graphql/objects#dataproduct) +- [Application](/docs/graphql/objects#application) +- [OwnershipTypeEntity](/docs/graphql/objects#ownershiptypeentity) +- [EntityTypeEntity](/docs/graphql/objects#entitytypeentity) +- [Restricted](/docs/graphql/objects#restricted) +- [BusinessAttribute](/docs/graphql/objects#businessattribute) +- [DataHubFile](/docs/graphql/objects#datahubfile) +- [Form](/docs/graphql/objects#form) +- [Incident](/docs/graphql/objects#incident) +- [ExecutionRequest](/docs/graphql/objects#executionrequest) +- [DataHubPageModule](/docs/graphql/objects#datahubpagemodule) +- [StructuredPropertyEntity](/docs/graphql/objects#structuredpropertyentity) +- [DataTypeEntity](/docs/graphql/objects#datatypeentity) +- [DataHubPageTemplate](/docs/graphql/objects#datahubpagetemplate) +- [Test](/docs/graphql/objects#test) +- [VersionSet](/docs/graphql/objects#versionset) +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key of the Metadata Entity

+
+type
+EntityType! +		 +

A standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

List of relationships between the source Entity and some destination entities with a given types

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## EntityWithRelationships + +Deprecated, use relationships field instead + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Implemented by

+ +- [Assertion](/docs/graphql/objects#assertion) +- [Dataset](/docs/graphql/objects#dataset) +- [ERModelRelationship](/docs/graphql/objects#ermodelrelationship) +- [SchemaFieldEntity](/docs/graphql/objects#schemafieldentity) +- [Dashboard](/docs/graphql/objects#dashboard) +- [Chart](/docs/graphql/objects#chart) +- [DataFlow](/docs/graphql/objects#dataflow) +- [DataJob](/docs/graphql/objects#datajob) +- [DataProcessInstance](/docs/graphql/objects#dataprocessinstance) +- [MLModel](/docs/graphql/objects#mlmodel) +- [MLModelGroup](/docs/graphql/objects#mlmodelgroup) +- [MLFeature](/docs/graphql/objects#mlfeature) +- [MLPrimaryKey](/docs/graphql/objects#mlprimarykey) +- [MLFeatureTable](/docs/graphql/objects#mlfeaturetable) +- [Restricted](/docs/graphql/objects#restricted) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the Metadata Entity

+
+type
+EntityType! +		 +

A standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+ +## HasExecutionRuns + + + +

Implemented by

+ +- [DataFlow](/docs/graphql/objects#dataflow) +- [DataJob](/docs/graphql/objects#datajob) + +

Fields

+ + + + + + + + + +
NameDescription
+runs
+DataProcessInstanceResult +		 +

History of runs of this task

+ +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+start
+Int +		 + +
+count
+Int +		 + +
+ +
+ +## HasLogicalParent + + + +

Implemented by

+ +- [Dataset](/docs/graphql/objects#dataset) +- [SchemaFieldEntity](/docs/graphql/objects#schemafieldentity) + +

Fields

+ + + + + + + + + +
NameDescription
+logicalParent
+Entity +		 +

If this entity represents a physical asset, this is its logical parent, from which metadata can propagate.

+
+ +## SupportsVersions + + + +

Implemented by

+ +- [Dataset](/docs/graphql/objects#dataset) +- [MLModel](/docs/graphql/objects#mlmodel) + +

Fields

+ + + + + + + + + +
NameDescription
+versionProperties
+VersionProperties +		 +

Indicates that this entity is versioned and provides information about the version.

+
+ +## TimeSeriesAspect + +A time series aspect, or a group of related metadata associated with an Entity and corresponding to a particular timestamp + +

Implemented by

+ +- [DataProcessRunEvent](/docs/graphql/objects#dataprocessrunevent) +- [DashboardUsageMetrics](/docs/graphql/objects#dashboardusagemetrics) +- [DatasetProfile](/docs/graphql/objects#datasetprofile) +- [AssertionRunEvent](/docs/graphql/objects#assertionrunevent) +- [Operation](/docs/graphql/objects#operation) + +

Fields

+ + + + + + + + + +
NameDescription
+timestampMillis
+Long! +		 +

The timestamp associated with the time series aspect in milliseconds

+
+ diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/mutations.md b/docs-archive/versioned_docs/version-1.5.0/graphql/mutations.md new file mode 100644 index 00000000..be3de9a5 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/mutations.md @@ -0,0 +1,4004 @@ +--- +id: mutations +title: Mutations +slug: mutations +sidebar_position: 2 +--- + +## acceptRole + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Accept role using invite token + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AcceptRoleInput! +		 + +
+ +## addBusinessAttribute + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add Business Attribute + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddBusinessAttributeInput! +		 + +
+ +## addGroupMembers + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add members to a group + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddGroupMembersInput! +		 + +
+ +## addLink + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add a link, or institutional memory, for a particular asset + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddLinkInput! +		 + +
+ +## addOwner + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add an owner to a particular Entity + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddOwnerInput! +		 + +
+ +## addOwners + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add multiple owners to a particular Entity + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddOwnersInput! +		 + +
+ +## addRelatedTerms + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add multiple related Terms to a Glossary Term to establish relationships + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedTermsInput! +		 + +
+ +## addTag + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add a tag to a particular Entity or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+TagAssociationInput! +		 + +
+ +## addTags + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add multiple tags to a particular Entity or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddTagsInput! +		 + +
+ +## addTerm + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add a glossary term to a particular Entity or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+TermAssociationInput! +		 + +
+ +## addTerms + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add multiple glossary terms to a particular Entity or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddTermsInput! +		 + +
+ +## batchAddOwners + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add owners to multiple Entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchAddOwnersInput! +		 + +
+ +## batchAddTags + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add tags to multiple Entities or subresources + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchAddTagsInput! +		 + +
+ +## batchAddTerms + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Add glossary terms to multiple Entities or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchAddTermsInput! +		 + +
+ +## batchAddToDataProducts + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Adds multiple assets to multiple data products (additive - preserves existing memberships). +Available when multipleDataProductsPerAsset feature flag is enabled. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchSetDataProductsInput! +		 +

Input for batch adding assets to data products

+
+ +## batchAssignForm + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Assign a form to different entities. This will be a patch by adding this form to the list +of forms on an entity. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchAssignFormInput! +		 + +
+ +## batchAssignRole + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Batch assign roles to users + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchAssignRoleInput! +		 + +
+ +## batchRemoveForm + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Remove a form from a given list of entities. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchRemoveFormInput! +		 + +
+ +## batchRemoveFromDataProducts + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Removes multiple assets from multiple data products (does not affect other memberships). +Available when multipleDataProductsPerAsset feature flag is enabled. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchSetDataProductsInput! +		 +

Input for batch removing assets from data products

+
+ +## batchRemoveOwners + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove owners from multiple Entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchRemoveOwnersInput! +		 + +
+ +## batchRemoveTags + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove tags from multiple Entities or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchRemoveTagsInput! +		 + +
+ +## batchRemoveTerms + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove glossary terms from multiple Entities or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchRemoveTermsInput! +		 + +
+ +## batchSetApplication + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Batch set or unset a Application to a list of entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchSetApplicationInput! +		 +

Input for batch setting application

+
+ +## batchSetDataProduct + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Batch set or unset a DataProduct to a list of entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchSetDataProductInput! +		 +

Input for batch setting data product

+
+ +## batchSetDomain + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Set domain for multiple Entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchSetDomainInput! +		 + +
+ +## batchUnsetApplication + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Batch unset a specific Application from a list of entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchUnsetApplicationInput! +		 +

Input for batch unsetting application

+
+ +## batchUpdateDeprecation + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Updates the deprecation status for a batch of assets. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchUpdateDeprecationInput! +		 + +
+ +## batchUpdateSoftDeleted + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Updates the soft deleted status for a batch of assets + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchUpdateSoftDeletedInput! +		 + +
+ +## batchUpdateStepStates + +**Type:** [BatchUpdateStepStatesResult!](/docs/graphql/objects#batchupdatestepstatesresult) + +Batch update the state for a set of steps. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchUpdateStepStatesInput! +		 + +
+ +## cancelIngestionExecutionRequest + +**Type:** [String](/docs/graphql/scalars#string) + +Cancel a running execution request, provided the urn of the original execution request + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CancelIngestionExecutionRequestInput! +		 + +
+ +## createAccessToken + +**Type:** [AccessToken](/docs/graphql/objects#accesstoken) + +Generates an access token for DataHub APIs for a particular user & of a particular type + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateAccessTokenInput! +		 + +
+ +## createApplication + +**Type:** [Application](/docs/graphql/objects#application) + +Create a new Application + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateApplicationInput! +		 +

Inputs required to create a new Application.

+
+ +## createBusinessAttribute + +**Type:** [BusinessAttribute](/docs/graphql/objects#businessattribute) + +Create Business Attribute Api + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateBusinessAttributeInput! +		 +

Inputs required to create a new BusinessAttribute.

+
+ +## createDataHubFile + +**Type:** [CreateDataHubFileResponse!](/docs/graphql/objects#createdatahubfileresponse) + +Create a new DataHub file entity + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateDataHubFileInput! +		 + +
+ +## createDataProduct + +**Type:** [DataProduct](/docs/graphql/objects#dataproduct) + +Create a new Data Product + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateDataProductInput! +		 +

Inputs required to create a new DataProduct.

+
+ +## createDocument + +**Type:** [String!](/docs/graphql/scalars#string) + +Create a new Document. Returns the urn of the newly created document. +Requires the CREATE_ENTITY privilege for documents or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateDocumentInput! +		 + +
+ +## createDomain + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new Domain. Returns the urn of the newly created Domain. Requires the 'Create Domains' or 'Manage Domains' Platform Privilege. If a Domain with the provided ID already exists, +it will be overwritten. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateDomainInput! +		 + +
+ +## createDynamicFormAssignment + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Creates a filter for a form to apply it to certain entities. Entities that match this filter will have +a given form applied to them. +This feature is ONLY supported in DataHub Cloud. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateDynamicFormAssignmentInput! +		 + +
+ +## createERModelRelationship + +**Type:** [ERModelRelationship](/docs/graphql/objects#ermodelrelationship) + +Create a ERModelRelationship + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ERModelRelationshipUpdateInput! +		 +

Input required to create a new ERModelRelationship

+
+ +## createForm + +**Type:** [Form!](/docs/graphql/objects#form) + +Create a new form based on the input + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateFormInput! +		 + +
+ +## createGlossaryNode + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new GlossaryNode. Returns the urn of the newly created GlossaryNode. If a node with the provided ID already exists, it will be overwritten. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateGlossaryEntityInput! +		 + +
+ +## createGlossaryTerm + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new GlossaryTerm. Returns the urn of the newly created GlossaryTerm. If a term with the provided ID already exists, it will be overwritten. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateGlossaryEntityInput! +		 + +
+ +## createGroup + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new group. Returns the urn of the newly created group. Requires the Manage Users & Groups Platform Privilege + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateGroupInput! +		 + +
+ +## createIngestionExecutionRequest + +**Type:** [String](/docs/graphql/scalars#string) + +Create a request to execute an ingestion job +input: Input required for creating an ingestion execution request + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateIngestionExecutionRequestInput! +		 + +
+ +## createIngestionSource + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new ingestion source + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateIngestionSourceInput! +		 + +
+ +## createInviteToken + +**Type:** [InviteToken](/docs/graphql/objects#invitetoken) + +Create invite token + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateInviteTokenInput! +		 + +
+ +## createNativeUserResetToken + +**Type:** [ResetToken](/docs/graphql/objects#resettoken) + +Generates a token that can be shared with existing native users to reset their credentials. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateNativeUserResetTokenInput! +		 + +
+ +## createOwnershipType + +**Type:** [OwnershipTypeEntity](/docs/graphql/objects#ownershiptypeentity) + +Create a Custom Ownership Type. This requires the 'Manage Ownership Types' Metadata Privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateOwnershipTypeInput! +		 +

Inputs required to create a new Query.

+
+ +## createPolicy + +**Type:** [String](/docs/graphql/scalars#string) + +Create a policy and returns the resulting urn + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+PolicyUpdateInput! +		 + +
+ +## createPost + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Create a post + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreatePostInput! +		 + +
+ +## createQuery + +**Type:** [QueryEntity](/docs/graphql/objects#queryentity) + +Create a new Query + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateQueryInput! +		 +

Inputs required to create a new Query.

+
+ +## createSecret + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new Secret + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateSecretInput! +		 + +
+ +## createServiceAccount + +**Type:** [ServiceAccount!](/docs/graphql/objects#serviceaccount) + +Creates a new service account. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateServiceAccountInput! +		 + +
+ +## createStructuredProperty + +**Type:** [StructuredPropertyEntity!](/docs/graphql/objects#structuredpropertyentity) + +Create a new structured property + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateStructuredPropertyInput! +		 + +
+ +## createTag + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new tag. Requires the 'Manage Tags' or 'Create Tags' Platform Privilege. If a Tag with the provided ID already exists, +it will be overwritten. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateTagInput! +		 +

Inputs required to create a new Tag.

+
+ +## createTest + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new test + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateTestInput! +		 + +
+ +## createTestConnectionRequest + +**Type:** [String](/docs/graphql/scalars#string) + +Create a request to execute a test ingestion connection job +input: Input required for creating a test connection request + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateTestConnectionRequestInput! +		 + +
+ +## createView + +**Type:** [DataHubView](/docs/graphql/objects#datahubview) + +Create a new DataHub View (Saved Filter) + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+CreateViewInput! +		 +

Input required to create a new DataHub View

+
+ +## deleteApplication + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a Application by urn. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the application to remove.

+
+ +## deleteAssertion + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove an assertion associated with an entity. Requires the 'Edit Assertions' privilege on the entity. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The assertion to remove

+
+ +## deleteBusinessAttribute + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a Business Attribute by urn. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the business attribute to remove.

+
+ +## deleteDataProduct + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a DataProduct by urn. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the data product to remove.

+
+ +## deleteDocument + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Delete a Document. +Requires the GET_ENTITY privilege for the document or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deleteDomain + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a Domain + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the Domain to delete

+
+ +## deleteERModelRelationship + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a ERModelRelationship + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the ERModelRelationship to delete

+
+ +## deleteForm + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Delete a given form + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+DeleteFormInput! +		 + +
+ +## deleteGlossaryEntity + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove a glossary entity (GlossaryTerm or GlossaryNode). Return boolean whether it was successful or not. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deleteIngestionSource + +**Type:** [String](/docs/graphql/scalars#string) + +Delete an existing ingestion source + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deleteOwnershipType + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a Custom Ownership Type by urn. This requires the 'Manage Ownership Types' Metadata Privilege. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the Custom Ownership Type to remove.

+
+deleteReferences
+Boolean +		 + +
+ +## deletePageModule + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Delete a DataHub page module + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+DeletePageModuleInput! +		 + +
+ +## deletePageTemplate + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Delete a DataHub page template + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+DeletePageTemplateInput! +		 + +
+ +## deletePolicy + +**Type:** [String](/docs/graphql/scalars#string) + +Remove an existing policy and returns the policy urn + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deletePost + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a post + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deleteQuery + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a Query by urn. This requires the 'Edit Queries' Metadata Privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the query to remove.

+
+ +## deleteSecret + +**Type:** [String](/docs/graphql/scalars#string) + +Delete a Secret + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deleteServiceAccount + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Deletes a service account by URN. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deleteStructuredProperty + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Delete an existing structured property + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+DeleteStructuredPropertyInput! +		 + +
+ +## deleteTag + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a Tag + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the Tag to delete

+
+ +## deleteTest + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete an existing test - note that this will NOT delete dangling pointers until the next execution of the test. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## deleteView + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Delete a DataHub View (Saved Filter) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the View to delete

+
+ +## linkAssetVersion + +**Type:** [VersionSet](/docs/graphql/objects#versionset) + +Link the latest versioned entity to a Version Set + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LinkVersionInput! +		 + +
+ +## moveDocument + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Move a Document to a different parent (or to root level if no parent is specified). +Requires the EDIT_ENTITY_DOCS or EDIT_ENTITY privilege for the document, or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+MoveDocumentInput! +		 + +
+ +## moveDomain + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Moves a domain to be parented under another domain. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+MoveDomainInput! +		 + +
+ +## patchEntities + +**Type:** [[PatchEntityResult!]!](/docs/graphql/objects#patchentityresult) + + + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+[PatchEntityInput!]! +		 + +
+ +## patchEntity + +**Type:** [PatchEntityResult!](/docs/graphql/objects#patchentityresult) + + + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+PatchEntityInput! +		 + +
+ +## raiseIncident + +**Type:** [String](/docs/graphql/scalars#string) + +Create a new incident for a resource (asset) + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RaiseIncidentInput! +		 +

Input required to create a new incident

+
+ +## removeBusinessAttribute + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove Business Attribute + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AddBusinessAttributeInput! +		 + +
+ +## removeGroup + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove a group. Requires Manage Users & Groups Platform Privilege + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## removeGroupMembers + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove members from a group + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RemoveGroupMembersInput! +		 + +
+ +## removeLink + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove a link, or institutional memory, from a particular Entity + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RemoveLinkInput! +		 + +
+ +## removeOwner + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove an owner from a particular Entity + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RemoveOwnerInput! +		 + +
+ +## removeRelatedTerms + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove multiple related Terms for a Glossary Term + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedTermsInput! +		 + +
+ +## removeStructuredProperties + +**Type:** [StructuredProperties!](/docs/graphql/objects#structuredproperties) + +Upsert structured properties onto a given asset + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RemoveStructuredPropertiesInput! +		 + +
+ +## removeTag + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove a tag from a particular Entity or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+TagAssociationInput! +		 + +
+ +## removeTerm + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove a glossary term from a particular Entity or subresource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+TermAssociationInput! +		 + +
+ +## removeUser + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Remove a user. Requires Manage Users & Groups Platform Privilege + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## reportAssertionResult + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Report result for an assertion + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of custom assertion.

+
+result
+AssertionResultInput! +		 +

Input for reporting result of the assertion

+
+ +## reportOperation + +**Type:** [String](/docs/graphql/scalars#string) + +Report a new operation for an asset + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ReportOperationInput! +		 +

Input required to report an operation

+
+ +## revokeAccessToken + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Revokes access tokens. + +

Arguments

+ + + + + + + + + +
NameDescription
+tokenId
+String! +		 + +
+ +## rollbackIngestion + +**Type:** [String](/docs/graphql/scalars#string) + +Rollback a specific ingestion execution run based on its runId + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RollbackIngestionInput! +		 + +
+ +## setDomain + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Sets the Domain for a Dataset, Chart, Dashboard, Data Flow (Pipeline), or Data Job (Task). Returns true if the Domain was successfully added, or already exists. Requires the Edit Domains privilege for the Entity. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+entityUrn
+String! +		 + +
+domainUrn
+String! +		 + +
+ +## setLogicalParent + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Set or unset an entity's logical parent + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SetLogicalParentInput! +		 + +
+ +## setTagColor + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Set the hex color associated with an existing Tag + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+colorHex
+String! +		 + +
+ +## submitFormPrompt + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Submit a response to a prompt from a form collecting metadata on different entities. +Provide the urn of the entity you're submitting a form response as well as the required input. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+SubmitFormPromptInput! +		 + +
+ +## unlinkAssetVersion + +**Type:** [VersionSet](/docs/graphql/objects#versionset) + +Unlink a versioned entity from a Version Set + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UnlinkVersionInput! +		 + +
+ +## unsetDomain + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Sets the Domain for a Dataset, Chart, Dashboard, Data Flow (Pipeline), or Data Job (Task). Returns true if the Domain was successfully removed, or was already removed. Requires the Edit Domains privilege for an asset. + +

Arguments

+ + + + + + + + + +
NameDescription
+entityUrn
+String! +		 + +
+ +## updateApplication + +**Type:** [Application](/docs/graphql/objects#application) + +Update a Application + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn identifier for the Application to update.

+
+input
+UpdateApplicationInput! +		 +

Inputs required to create a new Application.

+
+ +## updateApplicationsSettings + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the applications settings. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateApplicationsSettingsInput! +		 + +
+ +## updateAssetSettings + +**Type:** [AssetSettings!](/docs/graphql/objects#assetsettings) + +Update an asset's settings + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateAssetSettingsInput! +		 + +
+ +## updateBusinessAttribute + +**Type:** [BusinessAttribute](/docs/graphql/objects#businessattribute) + +Update Business Attribute + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn identifier for the Business Attribute to update.

+
+input
+UpdateBusinessAttributeInput! +		 +

Inputs required to create a new Business Attribute.

+
+ +## updateChart + +**Type:** [Chart](/docs/graphql/objects#chart) + +Update the metadata about a particular Chart + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+ChartUpdateInput! +		 + +
+ +## updateCorpGroupProperties + +**Type:** [CorpGroup](/docs/graphql/objects#corpgroup) + +Update a particular Corp Group's editable properties + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+CorpGroupUpdateInput! +		 + +
+ +## updateCorpUserProperties + +**Type:** [CorpUser](/docs/graphql/objects#corpuser) + +Update a particular Corp User's editable properties + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+CorpUserUpdateInput! +		 + +
+ +## updateCorpUserViewsSettings + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update the View-related settings for a user. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateCorpUserViewsSettingsInput! +		 + +
+ +## updateDashboard + +**Type:** [Dashboard](/docs/graphql/objects#dashboard) + +Update the metadata about a particular Dashboard + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+DashboardUpdateInput! +		 + +
+ +## updateDataFlow + +**Type:** [DataFlow](/docs/graphql/objects#dataflow) + +Update the metadata about a particular Data Flow (Pipeline) + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+DataFlowUpdateInput! +		 + +
+ +## updateDataJob + +**Type:** [DataJob](/docs/graphql/objects#datajob) + +Update the metadata about a particular Data Job (Task) + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+DataJobUpdateInput! +		 + +
+ +## updateDataProduct + +**Type:** [DataProduct](/docs/graphql/objects#dataproduct) + +Update a Data Product + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn identifier for the Data Product to update.

+
+input
+UpdateDataProductInput! +		 +

Inputs required to create a new DataProduct.

+
+ +## updateDataset + +**Type:** [Dataset](/docs/graphql/objects#dataset) + +Update the metadata about a particular Dataset + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+DatasetUpdateInput! +		 + +
+ +## updateDatasets + +**Type:** [[Dataset]](/docs/graphql/objects#dataset) + +Update the metadata about a batch of Datasets + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+[BatchDatasetUpdateInput!]! +		 + +
+ +## updateDeprecation + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Sets the Deprecation status for a Metadata Entity. Requires the Edit Deprecation status privilege for an entity. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateDeprecationInput! +		 +

Input required to set deprecation for an Entity.

+
+ +## updateDescription + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Incubating. Updates the description of a resource. Currently only supports Dataset Schema Fields, Containers + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+DescriptionUpdateInput! +		 + +
+ +## updateDisplayProperties + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update a particular entity's display properties + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+DisplayPropertiesUpdateInput! +		 + +
+ +## updateDocPropagationSettings + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the doc propagation settings. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateDocPropagationSettingsInput! +		 + +
+ +## updateDocumentContents + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the title or text of an existing Document. +Requires the EDIT_ENTITY_DOCS or EDIT_ENTITY privilege for the document, or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateDocumentContentsInput! +		 + +
+ +## updateDocumentRelatedEntities + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the related entities (assets and documents) for a Document. +Requires the EDIT_ENTITY_DOCS or EDIT_ENTITY privilege for the document, or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateDocumentRelatedEntitiesInput! +		 + +
+ +## updateDocumentSettings + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the settings of a Document (e.g., visibility in sidebar). +Requires the EDIT_ENTITY_DOCS or EDIT_ENTITY privilege for the document, or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateDocumentSettingsInput! +		 + +
+ +## updateDocumentStatus + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the status of a Document (published/unpublished). +Requires the EDIT_ENTITY_DOCS or EDIT_ENTITY privilege for the document, or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateDocumentStatusInput! +		 + +
+ +## updateDocumentSubType + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the sub-type of a Document (e.g., "FAQ", "Tutorial", "Runbook"). +Requires the EDIT_ENTITY_DOCS or EDIT_ENTITY privilege for the document, or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateDocumentSubTypeInput! +		 + +
+ +## updateEmbed + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update the Embed information for a Dataset, Dashboard, or Chart. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateEmbedInput! +		 + +
+ +## updateERModelRelationship + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update a ERModelRelationship + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the ERModelRelationship to update

+
+input
+ERModelRelationshipUpdateInput! +		 +

Input required to update an existing DataHub View

+
+ +## updateForm + +**Type:** [Form!](/docs/graphql/objects#form) + +Update an existing form based on the input + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateFormInput! +		 + +
+ +## updateGlobalViewsSettings + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Update the global settings related to the Views feature. +Requires the 'Manage Global Views' Platform Privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateGlobalViewsSettingsInput! +		 + +
+ +## updateIncident + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update an existing data incident. Any fields that are omitted will simply not be updated. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn for an existing incident

+
+input
+UpdateIncidentInput! +		 +

Input required to update an existing incident

+
+ +## updateIncidentStatus + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update the status of an existing incident for a resource (asset) + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn for an existing incident

+
+input
+IncidentStatusInput! +		 +

Input required to update the state of an existing incident

+
+ +## updateIngestionSource + +**Type:** [String](/docs/graphql/scalars#string) + +Update an existing ingestion source + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+UpdateIngestionSourceInput! +		 + +
+ +## updateLineage + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update lineage for an entity + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateLineageInput! +		 + +
+ +## updateLink + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update a link, or institutional memory, for a particular asset. +A combo of link label and url create a unique identifier for which link to update. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateLinkInput! +		 + +
+ +## updateName + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Updates the name of the entity. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateNameInput! +		 + +
+ +## updateNotebook + +**Type:** [Notebook](/docs/graphql/objects#notebook) + +Update the metadata about a particular Notebook + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+NotebookUpdateInput! +		 + +
+ +## updateOwnershipType + +**Type:** [OwnershipTypeEntity](/docs/graphql/objects#ownershiptypeentity) + +Update an existing Query. This requires the 'Manage Ownership Types' Metadata Privilege. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn identifier for the custom ownership type to update.

+
+input
+UpdateOwnershipTypeInput! +		 +

Inputs required to update an existing Custom Ownership Type.

+
+ +## updateParentNode + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Updates the parent node of a resource. Currently only GlossaryNodes and GlossaryTerms have parentNodes. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateParentNodeInput! +		 + +
+ +## updatePolicy + +**Type:** [String](/docs/graphql/scalars#string) + +Update an existing policy and returns the resulting urn + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+PolicyUpdateInput! +		 + +
+ +## updatePost + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update or edit a post + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdatePostInput! +		 + +
+ +## updateQuery + +**Type:** [QueryEntity](/docs/graphql/objects#queryentity) + +Update an existing Query + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn identifier for the query to update.

+
+input
+UpdateQueryInput! +		 +

Inputs required to update a Query.

+
+ +## updateSecret + +**Type:** [String](/docs/graphql/scalars#string) + +Update a Secret + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateSecretInput! +		 + +
+ +## updateStructuredProperty + +**Type:** [StructuredPropertyEntity!](/docs/graphql/objects#structuredpropertyentity) + +Update an existing structured property + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateStructuredPropertyInput! +		 + +
+ +## updateTag + +**Type:** [Tag](/docs/graphql/objects#tag) + +Update the information about a particular Entity Tag + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+TagUpdateInput! +		 + +
+ +## updateTest + +**Type:** [String](/docs/graphql/scalars#string) + +Update an existing test + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+UpdateTestInput! +		 + +
+ +## updateUserHomePageSettings + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update the HomePage-related settings for a user. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateUserHomePageSettingsInput! +		 + +
+ +## updateUserSetting + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Update a user setting + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpdateUserSettingInput! +		 + +
+ +## updateUserStatus + +**Type:** [String](/docs/graphql/scalars#string) + +Change the status of a user. Requires Manage Users & Groups Platform Privilege + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+status
+CorpUserStatus! +		 + +
+ +## updateView + +**Type:** [DataHubView](/docs/graphql/objects#datahubview) + +Delete a DataHub View (Saved Filter) + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the View to update

+
+input
+UpdateViewInput! +		 +

Input required to update an existing DataHub View

+
+ +## upsertConnection + +**Type:** [DataHubConnection!](/docs/graphql/objects#datahubconnection) + +Upsert a particular connection. +This requires the 'Manage Connections' platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpsertDataHubConnectionInput! +		 + +
+ +## upsertCustomAssertion + +**Type:** [Assertion!](/docs/graphql/objects#assertion) + +Upsert a Custom Assertion + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String +		 +

Urn of custom assertion. If not provided, one will be generated.

+
+input
+UpsertCustomAssertionInput! +		 +

Input for upserting a custom assertion.

+
+ +## upsertDataContract + +**Type:** [DataContract!](/docs/graphql/objects#datacontract) + +Create or update a data contract for a given dataset. Requires the "Edit Data Contract" privilege for the provided dataset. + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String +		 + +
+input
+UpsertDataContractInput! +		 + +
+ +## upsertLink + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Upsert a link, or institutional memory, for a particular asset. +A combo of link label and url create a unique identifier to update a link, +or add a new link if no links with label/url combo are found. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpsertLinkInput! +		 + +
+ +## upsertPageModule + +**Type:** [DataHubPageModule!](/docs/graphql/objects#datahubpagemodule) + +Create or update a DataHub page module + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpsertPageModuleInput! +		 + +
+ +## upsertPageTemplate + +**Type:** [DataHubPageTemplate!](/docs/graphql/objects#datahubpagetemplate) + +Batch update the state for a set of steps. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpsertPageTemplateInput! +		 + +
+ +## upsertStructuredProperties + +**Type:** [StructuredProperties!](/docs/graphql/objects#structuredproperties) + +Upsert structured properties onto a given asset + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+UpsertStructuredPropertiesInput! +		 + +
+ +## verifyForm + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Verifies a form on an entity when all of the required questions on the form are complete and the form +is of type VERIFICATION. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+VerifyFormInput! +		 + +
+ diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/objects.md b/docs-archive/versioned_docs/version-1.5.0/graphql/objects.md new file mode 100644 index 00000000..f836ab0e --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/objects.md @@ -0,0 +1,31150 @@ +--- +id: objects +title: Objects +slug: objects +sidebar_position: 3 +--- + +## Access + + + +

Fields

+ + + + + + + + + +
NameDescription
+roles
+[RoleAssociation!] +		 + +
+ +## AccessToken + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+accessToken
+String! +		 +

The access token itself

+
+metadata
+AccessTokenMetadata +		 +

Metadata about the generated token

+
+ +## AccessTokenMetadata + + + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the access token

+
+type
+EntityType! +		 +

The standard Entity Type

+
+id
+String! +		 +

The unique identifier of the token.

+
+name
+String! +		 +

The name of the token, if it exists.

+
+description
+String +		 +

The description of the token if defined.

+
+actorUrn
+String! +		 +

The actor associated with the Access Token.

+
+ownerUrn
+String! +		 +

The actor who created the Access Token.

+
+owner
+CorpUser +		 +

The owner (CorpUser) who created the Access Token.

+
+createdAt
+Long! +		 +

The time when token was generated at.

+
+expiresAt
+Long +		 +

Time when token will be expired.

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## ActiveIncidentHealthDetails + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+latestIncidentUrn
+String +		 +

The latest incident

+
+latestIncidentTitle
+String +		 +

The title of the latest incident

+
+lastActivityAt
+Long +		 +

The timestamp when the last incident was updated

+
+count
+Int! +		 +

The number of active incidents

+
+ +## Actor + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+users
+[RoleUser!] +		 +

List of users for which the role is provisioned

+
+groups
+[RoleGroup!] +		 +

List of groups for which the role is provisioned

+
+ +## ActorFilter + +The actors that a DataHub Access Policy applies to + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+users
+[String!] +		 +

A disjunctive set of users to apply the policy to

+
+groups
+[String!] +		 +

A disjunctive set of groups to apply the policy to

+
+roles
+[String!] +		 +

A disjunctive set of roles to apply the policy to

+
+resourceOwners
+Boolean! +		 +

Whether the filter should return TRUE for owners of a particular resource +Only applies to policies of type METADATA, which have a resource associated with them

+
+resourceOwnersTypes
+[String!] +		 +

Set of OwnershipTypes to apply the policy to (if resourceOwners field is set to True)

+
+resolvedOwnershipTypes
+[OwnershipTypeEntity!] +		 +

Set of OwnershipTypes to apply the policy to (if resourceOwners field is set to True), resolved.

+
+allUsers
+Boolean! +		 +

Whether the filter should apply to all users

+
+allGroups
+Boolean! +		 +

Whether the filter should apply to all groups

+
+resolvedUsers
+[CorpUser!] +		 +

The list of users on the Policy, resolved.

+
+resolvedGroups
+[CorpGroup!] +		 +

The list of groups on the Policy, resolved.

+
+resolvedRoles
+[DataHubRole!] +		 +

The list of roles on the Policy, resolved.

+
+ +## AggregateResults + +Results returned from aggregateAcrossEntities + +

Fields

+ + + + + + + + + +
NameDescription
+facets
+[FacetMetadata!] +		 +

Candidate facet aggregations used for search filtering

+
+ +## AggregationMetadata + +Information about the aggregation that can be used for filtering, included the field value and number of results + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+value
+String! +		 +

A particular value of a facet field

+
+count
+Long! +		 +

The number of search results containing the value

+
+entity
+Entity +		 +

Entity corresponding to the facet field

+
+displayName
+String +		 +

Optional display name to show in the UI for this filter value

+
+ +## AllowedValue + +An entry for an allowed value for a structured property + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+value
+PropertyValue! +		 +

The allowed value

+
+description
+String +		 +

The description of this allowed value

+
+ +## AnalyticsChartGroup + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+groupId
+String! +		 + +
+title
+String! +		 + +
+charts
+[AnalyticsChart!]! +		 + +
+ +## AnalyticsConfig + +Configurations related to the Analytics Feature + +

Fields

+ + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether the Analytics feature is enabled and should be displayed

+
+ +## AppConfig + +Config loaded at application boot time +This configuration dictates the behavior of the UI, such as which features are enabled or disabled + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+appVersion
+String +		 +

App version

+
+authConfig
+AuthConfig! +		 +

Auth-related configurations

+
+analyticsConfig
+AnalyticsConfig! +		 +

Configurations related to the Analytics Feature

+
+policiesConfig
+PoliciesConfig! +		 +

Configurations related to the Policies Feature

+
+identityManagementConfig
+IdentityManagementConfig! +		 +

Configurations related to the User & Group management

+
+managedIngestionConfig
+ManagedIngestionConfig! +		 +

Configurations related to UI-based ingestion

+
+lineageConfig
+LineageConfig! +		 +

Configurations related to Lineage

+
+visualConfig
+VisualConfig! +		 +

Configurations related to visual appearance, allows styling the UI without rebuilding the bundle

+
+telemetryConfig
+TelemetryConfig! +		 +

Configurations related to tracking users in the app

+
+testsConfig
+TestsConfig! +		 +

Configurations related to DataHub tests

+
+viewsConfig
+ViewsConfig! +		 +

Configurations related to DataHub Views

+
+searchBarConfig
+SearchBarConfig! +		 +

Configurations related to the Search bar

+
+searchCardConfig
+SearchCardConfig! +		 +

Configurations related to the Search card

+
+searchFlagsConfig
+SearchFlagsConfig! +		 +

Configurations related the Search Flags

+
+featureFlags
+FeatureFlagsConfig! +		 +

Feature flags telling the UI whether a feature is enabled or not

+
+chromeExtensionConfig
+ChromeExtensionConfig! +		 +

Configuration related to the DataHub Chrome Extension

+
+homePageConfig
+HomePageConfig! +		 +

Configuration related to the home page

+
+semanticSearchConfig
+SemanticSearchConfig +		 +

Semantic search and embedding configuration

+
+ +## Application + +An Application, or a grouping of Entities for a single business purpose. Compared with Data Products, Applications represent a grouping of tables that exist to serve a specific +purpose. However, unlike Data Products, they don't represent groups that are tailored to be consumed for any particular purpose. Often, the assets in Applications power specific +outcomes, for example a Pricing Application. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Application

+
+type
+EntityType! +		 +

A standard Entity Type

+
+properties
+ApplicationProperties +		 +

Properties about an Application

+
+ownership
+Ownership +		 +

Ownership metadata of the Application

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the Application

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the Application

+
+domain
+DomainAssociation +		 +

The Domain associated with the Application

+
+tags
+GlobalTags +		 +

Tags used for searching Application

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Application

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## ApplicationAssociation + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+application
+Application! +		 +

The application related to the assocaited urn

+
+associatedUrn
+String! +		 +

Reference back to the tagged urn for tracking purposes e.g. when sibling nodes are merged together

+
+ +## ApplicationConfig + +Configuration for the application sidebar section + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+showSidebarSectionWhenEmpty
+Boolean +		 +

Whether to show the application sidebar section even when empty

+
+showApplicationInNavigation
+Boolean +		 +

Whether to show the application in the navigation sidebar

+
+ +## ApplicationProperties + +Properties about an Application + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the Application

+
+description
+String +		 +

Description of the Application

+
+externalUrl
+String +		 +

External URL for the Appliation (most likely GitHub repo where Application may be managed as code)

+
+numAssets
+Int +		 +

Number of children entities inside of the Application. This number includes soft deleted entities.

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the Application

+
+ +## AspectRenderSpec + +Details for the frontend on how the raw aspect should be rendered + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+displayType
+String +		 +

Format the aspect should be displayed in for the UI. Powered by the renderSpec annotation on the aspect model

+
+displayName
+String +		 +

Name to refer to the aspect type by for the UI. Powered by the renderSpec annotation on the aspect model

+
+key
+String +		 +

Field in the aspect payload to index into for rendering.

+
+ +## Assertion + +An assertion represents a programmatic validation, check, or test performed periodically against another Entity. + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+actions
+AssertionActions +		 +

The actions associated with the Assertion

+
+urn
+String! +		 +

The primary key of the Assertion

+
+type
+EntityType! +		 +

The standard Entity Type

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the assertion is evaluated

+
+info
+AssertionInfo +		 +

Details about assertion

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+runEvents
+AssertionRunEventsResult +		 +

Lifecycle events detailing individual runs of this assertion. If startTimeMillis & endTimeMillis are not provided, the most +recent events will be returned.

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+status
+AssertionRunStatus +		 + +
+startTimeMillis
+Long +		 + +
+endTimeMillis
+Long +		 + +
+filter
+FilterInput +		 + +
+limit
+Int +		 + +
+ +
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+status
+Status +		 +

Status metadata of the assertion

+
+tags
+GlobalTags +		 +

The standard tags for the Assertion

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra aspects that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+ +## AssertionAction + +An action associated with an assertion + +

Fields

+ + + + + + + + + +
NameDescription
+type
+AssertionActionType! +		 +

The type of the actions

+
+ +## AssertionActions + +Some actions associated with an assertion + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+onSuccess
+[AssertionAction!]! +		 +

Actions to be executed on successful assertion run.

+
+onFailure
+[AssertionAction!]! +		 +

Actions to be executed on failed assertion run.

+
+ +## AssertionHealthStatusByType + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+AssertionType! +		 +

The type group of assertions

+
+status
+HealthStatus! +		 +

The status of the assertions in the given type group

+
+total
+Int! +		 +

The number of assertions in the given type group

+
+statusCount
+Int! +		 +

The number of assertions in the given type group that have the given status (PASS, WARN, FAIL)

+
+lastStatusResultAt
+Long +		 +

The timestamp when the last assertion of this type group with the given status ran

+
+ +## AssertionInfo + +Type of assertion. Assertion types can evolve to span Datasets, Flows (Pipelines), Models, Features etc. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+freshnessAssertion
+FreshnessAssertionInfo +		 +

Information about an Freshness Assertion

+
+volumeAssertion
+VolumeAssertionInfo +		 +

Information about an Volume Assertion

+
+sqlAssertion
+SqlAssertionInfo +		 +

Information about a SQL Assertion

+
+fieldAssertion
+FieldAssertionInfo +		 +

Information about a Field Assertion

+
+schemaAssertion
+SchemaAssertionInfo +		 +

Schema assertion, e.g. defining the expected structure for an asset.

+
+customAssertion
+CustomAssertionInfo +		 +

Information about Custom assertion

+
+source
+AssertionSource +		 +

The source or origin of the Assertion definition.

+
+lastUpdated
+AuditStamp +		 +

The time that the status last changed and the actor who changed it

+
+type
+AssertionType! +		 +

Top-level type of the assertion.

+
+datasetAssertion
+DatasetAssertionInfo +		 +

Dataset-specific assertion information

+
+description
+String +		 +

An optional human-readable description of the assertion

+
+externalUrl
+String +		 +

URL where assertion details are available

+
+ +## AssertionResult + +The result of evaluating an assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+AssertionResultType! +		 +

The final result, e.g. either SUCCESS or FAILURE.

+
+rowCount
+Long +		 +

Number of rows for evaluated batch

+
+missingCount
+Long +		 +

Number of rows with missing value for evaluated batch

+
+unexpectedCount
+Long +		 +

Number of rows with unexpected value for evaluated batch

+
+actualAggValue
+Float +		 +

Observed aggregate value for evaluated batch

+
+externalUrl
+String +		 +

URL where full results are available

+
+nativeResults
+[StringMapEntry!] +		 +

Native results / properties of evaluation

+
+error
+AssertionResultError +		 +

Error details, if type is ERROR

+
+ +## AssertionResultError + +An error encountered when evaluating an AssertionResult + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+AssertionResultErrorType! +		 +

The type of error encountered

+
+properties
+[StringMapEntry!] +		 +

Additional metadata depending on the type of error

+
+ +## AssertionRunEvent + +An event representing an event in the assertion evaluation lifecycle. + +

Implements

+ +- [TimeSeriesAspect](/docs/graphql/interfaces#timeseriesaspect) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+timestampMillis
+Long! +		 +

The time at which the assertion was evaluated

+
+lastObservedMillis
+Long +		 +

The time at which the run event was last observed by the DataHub system - ie, when it was reported by external systems

+
+assertionUrn
+String! +		 +

Urn of assertion which is evaluated

+
+asserteeUrn
+String! +		 +

Urn of entity on which the assertion is applicable

+
+runId
+String! +		 +

Native (platform-specific) identifier for this run

+
+status
+AssertionRunStatus! +		 +

The status of the assertion run as per this timeseries event

+
+batchSpec
+BatchSpec +		 +

Specification of the batch which this run is evaluating

+
+partitionSpec
+PartitionSpec +		 +

Information about the partition that was evaluated

+
+runtimeContext
+[StringMapEntry!] +		 +

Runtime parameters of evaluation

+
+result
+AssertionResult +		 +

Results of assertion, present if the status is COMPLETE

+
+ +## AssertionRunEventsResult + +Result returned when fetching run events for an assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+total
+Int! +		 +

The total number of run events returned

+
+failed
+Int! +		 +

The number of failed run events

+
+succeeded
+Int! +		 +

The number of succeeded run events

+
+errored
+Int! +		 +

The number of errored run events

+
+runEvents
+[AssertionRunEvent!]! +		 +

The run events themselves

+
+ +## AssertionSource + +The source of an Assertion + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+AssertionSourceType! +		 +

The source type

+
+created
+AuditStamp +		 +

The time at which the assertion was initially created and the actor who created it

+
+ +## AssertionStdParameter + +Parameter for AssertionStdOperator. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+value
+String! +		 +

The parameter value

+
+type
+AssertionStdParameterType! +		 +

The type of the parameter

+
+ +## AssertionStdParameters + +Parameters for AssertionStdOperators + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+value
+AssertionStdParameter +		 +

The value parameter of an assertion

+
+maxValue
+AssertionStdParameter +		 +

The maxValue parameter of an assertion

+
+minValue
+AssertionStdParameter +		 +

The minValue parameter of an assertion

+
+ +## AssetCollectionModuleParams + +The params required if the module is type ASSET_COLLECTION + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+assetUrns
+[String!]! +		 +

The list of asset urns for the asset collection module

+
+dynamicFilterJson
+String +		 +

Optional dynamic filter

+

The stringified json representing the logical predicate built in the UI to select assets. +This predicate is turned into orFilters to send through graphql since graphql doesn't support +arbitrary nesting. This string is used to restore the UI for this logical predicate.

+
+ +## AssetSettings + +Settings associated with this asset + +

Fields

+ + + + + + + + + +
NameDescription
+assetSummary
+AssetSummarySettings +		 +

Information related to the asset summary for this asset

+
+ +## AssetStatsResult + +Information regarding asset stats + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+oldestOperationTime
+Long +		 +

The oldest dataset operation in our index

+
+oldestDatasetUsageTime
+Long +		 +

The oldest dataset usage in our index

+
+oldestDatasetProfileTime
+Long +		 +

The oldest dataset profile in our index

+
+ +## AssetSummarySettings + +Information related to the asset summary for this asset + +

Fields

+ + + + + + + + + +
NameDescription
+templates
+[AssetSummarySettingsTemplate!] +		 +

The list of templates applied to this asset in order. Right now we only expect one.

+
+ +## AssetSummarySettingsTemplate + +Object containing the template and any additional info for asset summary settings + +

Fields

+ + + + + + + + + +
NameDescription
+template
+DataHubPageTemplate +		 +

The page template entity

+
+ +## AuditStamp + +A time stamp along with an optional actor + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+time
+Long! +		 +

When the audited action took place

+
+actor
+String +		 +

Who performed the audited action

+
+ +## AuthConfig + +Configurations related to auth + +

Fields

+ + + + + + + + + +
NameDescription
+tokenAuthEnabled
+Boolean! +		 +

Whether token-based auth is enabled.

+
+ +## AuthenticatedUser + +Information about the currently authenticated user + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+corpUser
+CorpUser! +		 +

The user information associated with the authenticated user, including properties used in rendering the profile

+
+platformPrivileges
+PlatformPrivileges! +		 +

The privileges assigned to the currently authenticated user, which dictates which parts of the UI they should be able to use

+
+ +## AutoCompleteMultipleResults + +The results returned on a multi entity autocomplete query + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+query
+String! +		 +

The raw query string

+
+suggestions
+[AutoCompleteResultForEntity!]! +		 +

The autocompletion suggestions

+
+ +## AutoCompleteResultForEntity + +An individual auto complete result specific to an individual Metadata Entity Type + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+EntityType! +		 +

Entity type

+
+suggestions
+[String!]! +		 +

The autocompletion results for specified entity type

+
+entities
+[Entity!]! +		 +

A list of entities to render in autocomplete

+
+ +## AutoCompleteResults + +The results returned on a single entity autocomplete query + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+query
+String! +		 +

The query string

+
+suggestions
+[String!]! +		 +

The autocompletion results

+
+entities
+[Entity!]! +		 +

A list of entities to render in autocomplete

+
+ +## AwsProviderConfig + +AWS Bedrock provider-specific configuration + +

Fields

+ + + + + + + + + +
NameDescription
+region
+String! +		 +

AWS region where Bedrock is accessed (e.g., "us-west-2")

+
+ +## BarChart + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+title
+String! +		 + +
+bars
+[NamedBar!]! +		 + +
+ +## BarSegment + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+label
+String! +		 + +
+value
+Int! +		 + +
+ +## BaseData + + + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+dataset
+String! +		 +

Dataset used for the Training or Evaluation of the MLModel

+
+motivation
+String +		 +

Motivation to pick these datasets

+
+preProcessing
+[String!] +		 +

Details of Data Proprocessing

+
+ +## BatchGetStepStatesResult + +Result returned when fetching step state + +

Fields

+ + + + + + + + + +
NameDescription
+results
+[StepStateResult!]! +		 +

The step states

+
+ +## BatchSpec + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+nativeBatchId
+String +		 +

The native identifier as specified by the system operating on the batch.

+
+query
+String +		 +

A query that identifies a batch of data

+
+limit
+Int +		 +

Any limit to the number of rows in the batch, if applied

+
+customProperties
+[StringMapEntry!] +		 +

Custom properties of the Batch

+
+ +## BatchUpdateStepStatesResult + +Result returned when fetching step state + +

Fields

+ + + + + + + + + +
NameDescription
+results
+[UpdateStepStateResult!]! +		 +

Results for each step

+
+ +## BooleanBox + + + +

Fields

+ + + + + + + + + +
NameDescription
+booleanValue
+Boolean! +		 + +
+ +## BrowsePath + +A hierarchical entity path + +

Fields

+ + + + + + + + + +
NameDescription
+path
+[String!]! +		 +

The components of the browse path

+
+ +## BrowsePathEntry + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The path name of a group of browse results

+
+entity
+Entity +		 +

An optional entity associated with this browse entry. This will usually be a container entity. +If this entity is not populated, the name must be used.

+
+ +## BrowsePathV2 + +A hierarchical entity path V2 + +

Fields

+ + + + + + + + + +
NameDescription
+path
+[BrowsePathEntry!]! +		 +

The components of the browse path

+
+ +## BrowseResultGroup + +A group of Entities under a given browse path + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The path name of a group of browse results

+
+count
+Long! +		 +

The number of entities within the group

+
+ +## BrowseResultGroupV2 + +A group of Entities under a given browse path + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The path name of a group of browse results

+
+entity
+Entity +		 +

An optional entity associated with this browse group. This will usually be a container entity. +If this entity is not populated, the name must be used.

+
+count
+Long! +		 +

The number of entities within the group

+
+hasSubGroups
+Boolean! +		 +

Whether or not there are any more groups underneath this group

+
+ +## BrowseResultMetadata + +Metadata about the Browse Paths response + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+path
+[String!]! +		 +

The provided path

+
+totalNumEntities
+Long! +		 +

The total number of entities under the provided browse path

+
+ +## BrowseResults + +The results of a browse path traversal query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entities
+[Entity!]! +		 +

The browse results

+
+groups
+[BrowseResultGroup!]! +		 +

The groups present at the provided browse path

+
+start
+Int! +		 +

The starting point of paginated results

+
+count
+Int! +		 +

The number of elements included in the results

+
+total
+Int! +		 +

The total number of browse results under the path with filters applied

+
+metadata
+BrowseResultMetadata! +		 +

Metadata containing resulting browse groups

+
+ +## BrowseResultsV2 + +The results of a browse path V2 traversal query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+groups
+[BrowseResultGroupV2!]! +		 +

The groups present at the provided browse path V2

+
+start
+Int! +		 +

The starting point of paginated results

+
+count
+Int! +		 +

The number of groups included in the results

+
+total
+Int! +		 +

The total number of browse groups under the path with filters applied

+
+metadata
+BrowseResultMetadata! +		 +

Metadata containing resulting browse groups

+
+ +## BucketStorageLocation + +Information where a file is stored + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+storageBucket
+String! +		 +

The storage bucket this file is stored in

+
+storageKey
+String! +		 +

The key for where this file is stored inside of the given bucket

+
+ +## BusinessAttribute + +A Business Attribute, or a logical schema Field + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Data Product

+
+type
+EntityType! +		 +

A standard Entity Type

+
+properties
+BusinessAttributeInfo +		 +

Properties about a Business Attribute

+
+ownership
+Ownership +		 +

Ownership metadata of the Business Attribute

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to Business Attribute

+
+status
+Status +		 +

Status of the Dataset

+
+relationships
+EntityRelationshipsResult +		 +

List of relationships between the source Entity and some destination entities with a given types

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## BusinessAttributeAssociation + +Input required to attach business attribute to an entity + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+businessAttribute
+BusinessAttribute! +		 +

Business Attribute itself

+
+associatedUrn
+String! +		 +

Reference back to the associated urn for tracking purposes e.g. when sibling nodes are merged together

+
+ +## BusinessAttributeInfo + +Business Attribute type + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

name of the business attribute

+
+description
+String +		 +

description of business attribute

+
+tags
+GlobalTags +		 +

Tags associated with the business attribute

+
+glossaryTerms
+GlossaryTerms +		 +

Glossary terms associated with the business attribute

+
+type
+SchemaFieldDataType +		 +

Platform independent field type of the field

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+created
+AuditStamp! +		 +

An AuditStamp corresponding to the creation of this chart

+
+lastModified
+AuditStamp! +		 +

An AuditStamp corresponding to the modification of this chart

+
+deleted
+AuditStamp +		 +

An optional AuditStamp corresponding to the deletion of this chart

+
+ +## BusinessAttributes + +Business attributes attached to the metadata + +

Fields

+ + + + + + + + + +
NameDescription
+businessAttribute
+BusinessAttributeAssociation +		 +

Business Attribute attached to the Metadata Entity

+
+ +## CaveatDetails + + + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+needsFurtherTesting
+Boolean +		 +

Did the results suggest any further testing

+
+caveatDescription
+String +		 +

Caveat Description

+
+groupsNotRepresented
+[String!] +		 +

Relevant groups that were not represented in the evaluation dataset

+
+ +## CaveatsAndRecommendations + + + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+caveats
+CaveatDetails +		 +

Caveats on using this MLModel

+
+recommendations
+String +		 +

Recommendations on where this MLModel should be used

+
+idealDatasetCharacteristics
+[String!] +		 +

Ideal characteristics of an evaluation dataset for this MLModel

+
+ +## Cell + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+value
+String! +		 + +
+entity
+Entity +		 + +
+linkParams
+LinkParams +		 + +
+ +## ChangeAuditStamps + +Captures information about who created/last modified/deleted the entity and when + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+created
+AuditStamp! +		 +

An AuditStamp corresponding to the creation

+
+lastModified
+AuditStamp! +		 +

An AuditStamp corresponding to the modification

+
+deleted
+AuditStamp +		 +

An optional AuditStamp corresponding to the deletion

+
+ +## ChangeEvent + +An individual change in a transaction + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the entity that was changed

+
+category
+ChangeCategoryType +		 +

The category of the change

+
+operation
+ChangeOperationType +		 +

The operation of the change

+
+modifier
+String +		 +

The modifier of the change

+
+parameters
+[TimelineParameterEntry!] +		 +

The parameters of the change

+
+auditStamp
+AuditStamp +		 +

The audit stamp of the change

+
+description
+String +		 +

description of the change

+
+ +## ChangeTransaction + +A change transaction is a set of changes that were committed together. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+timestampMillis
+Long! +		 +

The time at which the transaction was committed

+
+lastSemanticVersion
+String! +		 +

The last semantic version that this schema was changed in

+
+versionStamp
+String! +		 +

Version stamp of the change

+
+changeType
+ChangeOperationType! +		 +

The type of the change

+
+changes
+[ChangeEvent!] +		 +

The list of changes in this transaction

+
+ +## Chart + +A Chart Metadata Entity + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Chart

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+container
+Container +		 +

The parent container in which the entity resides

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+tool
+String! +		 +

The chart tool name +Note that this field will soon be deprecated in favor a unified notion of Data Platform

+
+chartId
+String! +		 +

An id unique within the charting tool

+
+properties
+ChartProperties +		 +

Additional read only properties about the Chart

+
+editableProperties
+ChartEditableProperties +		 +

Additional read write properties about the Chart

+
+query
+ChartQuery +		 +

Info about the query which is used to render the chart

+
+ownership
+Ownership +		 +

Ownership metadata of the chart

+
+status
+Status +		 +

Status metadata of the chart

+
+deprecation
+Deprecation +		 +

The deprecation status of the chart

+
+embed
+Embed +		 +

Embed information about the Chart

+
+tags
+GlobalTags +		 +

The tags associated with the chart

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dashboard

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dashboard

+
+domain
+DomainAssociation +		 +

The Domain associated with the Chart

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+statsSummary
+ChartStatsSummary +		 +

Not yet implemented.

+

Experimental - Summary operational & usage statistics about a Chart

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the chart. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+info
+ChartInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +Additional read only information about the chart

+
+editableInfo
+ChartEditableProperties +		 +
Deprecated: No longer supported
+ +

Deprecated, use editableProperties field instead +Additional read write information about the Chart

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use tags instead +The structured tags associated with the chart

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the chart is defined

+
+inputFields
+InputFields +		 +

Input fields to power the chart

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+subTypes
+SubTypes +		 +

Sub Types that this entity implements

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+health
+[Health!] +		 +

Experimental! The resolved health statuses of the asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+incidents
+EntityIncidentsResult +		 +

Incidents associated with the Chart

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState +		 +

Optional incident state to filter by, defaults to any state.

+
+stage
+IncidentStage +		 +

Optional incident stage to filter by, defaults to any state.

+
+priority
+IncidentPriority +		 +

Optional incident priority to filter by, defaults to any state.

+
+assigneeUrns
+[String!] +		 +

Optional assignee urns for an incident.

+
+start
+Int +		 +

Optional start offset, defaults to 0.

+
+count
+Int +		 +

Optional start offset, defaults to 20.

+
+ +
+ +## ChartCell + +A Notebook cell which contains chart as content + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+cellTitle
+String! +		 +

Title of the cell

+
+cellId
+String! +		 +

Unique id for the cell.

+
+changeAuditStamps
+ChangeAuditStamps +		 +

Captures information about who created/last modified/deleted this TextCell and when

+
+ +## ChartEditableProperties + +Chart properties that are editable via the UI This represents logical metadata, +as opposed to technical metadata + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

Description of the Chart

+
+ +## ChartInfo + +Deprecated, use ChartProperties instead +Additional read only information about the chart + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the chart

+
+description
+String +		 +

Description of the chart

+
+inputs
+[Dataset!] +		 +
Deprecated: No longer supported
+ +

Deprecated, use relationship Consumes instead +Data sources for the chart

+
+externalUrl
+String +		 +

Native platform URL of the chart

+
+type
+ChartType +		 +

Access level for the chart

+
+access
+AccessLevel +		 +

Access level for the chart

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+lastRefreshed
+Long +		 +

The time when this chart last refreshed

+
+created
+AuditStamp! +		 +

An AuditStamp corresponding to the creation of this chart

+
+lastModified
+AuditStamp! +		 +

An AuditStamp corresponding to the modification of this chart

+
+deleted
+AuditStamp +		 +

An optional AuditStamp corresponding to the deletion of this chart

+
+ +## ChartProperties + +Additional read only properties about the chart + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the chart

+
+description
+String +		 +

Description of the chart

+
+externalUrl
+String +		 +

Native platform URL of the chart

+
+type
+ChartType +		 +

Access level for the chart

+
+access
+AccessLevel +		 +

Access level for the chart

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+lastRefreshed
+Long +		 +

The time when this chart last refreshed

+
+created
+AuditStamp! +		 +

An AuditStamp corresponding to the creation of this chart

+
+lastModified
+AuditStamp! +		 +

An AuditStamp corresponding to the modification of this chart

+
+deleted
+AuditStamp +		 +

An optional AuditStamp corresponding to the deletion of this chart

+
+ +## ChartQuery + +The query that was used to populate a Chart + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+rawQuery
+String! +		 +

Raw query to build a chart from input datasets

+
+type
+ChartQueryType! +		 +

The type of the chart query

+
+ +## ChartStatsSummary + +Experimental - subject to change. A summary of usage metrics about a Chart. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+viewCount
+Int +		 +

The total view count for the chart

+
+viewCountLast30Days
+Int +		 +

The view count in the last 30 days

+
+uniqueUserCountLast30Days
+Int +		 +

The unique user count in the past 30 days

+
+topUsersLast30Days
+[CorpUser!] +		 +

The top users in the past 30 days

+
+ +## ChromeExtensionConfig + +Configurations related to DataHub Chrome extension + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether the Chrome Extension is enabled

+
+lineageEnabled
+Boolean! +		 +

Whether lineage is enabled

+
+ +## Container + +A container of other Metadata Entities + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the container

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+platform
+DataPlatform! +		 +

Standardized platform.

+
+container
+Container +		 +

Fetch an Entity Container by primary key (urn)

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+properties
+ContainerProperties +		 +

Read-only properties that originate in the source data platform

+
+editableProperties
+ContainerEditableProperties +		 +

Read-write properties that originate in DataHub

+
+ownership
+Ownership +		 +

Ownership metadata of the dataset

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dataset

+
+tags
+GlobalTags +		 +

Tags used for searching dataset

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dataset

+
+subTypes
+SubTypes +		 +

Sub types of the container, e.g. "Database" etc

+
+domain
+DomainAssociation +		 +

The Domain associated with the Dataset

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+deprecation
+Deprecation +		 +

The deprecation status of the container

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+entities
+SearchResults +		 +

Children entities inside of the Container

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ContainerEntitiesInput +		 + +
+ +
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+status
+Status +		 +

Status metadata of the container

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+access
+Access +		 +

The Roles and the properties to access the container

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+ +## ContainerEditableProperties + +Read-write properties that originate in DataHub + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

DataHub description of the Container

+
+ +## ContainerProperties + +Read-only properties that originate in the source data platform + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the Container

+
+description
+String +		 +

System description of the Container

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the Container

+
+externalUrl
+String +		 +

Native platform URL of the Container

+
+qualifiedName
+String +		 +

Fully-qualified name of the Container

+
+ +## ContentParams + +Params about the recommended content + +

Fields

+ + + + + + + + + +
NameDescription
+count
+Long +		 +

Number of entities corresponding to the recommended content

+
+ +## CorpGroup + +A DataHub Group entity, which represents a Person on the Metadata Entity Graph + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the group

+
+type
+EntityType! +		 +

A standard Entity Type

+
+name
+String! +		 +

Group name eg wherehows dev, ask_metadata

+
+ownership
+Ownership +		 +

Ownership metadata of the Corp Group

+
+properties
+CorpGroupProperties +		 +

Additional read only properties about the group

+
+editableProperties
+CorpGroupEditableProperties +		 +

Additional read write properties about the group

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+origin
+Origin +		 +

Origin info about this group.

+
+info
+CorpGroupInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +Additional read only info about the group

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## CorpGroupEditableProperties + +Additional read write properties about a group + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+description
+String +		 +

DataHub description of the group

+
+slack
+String +		 +

Slack handle for the group

+
+email
+String +		 +

Email address for the group

+
+pictureLink
+String +		 +

A URL which points to a picture which user wants to set as a profile photo

+
+ +## CorpGroupInfo + +Deprecated, use CorpUserProperties instead +Additional read only info about a group + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+displayName
+String +		 +

The name to display when rendering the group

+
+description
+String +		 +

The description provided for the group

+
+email
+String +		 +

email of this group

+
+admins
+[CorpUser!] +		 +
Deprecated: No longer supported
+ +

Deprecated, do not use +owners of this group

+
+members
+[CorpUser!] +		 +
Deprecated: No longer supported
+ +

Deprecated, use relationship IsMemberOfGroup instead +List of ldap urn in this group

+
+groups
+[String!] +		 +
Deprecated: No longer supported
+ +

Deprecated, do not use +List of groups urns in this group

+
+ +## CorpGroupProperties + +Additional read only properties about a group + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+displayName
+String +		 +

display name of this group

+
+description
+String +		 +

The description provided for the group

+
+email
+String +		 +

email of this group

+
+slack
+String +		 +

Slack handle for the group

+
+ +## CorpUser + +A DataHub User entity, which represents a Person on the Metadata Entity Graph + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the user

+
+type
+EntityType! +		 +

The standard Entity Type

+
+username
+String! +		 +

A username associated with the user +This uniquely identifies the user within DataHub

+
+properties
+CorpUserProperties +		 +

Additional read only properties about the corp user

+
+editableProperties
+CorpUserEditableProperties +		 +

Read write properties about the corp user

+
+status
+CorpUserStatus +		 +

The status of the user

+
+tags
+GlobalTags +		 +

The tags associated with the user

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+isNativeUser
+Boolean +		 +

Whether or not this user is a native DataHub user

+
+info
+CorpUserInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +Additional read only info about the corp user

+
+editableInfo
+CorpUserEditableInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use editableProperties field instead +Read write info about the corp user

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use the tags field instead +The structured tags associated with the user

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+settings
+CorpUserSettings +		 +

Settings that a user can customize through the datahub ui

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra aspects that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## CorpUserAppearanceSettings + +Settings that control look and feel of the DataHub UI for the user + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+showSimplifiedHomepage
+Boolean +		 +

Flag whether the user should see a homepage with only datasets, charts & dashboards. Intended for users +who have less operational use cases for the datahub tool.

+
+showThemeV2
+Boolean +		 +

Flag controlling whether the V2 UI for DataHub is shown.

+
+ +## CorpUserEditableInfo + +Deprecated, use CorpUserEditableProperties instead +Additional read write info about a user + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+displayName
+String +		 +

Display name to show on DataHub

+
+title
+String +		 +

Title to show on DataHub

+
+aboutMe
+String +		 +

About me section of the user

+
+teams
+[String!] +		 +

Teams that the user belongs to

+
+skills
+[String!] +		 +

Skills that the user possesses

+
+pictureLink
+String +		 +

A URL which points to a picture which user wants to set as a profile photo

+
+ +## CorpUserEditableProperties + +Additional read write properties about a user + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+displayName
+String +		 +

Display name to show on DataHub

+
+title
+String +		 +

Title to show on DataHub

+
+aboutMe
+String +		 +

About me section of the user

+
+teams
+[String!] +		 +

Teams that the user belongs to

+
+skills
+[String!] +		 +

Skills that the user possesses

+
+pictureLink
+String +		 +

A URL which points to a picture which user wants to set as a profile photo

+
+slack
+String +		 +

The slack handle of the user

+
+phone
+String +		 +

Phone number for the user

+
+email
+String +		 +

Email address for the user

+
+persona
+DataHubPersona +		 +

User persona, if present

+
+platforms
+[DataPlatform!] +		 +

Platforms commonly used by the user, if present.

+
+ +## CorpUserHomePageSettings + +Settings related to the home page for a user + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+pageTemplate
+DataHubPageTemplate +		 +

The default page template for the User.

+
+dismissedAnnouncementUrns
+[String] +		 +

List of urns of the announcements dismissed by the User.

+
+ +## CorpUserInfo + +Deprecated, use CorpUserProperties instead +Additional read only info about a user + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+active
+Boolean! +		 +

Whether the user is active

+
+displayName
+String +		 +

Display name of the user

+
+email
+String +		 +

Email address of the user

+
+title
+String +		 +

Title of the user

+
+manager
+CorpUser +		 +

Direct manager of the user

+
+departmentId
+Long +		 +

department id the user belong to

+
+departmentName
+String +		 +

department name this user belong to

+
+firstName
+String +		 +

first name of the user

+
+lastName
+String +		 +

last name of the user

+
+fullName
+String +		 +

Common name of this user, format is firstName plus lastName

+
+countryCode
+String +		 +

two uppercase letters country code

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the ldap

+
+ +## CorpUserProperties + +Additional read only properties about a user + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+active
+Boolean! +		 +

Whether the user is active

+
+displayName
+String +		 +

Display name of the user

+
+email
+String +		 +

Email address of the user

+
+title
+String +		 +

Title of the user

+
+manager
+CorpUser +		 +

Direct manager of the user

+
+departmentId
+Long +		 +

department id the user belong to

+
+departmentName
+String +		 +

department name this user belong to

+
+firstName
+String +		 +

first name of the user

+
+lastName
+String +		 +

last name of the user

+
+fullName
+String +		 +

Common name of this user, format is firstName plus lastName

+
+countryCode
+String +		 +

two uppercase letters country code

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the ldap

+
+ +## CorpUserSettings + +Settings that a user can customize through the datahub ui + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+appearance
+CorpUserAppearanceSettings +		 +

Settings that control look and feel of the DataHub UI for the user

+
+views
+CorpUserViewsSettings +		 +

Settings related to the DataHub Views feature

+
+homePage
+CorpUserHomePageSettings +		 +

Settings related to the home page for a user

+
+ +## CorpUserViewsSettings + +Settings related to the Views feature of DataHub. + +

Fields

+ + + + + + + + + +
NameDescription
+defaultView
+DataHubView +		 +

The default view for the User.

+
+ +## Cost + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+costType
+CostType! +		 +

Type of Cost Code

+
+costValue
+CostValue! +		 +

Code to which the Cost of this entity should be attributed to ie organizational cost ID

+
+ +## CostValue + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+costId
+Float +		 +

Organizational Cost ID

+
+costCode
+String +		 +

Organizational Cost Code

+
+ +## CreateDataHubFileResponse + +Response from creating a DataHub file + +

Fields

+ + + + + + + + + +
NameDescription
+file
+DataHubFile! +		 +

The created DataHub file entity

+
+ +## CronSchedule + +A cron schedule + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+cron
+String! +		 +

A cron-formatted execution interval, as a cron string, e.g. 1 * * * *

+
+timezone
+String! +		 +

Timezone in which the cron interval applies, e.g. America/Los_Angeles

+
+ +## CustomAssertionInfo + +Information about a custom assertion + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+String! +		 +

The type of custom assertion.

+
+entityUrn
+String! +		 +

The entity targeted by this custom assertion.

+
+field
+SchemaFieldRef +		 +

The field serving as input to the assertion, if any.

+
+logic
+String +		 +

Logic comprising a raw, unstructured assertion.

+
+ +## CustomPropertiesEntry + +An entry in a custom properties map represented as a tuple + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+key
+String! +		 +

The key of the map entry

+
+value
+String +		 +

The value fo the map entry

+
+associatedUrn
+String! +		 +

The urn of the entity this property came from for tracking purposes e.g. when sibling nodes are merged together

+
+ +## Dashboard + +A Dashboard Metadata Entity + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Dashboard

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+container
+Container +		 +

The parent container in which the entity resides

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+tool
+String! +		 +

The dashboard tool name +Note that this will soon be deprecated in favor of a standardized notion of Data Platform

+
+dashboardId
+String! +		 +

An id unique within the dashboard tool

+
+properties
+DashboardProperties +		 +

Additional read only properties about the dashboard

+
+editableProperties
+DashboardEditableProperties +		 +

Additional read write properties about the dashboard

+
+ownership
+Ownership +		 +

Ownership metadata of the dashboard

+
+status
+Status +		 +

Status metadata of the dashboard

+
+embed
+Embed +		 +

Embed information about the Dashboard

+
+deprecation
+Deprecation +		 +

The deprecation status of the dashboard

+
+tags
+GlobalTags +		 +

The tags associated with the dashboard

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dashboard

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dashboard

+
+domain
+DomainAssociation +		 +

The Domain associated with the Dashboard

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the dashboard. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+usageStats
+DashboardUsageQueryResult +		 +

Experimental (Subject to breaking change) -- Statistics about how this Dashboard is used

+ +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+startTimeMillis
+Long +		 + +
+endTimeMillis
+Long +		 + +
+limit
+Int +		 + +
+ +
+statsSummary
+DashboardStatsSummary +		 +

Experimental - Summary operational & usage statistics about a Dashboard

+
+info
+DashboardInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +Additional read only information about the dashboard

+
+editableInfo
+DashboardEditableProperties +		 +
Deprecated: No longer supported
+ +

Deprecated, use editableProperties instead +Additional read write properties about the Dashboard

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use tags field instead +The structured tags associated with the dashboard

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the dashboard is defined

+
+inputFields
+InputFields +		 +

Input fields that power all the charts in the dashboard

+
+subTypes
+SubTypes +		 +

Sub Types of the dashboard

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+health
+[Health!] +		 +

Experimental! The resolved health statuses of the asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+incidents
+EntityIncidentsResult +		 +

Incidents associated with the Dashboard

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState +		 +

Optional incident state to filter by, defaults to any state.

+
+stage
+IncidentStage +		 +

Optional incident stage to filter by, defaults to any state.

+
+priority
+IncidentPriority +		 +

Optional incident priority to filter by, defaults to any state.

+
+assigneeUrns
+[String!] +		 +

Optional assignee urns for an incident.

+
+start
+Int +		 +

Optional start offset, defaults to 0.

+
+count
+Int +		 +

Optional start offset, defaults to 20.

+
+ +
+ +## DashboardEditableProperties + +Dashboard properties that are editable via the UI This represents logical metadata, +as opposed to technical metadata + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

Description of the Dashboard

+
+ +## DashboardInfo + +Deprecated, use DashboardProperties instead +Additional read only info about a Dashboard + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display of the dashboard

+
+description
+String +		 +

Description of the dashboard

+
+charts
+[Chart!]! +		 +
Deprecated: No longer supported
+ +

Deprecated, use relationship Contains instead +Charts that comprise the dashboard

+
+externalUrl
+String +		 +

Native platform URL of the dashboard

+
+access
+AccessLevel +		 +

Access level for the dashboard +Note that this will soon be deprecated for low usage

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+lastRefreshed
+Long +		 +

The time when this dashboard last refreshed

+
+created
+AuditStamp! +		 +

An AuditStamp corresponding to the creation of this dashboard

+
+lastModified
+AuditStamp! +		 +

An AuditStamp corresponding to the modification of this dashboard

+
+deleted
+AuditStamp +		 +

An optional AuditStamp corresponding to the deletion of this dashboard

+
+ +## DashboardProperties + +Additional read only properties about a Dashboard + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display of the dashboard

+
+description
+String +		 +

Description of the dashboard

+
+externalUrl
+String +		 +

Native platform URL of the dashboard

+
+access
+AccessLevel +		 +

Access level for the dashboard +Note that this will soon be deprecated for low usage

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+lastRefreshed
+Long +		 +

The time when this dashboard last refreshed

+
+created
+AuditStamp! +		 +

An AuditStamp corresponding to the creation of this dashboard

+
+lastModified
+AuditStamp! +		 +

An AuditStamp corresponding to the modification of this dashboard

+
+deleted
+AuditStamp +		 +

An optional AuditStamp corresponding to the deletion of this dashboard

+
+ +## DashboardStatsSummary + +Experimental - subject to change. A summary of usage metrics about a Dashboard. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+viewCount
+Int +		 +

The total view count for the dashboard

+
+viewCountLast30Days
+Int +		 +

The view count in the last 30 days

+
+uniqueUserCountLast30Days
+Int +		 +

The unique user count in the past 30 days

+
+topUsersLast30Days
+[CorpUser!] +		 +

The top users in the past 30 days

+
+ +## DashboardUsageAggregation + +An aggregation of Dashboard usage statistics + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+bucket
+Long +		 +

The time window start time

+
+duration
+WindowDuration +		 +

The time window span

+
+resource
+String +		 +

The resource urn associated with the usage information, eg a Dashboard urn

+
+metrics
+DashboardUsageAggregationMetrics +		 +

The rolled up usage metrics

+
+ +## DashboardUsageAggregationMetrics + +Rolled up metrics about Dashboard usage over time + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+uniqueUserCount
+Int +		 +

The unique number of dashboard users within the time range

+
+viewsCount
+Int +		 +

The total number of dashboard views within the time range

+
+executionsCount
+Int +		 +

The total number of dashboard executions within the time range

+
+ +## DashboardUsageMetrics + +A set of absolute dashboard usage metrics + +

Implements

+ +- [TimeSeriesAspect](/docs/graphql/interfaces#timeseriesaspect) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+timestampMillis
+Long! +		 +

The time at which the metrics were reported

+
+favoritesCount
+Int +		 +

The total number of times dashboard has been favorited +FIXME: Qualifies as Popularity Metric rather than Usage Metric?

+
+viewsCount
+Int +		 +

The total number of dashboard views

+
+executionsCount
+Int +		 +

The total number of dashboard execution

+
+lastViewed
+Long +		 +

The time when this dashboard was last viewed

+
+ +## DashboardUsageQueryResult + +The result of a dashboard usage query + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+buckets
+[DashboardUsageAggregation] +		 +

A set of relevant time windows for use in displaying usage statistics

+
+aggregations
+DashboardUsageQueryResultAggregations +		 +

A set of rolled up aggregations about the dashboard usage

+
+metrics
+[DashboardUsageMetrics!] +		 +

A set of absolute dashboard usage metrics

+
+ +## DashboardUsageQueryResultAggregations + +A set of rolled up aggregations about the Dashboard usage + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+uniqueUserCount
+Int +		 +

The count of unique Dashboard users within the queried time range

+
+users
+[DashboardUserUsageCounts] +		 +

The specific per user usage counts within the queried time range

+
+viewsCount
+Int +		 +

The total number of dashboard views within the queried time range

+
+executionsCount
+Int +		 +

The total number of dashboard executions within the queried time range

+
+ +## DashboardUserUsageCounts + +Information about individual user usage of a Dashboard + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+user
+CorpUser +		 +

The user of the Dashboard

+
+viewsCount
+Int +		 +

number of times dashboard has been viewed by the user

+
+executionsCount
+Int +		 +

number of dashboard executions by the user

+
+usageCount
+Int +		 +

Normalized numeric metric representing user's dashboard usage +Higher value represents more usage

+
+ +## DataContract + +A Data Contract Entity. A Data Contract is a verifiable group of assertions regarding various aspects of the data: its freshness (sla), +schema, and data quality or validity. This group of assertions represents a data owner's commitment to producing data that confirms to the agreed +upon contract. Each dataset can have a single contract. The contract can be in a "passing" or "violating" state, depending +on whether the assertions that compose the contract are passing or failing. +Note that the data contract entity is currently in early preview (beta). + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key of the data contract

+
+type
+EntityType! +		 +

The standard entity type

+
+properties
+DataContractProperties +		 +

Properties describing the data contract

+
+status
+DataContractStatus +		 +

The status of the data contract

+
+structuredProperties
+StructuredProperties +		 +

Structured properties about this Data Contract

+
+relationships
+EntityRelationshipsResult +		 +

List of relationships between the source Entity and some destination entities with a given types

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataContractProperties + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entityUrn
+String! +		 +

The urn of the related entity, e.g. the Dataset today. In the future, we may support additional contract entities.

+
+freshness
+[FreshnessContract!] +		 +

The Freshness (SLA) portion of the contract. +As of today, it is expected that there will not be more than 1 Freshness contract. If there are, only the first will be displayed.

+
+schema
+[SchemaContract!] +		 +

The schema / structural portion of the contract. +As of today, it is expected that there will not be more than 1 Schema contract. If there are, only the first will be displayed.

+
+dataQuality
+[DataQualityContract!] +		 +

A set of data quality related contracts, e.g. table and column-level contract constraints.

+
+ +## DataContractStatus + + + +

Fields

+ + + + + + + + + +
NameDescription
+state
+DataContractState! +		 +

The state of the data contract

+
+ +## DataFlow + +A Data Flow Metadata Entity, representing an set of pipelined Data Job or Tasks required +to produce an output Dataset Also known as a Data Pipeline + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) +- [HasExecutionRuns](/docs/graphql/interfaces#hasexecutionruns) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of a Data Flow

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+orchestrator
+String! +		 +

Workflow orchestrator ei Azkaban, Airflow

+
+flowId
+String! +		 +

Id of the flow

+
+cluster
+String! +		 +

Cluster of the flow

+
+properties
+DataFlowProperties +		 +

Additional read only properties about a Data flow

+
+editableProperties
+DataFlowEditableProperties +		 +

Additional read write properties about a Data Flow

+
+ownership
+Ownership +		 +

Ownership metadata of the flow

+
+tags
+GlobalTags +		 +

The tags associated with the dataflow

+
+status
+Status +		 +

Status metadata of the dataflow

+
+deprecation
+Deprecation +		 +

The deprecation status of the Data Flow

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dashboard

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dashboard

+
+domain
+DomainAssociation +		 +

The Domain associated with the DataFlow

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+container
+Container +		 +

The parent container in which the entity resides

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the data flow. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+info
+DataFlowInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +Additional read only information about a Data flow

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use tags field instead +The structured tags associated with the dataflow

+
+dataJobs
+DataFlowDataJobsRelationships +		 +
Deprecated: No longer supported
+ +

Deprecated, use relationship IsPartOf instead +Data Jobs

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the datflow is defined

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+health
+[Health!] +		 +

Experimental! The resolved health statuses of the asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+subTypes
+SubTypes +		 +

Sub Types that this entity implements

+
+incidents
+EntityIncidentsResult +		 +

Incidents associated with the DataFlow

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState +		 +

Optional incident state to filter by, defaults to any state.

+
+stage
+IncidentStage +		 +

Optional incident stage to filter by, defaults to any state.

+
+priority
+IncidentPriority +		 +

Optional incident priority to filter by, defaults to any state.

+
+assigneeUrns
+[String!] +		 +

Optional assignee urns for an incident.

+
+start
+Int +		 +

Optional start offset, defaults to 0.

+
+count
+Int +		 +

Optional start offset, defaults to 20.

+
+ +
+runs
+DataProcessInstanceResult +		 + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+start
+Int +		 + +
+count
+Int +		 + +
+ +
+ +## DataFlowDataJobsRelationships + +Deprecated, use relationships query instead + +

Fields

+ + + + + + + + + +
NameDescription
+entities
+[EntityRelationshipLegacy] +		 + +
+ +## DataFlowEditableProperties + +Data Flow properties that are editable via the UI This represents logical metadata, +as opposed to technical metadata + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

Description of the Data Flow

+
+ +## DataFlowInfo + +Deprecated, use DataFlowProperties instead +Additional read only properties about a Data Flow aka Pipeline + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the flow

+
+description
+String +		 +

Description of the flow

+
+project
+String +		 +

Optional project or namespace associated with the flow

+
+externalUrl
+String +		 +

External URL associated with the DataFlow

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+ +## DataFlowProperties + +Additional read only properties about a Data Flow aka Pipeline + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the flow

+
+description
+String +		 +

Description of the flow

+
+project
+String +		 +

Optional project or namespace associated with the flow

+
+externalUrl
+String +		 +

External URL associated with the DataFlow

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+ +## DataHubConnection + +A connection between DataHub and an external Platform. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the connection

+
+type
+EntityType! +		 +

The standard Entity Type field

+
+details
+DataHubConnectionDetails! +		 +

The connection details

+
+platform
+DataPlatform! +		 +

The external Data Platform associated with the connection

+
+relationships
+EntityRelationshipsResult +		 +

Not implemented!

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataHubConnectionDetails + +The details of the Connection + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+DataHubConnectionDetailsType! +		 +

The type or format of connection

+
+json
+DataHubJsonConnection +		 +

A JSON-encoded connection. Present when type is JSON.

+
+name
+String +		 +

The name for this DataHub connection

+
+ +## DataHubFile + +A DataHub file entity representing a file stored in S3 + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the DataHub File

+
+type
+EntityType! +		 +

A standard Entity Type

+
+info
+DataHubFileInfo! +		 +

The main information about a DataHub file

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataHubFileInfo + +Information about a DataHub file + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+bucketStorageLocation
+BucketStorageLocation! +		 +

Info about where a file is stored

+
+originalFileName
+String! +		 +

The original filename as uploaded by the user

+
+mimeType
+String! +		 +

MIME type of the file (e.g., image/png, application/pdf)

+
+sizeInBytes
+Long! +		 +

Size of the file in bytes

+
+scenario
+UploadDownloadScenario! +		 +

The scenario/context in which this file was uploaded

+
+referencedByAsset
+Entity +		 +

Optional entity this file is associated with

+
+schemaField
+SchemaFieldEntity +		 +

The dataset schema field this file is referenced by

+
+created
+ResolvedAuditStamp! +		 +

Audit stamp for when and by whom this file was created

+
+ +## DataHubJsonConnection + +The details of a JSON Connection + +

Fields

+ + + + + + + + + +
NameDescription
+blob
+String! +		 +

The JSON blob containing the specific connection details.

+
+ +## DataHubPageModule + +A Page Module used for rendering custom or default layouts in the UI + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the Page Module

+
+type
+EntityType! +		 +

A standard Entity Type

+
+exists
+Boolean +		 +

Whether or not this module exists on DataHub

+
+properties
+DataHubPageModuleProperties! +		 +

The main properties of a DataHub page module

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataHubPageModuleParams + +The specific parameters stored for a module + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+linkParams
+LinkModuleParams +		 +

The params required if the module is type LINK

+
+richTextParams
+RichTextModuleParams +		 +

The params required if the module is type RICH_TEXT

+
+assetCollectionParams
+AssetCollectionModuleParams +		 +

The params required if the module is type ASSET_COLLECTION

+
+hierarchyViewParams
+HierarchyViewModuleParams +		 +

The params required if the module is type HIERARCHY_VIEW

+
+ +## DataHubPageModuleProperties + +The main properties of a DataHub page module + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The display name of this module

+
+type
+DataHubPageModuleType! +		 +

Info about the surface area of the product that this module is deployed in

+
+visibility
+DataHubPageModuleVisibility! +		 +

Info about the visibility of this module

+
+params
+DataHubPageModuleParams! +		 +

The specific parameters stored for this module

+
+created
+ResolvedAuditStamp! +		 +

Audit stamp for when and by whom this module was created

+
+lastModified
+ResolvedAuditStamp! +		 +

Audit stamp for when and by whom this module was last updated

+
+ +## DataHubPageModuleVisibility + +Info about the visibility of this module + +

Fields

+ + + + + + + + + +
NameDescription
+scope
+PageModuleScope +		 +

The scope of this module and who can use/see it

+
+ +## DataHubPageTemplate + +A Page Template used for rendering custom or default layouts in the UI + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the Page Template

+
+type
+EntityType! +		 +

A standard Entity Type

+
+properties
+DataHubPageTemplateProperties! +		 +

The main properties of a DataHub page template

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataHubPageTemplateAssetSummary + +The page template info for asset summaries + +

Fields

+ + + + + + + + + +
NameDescription
+summaryElements
+[SummaryElement!] +		 +

The list of properties shown on an asset summary page header.

+
+ +## DataHubPageTemplateProperties + +The main properties of a DataHub page template + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+rows
+[DataHubPageTemplateRow!]! +		 +

The rows of modules contained in this template

+
+assetSummary
+DataHubPageTemplateAssetSummary +		 +

The optional info for asset summaries. Should be populated if surfaceType is ASSET_SUMMARY

+
+surface
+DataHubPageTemplateSurface! +		 +

Info about the surface area of the product that this template is deployed in

+
+visibility
+DataHubPageTemplateVisibility! +		 +

Info about the visibility of this template

+
+created
+ResolvedAuditStamp! +		 +

Audit stamp for when and by whom this template was created

+
+lastModified
+ResolvedAuditStamp! +		 +

Audit stamp for when and by whom this template was last updated

+
+ +## DataHubPageTemplateRow + +A row of modules contained in a template + +

Fields

+ + + + + + + + + +
NameDescription
+modules
+[DataHubPageModule!]! +		 +

The modules that exist in this template row

+
+ +## DataHubPageTemplateSurface + +Info about the surface area of the product that this template is deployed in + +

Fields

+ + + + + + + + + +
NameDescription
+surfaceType
+PageTemplateSurfaceType +		 +

Where exactly is this template bing used

+
+ +## DataHubPageTemplateVisibility + +Info about the visibility of this template + +

Fields

+ + + + + + + + + +
NameDescription
+scope
+PageTemplateScope +		 +

The scope of this template and who can use/see it

+
+ +## DataHubPersona + +A standardized type of a user + +

Fields

+ + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the persona type

+
+ +## DataHubPolicy + +An DataHub Platform Access Policy - Policies determine who can perform what actions against which resources on the platform + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Policy

+
+type
+EntityType! +		 +

The standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from the Role

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+policyType
+PolicyType! +		 +

The type of the Policy

+
+name
+String! +		 +

The name of the Policy

+
+state
+PolicyState! +		 +

The present state of the Policy

+
+description
+String +		 +

The description of the Policy

+
+resources
+ResourceFilter +		 +

The resources that the Policy privileges apply to

+
+privileges
+[String!]! +		 +

The privileges that the Policy grants

+
+actors
+ActorFilter! +		 +

The actors that the Policy grants privileges to

+
+editable
+Boolean! +		 +

Whether the Policy is editable, ie system policies, or not

+
+ +## DataHubRole + +A DataHub Role is a high-level abstraction on top of Policies that dictates what actions users can take. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the role

+
+type
+EntityType! +		 +

The standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from the Role

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+name
+String! +		 +

The name of the Role.

+
+description
+String! +		 +

The description of the Role

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+ +## DataHubView + +An DataHub View - Filters that are applied across the application automatically. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the View

+
+type
+EntityType! +		 +

The standard Entity Type

+
+viewType
+DataHubViewType! +		 +

The type of the View

+
+name
+String! +		 +

The name of the View

+
+description
+String +		 +

The description of the View

+
+definition
+DataHubViewDefinition! +		 +

The definition of the View

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from the View

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataHubViewDefinition + +An DataHub View Definition + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+entityTypes
+[EntityType!]! +		 +

A set of filters to apply. If left empty, then ALL entity types are in scope.

+
+filter
+DataHubViewFilter! +		 +

A set of filters to apply. If left empty, then no filters will be applied.

+
+ +## DataHubViewFilter + +A DataHub View Filter. Note that + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+operator
+LogicalOperator! +		 +

The operator used to combine the filters.

+
+filters
+[FacetFilter!]! +		 +

A set of filters combined using the operator. If left empty, then no filters will be applied.

+
+ +## DataJob + +A Data Job Metadata Entity, representing an individual unit of computation or Task +to produce an output Dataset Always part of a parent Data Flow aka Pipeline + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) +- [HasExecutionRuns](/docs/graphql/interfaces#hasexecutionruns) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Data Job

+
+type
+EntityType! +		 +

A standard Entity Type

+
+subTypes
+SubTypes +		 +

Sub Types that this entity implements

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+dataFlow
+DataFlow +		 +

Deprecated, use relationship IsPartOf instead +The associated data flow

+
+jobId
+String! +		 +

Id of the job

+
+properties
+DataJobProperties +		 +

Additional read only properties associated with the Data Job

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+platform
+DataPlatform +		 +

Standardized platform urn where the data job is defined

+
+container
+Container +		 +

The parent container in which the entity resides

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+editableProperties
+DataJobEditableProperties +		 +

Additional read write properties associated with the Data Job

+
+tags
+GlobalTags +		 +

The tags associated with the DataJob

+
+ownership
+Ownership +		 +

Ownership metadata of the job

+
+status
+Status +		 +

Status metadata of the DataJob

+
+deprecation
+Deprecation +		 +

The deprecation status of the Data Flow

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dashboard

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dashboard

+
+domain
+DomainAssociation +		 +

The Domain associated with the Data Job

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the data job. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+info
+DataJobInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +Additional read only information about a Data processing job

+
+inputOutput
+DataJobInputOutput +		 +

Information about the inputs and outputs of a Data processing job including column-level lineage.

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use the tags field instead +The structured tags associated with the DataJob

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+health
+[Health!] +		 +

Experimental! The resolved health statuses of the asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+dataTransformLogic
+DataTransformLogic +		 +

Data Transform Logic associated with the Data Job

+
+incidents
+EntityIncidentsResult +		 +

Incidents associated with the DataJob

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState +		 +

Optional incident state to filter by, defaults to any state.

+
+stage
+IncidentStage +		 +

Optional incident stage to filter by, defaults to any state.

+
+priority
+IncidentPriority +		 +

Optional incident priority to filter by, defaults to any state.

+
+assigneeUrns
+[String!] +		 +

Optional assignee urns for an incident.

+
+start
+Int +		 +

Optional start offset, defaults to 0.

+
+count
+Int +		 +

Optional start offset, defaults to 20.

+
+ +
+runs
+DataProcessInstanceResult +		 + + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+start
+Int +		 + +
+count
+Int +		 + +
+ +
+ +## DataJobEditableProperties + +Data Job properties that are editable via the UI This represents logical metadata, +as opposed to technical metadata + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

Description of the Data Job

+
+ +## DataJobInfo + +Deprecated, use DataJobProperties instead +Additional read only information about a Data Job aka Task + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Job display name

+
+description
+String +		 +

Job description

+
+externalUrl
+String +		 +

External URL associated with the DataJob

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+ +## DataJobInputOutput + +The lineage information for a DataJob +TODO Rename this to align with other Lineage models + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+inputDatasets
+[Dataset!] +		 +
Deprecated: No longer supported
+ +

Deprecated, use relationship Consumes instead +Input datasets produced by the data job during processing

+
+outputDatasets
+[Dataset!] +		 +
Deprecated: No longer supported
+ +

Deprecated, use relationship Produces instead +Output datasets produced by the data job during processing

+
+inputDatajobs
+[DataJob!] +		 +
Deprecated: No longer supported
+ +

Deprecated, use relationship DownstreamOf instead +Input datajobs that this data job depends on

+
+fineGrainedLineages
+[FineGrainedLineage!] +		 +

Lineage information for the column-level. Includes a list of objects +detailing which columns are upstream and which are downstream of each other. +The upstream and downstream columns are from datasets.

+
+ +## DataJobProperties + +Additional read only properties about a Data Job aka Task + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Job display name

+
+description
+String +		 +

Job description

+
+externalUrl
+String +		 +

External URL associated with the DataJob

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+ +## DataPlatform + +A Data Platform represents a specific third party Data System or Tool Examples include +warehouses like Snowflake, orchestrators like Airflow, and dashboarding tools like Looker + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the data platform

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+name
+String! +		 +

Name of the data platform

+
+properties
+DataPlatformProperties +		 +

Additional read only properties associated with a data platform

+
+displayName
+String +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties displayName instead +Display name of the data platform

+
+info
+DataPlatformInfo +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +Additional properties associated with a data platform

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataPlatformInfo + +Deprecated, use DataPlatformProperties instead +Additional read only information about a Data Platform + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+PlatformType! +		 +

The platform category

+
+displayName
+String +		 +

Display name associated with the platform

+
+datasetNameDelimiter
+String! +		 +

The delimiter in the dataset names on the data platform

+
+logoUrl
+String +		 +

A logo URL associated with the platform

+
+ +## DataPlatformInstance + +A Data Platform instance represents an instance of a 3rd party platform like Looker, Snowflake, etc. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the data platform

+
+type
+EntityType! +		 +

A standard Entity Type

+
+platform
+DataPlatform! +		 +

Name of the data platform

+
+instanceId
+String! +		 +

The platform instance id

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+properties
+DataPlatformInstanceProperties +		 +

Additional read only properties associated with a data platform instance

+
+ownership
+Ownership +		 +

Ownership metadata of the data platform instance

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the data platform instance

+
+tags
+GlobalTags +		 +

Tags used for searching the data platform instance

+
+deprecation
+Deprecation +		 +

The deprecation status of the data platform instance

+
+status
+Status +		 +

Status metadata of the container

+
+ +## DataPlatformInstanceProperties + +Additional read only properties about a DataPlatformInstance + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

The name of the data platform instance used in display

+
+description
+String +		 +

Read only technical description for the data platform instance

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the data platform instance

+
+externalUrl
+String +		 +

External URL associated with the data platform instance

+
+ +## DataPlatformProperties + +Additional read only properties about a Data Platform + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+PlatformType! +		 +

The platform category

+
+displayName
+String +		 +

Display name associated with the platform

+
+datasetNameDelimiter
+String! +		 +

The delimiter in the dataset names on the data platform

+
+logoUrl
+String +		 +

A logo URL associated with the platform

+
+ +## DataProcessInstance + +A DataProcessInstance Metadata Entity, representing an individual run of +a task or datajob. + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the DataProcessInstance

+
+type
+EntityType! +		 +

The standard Entity Type

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+status
+Status +		 +

Status metadata of the data process instance

+
+state
+[DataProcessRunEvent] +		 +

The history of state changes for the run

+ +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+startTimeMillis
+Long +		 + +
+endTimeMillis
+Long +		 + +
+limit
+Int +		 + +
+ +
+created
+AuditStamp +		 +
Deprecated: Use `properties.created`
+ +

When the run was kicked off

+
+name
+String +		 +
Deprecated: Use `properties.name`
+ +

The name of the data process

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity. +In the UI, used for inputs, outputs and parentTemplate

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+externalUrl
+String +		 +

The link to view the task run in the source system

+
+properties
+DataProcessInstanceProperties +		 +

Additional read only properties associated with the Data Process Instance

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+subTypes
+SubTypes +		 +

Sub Types that this entity implements

+
+container
+Container +		 +

The parent container in which the entity resides

+
+platform
+DataPlatform +		 +

Standardized platform urn where the data process instance is defined

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+mlTrainingRunProperties
+MLTrainingRunProperties +		 +

Additional properties when subtype is Training Run

+
+parentTemplate
+Entity +		 +

The parent entity whose run instance it is

+
+ +## DataProcessInstanceProperties + +Properties describing a data process instance's execution metadata + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The display name of this process instance

+
+externalUrl
+String +		 +

URL to view this process instance in the external system

+
+created
+AuditStamp! +		 +

When this process instance was created

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Additional custom properties specific to this process instance

+
+ +## DataProcessInstanceResult + +Data Process instances that match the provided query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+count
+Int +		 +

The number of entities to include in result set

+
+start
+Int +		 +

The offset of the result set

+
+total
+Int +		 +

The total number of run events returned

+
+runs
+[DataProcessInstance] +		 +

The data process instances that produced or consumed the entity

+
+ +## DataProcessInstanceRunResult + +the result of a run, part of the run state + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+resultType
+DataProcessInstanceRunResultType +		 +

The outcome of the run

+
+nativeResultType
+String +		 +

The outcome of the run in the data platforms native language

+
+ +## DataProcessRunEvent + +A state change event in the data process instance lifecycle + +

Implements

+ +- [TimeSeriesAspect](/docs/graphql/interfaces#timeseriesaspect) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+status
+DataProcessRunStatus +		 +

The status of the data process instance

+
+attempt
+Int +		 +

The try number that this instance run is in

+
+result
+DataProcessInstanceRunResult +		 +

The result of a run

+
+timestampMillis
+Long! +		 +

The timestamp associated with the run event in milliseconds

+
+durationMillis
+Long +		 +

The duration of the run in milliseconds

+
+ +## DataProduct + +A Data Product, or a logical grouping of Metadata Entities + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Data Product

+
+type
+EntityType! +		 +

A standard Entity Type

+
+properties
+DataProductProperties +		 +

Properties about a Data Product

+
+ownership
+Ownership +		 +

Ownership metadata of the Data Product

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the Data Product

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+entities
+SearchResults +		 +

Children entities inside of the DataProduct

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchAcrossEntitiesInput +		 + +
+ +
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the Data Product

+
+domain
+DomainAssociation +		 +

The Domain associated with the Data Product

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the data product

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the data product

+
+tags
+GlobalTags +		 +

Tags used for searching Data Product

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+settings
+AssetSettings +		 +

Settings associated with this asset

+
+ +## DataProductProperties + +Properties about a domain + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the Data Product

+
+description
+String +		 +

Description of the Data Product

+
+externalUrl
+String +		 +

External URL for the DataProduct (most likely GitHub repo where Data Products are managed as code)

+
+numAssets
+Int +		 +

Number of children entities inside of the Data Product. This number includes soft deleted entities.

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the Data Product

+
+createdOn
+ResolvedAuditStamp +		 +

A Resolved Audit Stamp corresponding to the creation of this resource

+
+ +## DataQualityContract + + + +

Fields

+ + + + + + + + + +
NameDescription
+assertion
+Assertion! +		 +

The assertion representing the schema contract.

+
+ +## Dataset + +A Dataset entity, which encompasses Relational Tables, Document store collections, streaming topics, and other sets of data having an independent lifecycle + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) +- [HasLogicalParent](/docs/graphql/interfaces#haslogicalparent) +- [SupportsVersions](/docs/graphql/interfaces#supportsversions) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+contract
+DataContract +		 +

An optional Data Contract defined for the Dataset.

+
+urn
+String! +		 +

The primary key of the Dataset

+
+type
+EntityType! +		 +

The standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the dataset is defined

+
+container
+Container +		 +

The parent container in which the entity resides

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+name
+String! +		 +

Unique guid for dataset +No longer to be used as the Dataset display name. Use properties.name instead

+
+properties
+DatasetProperties +		 +

An additional set of read only properties

+
+editableProperties
+DatasetEditableProperties +		 +

An additional set of of read write properties

+
+ownership
+Ownership +		 +

Ownership metadata of the dataset

+
+deprecation
+Deprecation +		 +

The deprecation status of the dataset

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dataset

+
+schemaMetadata
+SchemaMetadata +		 +

Schema metadata of the dataset, available by version number

+ +

Arguments

+ + + + + + + + + +
NameDescription
+version
+Long +		 + +
+ +
+editableSchemaMetadata
+EditableSchemaMetadata +		 +

Editable schema metadata of the dataset

+
+status
+Status +		 +

Status of the Dataset

+
+embed
+Embed +		 +

Embed information about the Dataset

+
+tags
+GlobalTags +		 +

Tags used for searching dataset

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dataset

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+domain
+DomainAssociation +		 +

The Domain associated with the Dataset

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the dataset

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the dataset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+access
+Access +		 +

The Roles and the properties to access the dataset

+
+usageStats
+UsageQueryResult +		 +

Statistics about how this Dataset is used +The first parameter, resource, is deprecated and no longer needs to be provided +timeZone accepts standard IANA time zone identifier ie. America/New_York

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+resource
+String +		 + +
+range
+TimeRange +		 + +
+startTimeMillis
+Long +		 + +
+timeZone
+String +		 + +
+ +
+statsSummary
+DatasetStatsSummary +		 +

Experimental - Summary operational & usage statistics about a Dataset

+
+datasetProfiles
+[DatasetProfile!] +		 +

Profile Stats resource that retrieves the events in a previous unit of time in descending order +If no start or end time are provided, the most recent events will be returned

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+startTimeMillis
+Long +		 + +
+endTimeMillis
+Long +		 + +
+filter
+FilterInput +		 + +
+limit
+Int +		 + +
+ +
+operations
+[Operation!] +		 +

Operational events for an entity.

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+startTimeMillis
+Long +		 + +
+endTimeMillis
+Long +		 + +
+filter
+FilterInput +		 + +
+limit
+Int +		 + +
+ +
+assertions
+EntityAssertionsResult +		 +

Assertions associated with the Dataset

+ +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 + +
+count
+Int +		 + +
+includeSoftDeleted
+Boolean +		 + +
+ +
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the dataset. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+health
+[Health!] +		 +

Experimental! The resolved health statuses of the Dataset

+
+schema
+Schema +		 +
Deprecated: Use `schemaMetadata`
+ +

Schema metadata of the dataset

+
+externalUrl
+String +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +External URL associated with the Dataset

+
+origin
+FabricType! +		 +
Deprecated: No longer supported
+ +

Deprecated, see the properties field instead +Environment in which the dataset belongs to or where it was generated +Note that this field will soon be deprecated in favor of a more standardized concept of Environment

+
+description
+String +		 +
Deprecated: No longer supported
+ +

Deprecated, use the properties field instead +Read only technical description for dataset

+
+platformNativeType
+PlatformNativeType +		 +
Deprecated: No longer supported
+ +

Deprecated, do not use this field +The logical type of the dataset ie table, stream, etc

+
+uri
+String +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties instead +Native Dataset Uri +Uri should not include any environment specific properties

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use tags field instead +The structured tags associated with the dataset

+
+subTypes
+SubTypes +		 +

Sub Types that this entity implements

+
+viewProperties
+ViewProperties +		 +

View related properties. Only relevant if subtypes field contains view.

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+runs
+DataProcessInstanceResult +		 +

History of datajob runs that either produced or consumed this dataset

+ +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 + +
+count
+Int +		 + +
+direction
+RelationshipDirection! +		 + +
+ +
+siblings
+SiblingProperties +		 +

Metadata about the datasets siblings

+
+siblingsSearch
+ScrollResults +		 +

Executes a search on only the siblings of an entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ScrollAcrossEntitiesInput! +		 + +
+ +
+fineGrainedLineages
+[FineGrainedLineage!] +		 +

Lineage information for the column-level. Includes a list of objects +detailing which columns are upstream and which are downstream of each other. +The upstream and downstream columns are from datasets.

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+structuredProperties
+StructuredProperties +		 +

Structured properties about this Dataset

+
+operationsStats
+OperationsQueryResult +		 +

Statistics about how this Dataset has been operated on

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+OperationsStatsInput +		 + +
+ +
+incidents
+EntityIncidentsResult +		 +

Incidents associated with the Dataset

+ +

Arguments

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState +		 +

Optional incident state to filter by, defaults to any state.

+
+stage
+IncidentStage +		 +

Optional incident stage to filter by, defaults to any state.

+
+priority
+IncidentPriority +		 +

Optional incident priority to filter by, defaults to any state.

+
+assigneeUrns
+[String!] +		 +

Optional assignee urns for an incident.

+
+start
+Int +		 +

Optional start offset, defaults to 0.

+
+count
+Int +		 +

Optional start offset, defaults to 20.

+
+ +
+logicalParent
+Entity +		 +

If this entity represents a physical asset, this is its logical parent, from which metadata can propagate.

+
+settings
+AssetSettings +		 +

Settings associated with this asset

+
+testResults
+TestResults +		 +

The results of evaluating tests

+
+timeseriesCapabilities
+TimeseriesCapabilitiesResult +		 +

Returns a set of capabilities regarding our timerseries indices

+
+versionProperties
+VersionProperties +		 + +
+ +## DatasetAssertionInfo + +Detailed information about a Dataset Assertion + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+datasetUrn
+String! +		 +

The urn of the dataset that the assertion is related to

+
+scope
+DatasetAssertionScope! +		 +

The scope of the Dataset assertion.

+
+fields
+[SchemaFieldRef!] +		 +

The fields serving as input to the assertion. Empty if there are none.

+
+aggregation
+AssertionStdAggregation +		 +

Standardized assertion operator

+
+operator
+AssertionStdOperator! +		 +

Standardized assertion operator

+
+parameters
+AssertionStdParameters +		 +

Standard parameters required for the assertion. e.g. min_value, max_value, value, columns

+
+nativeType
+String +		 +

The native operator for the assertion. For Great Expectations, this will contain the original expectation name.

+
+nativeParameters
+[StringMapEntry!] +		 +

Native parameters required for the assertion.

+
+logic
+String +		 +

Logic comprising a raw, unstructured assertion.

+
+ +## DatasetDeprecation + +Deprecated, use Deprecation instead +Information about Dataset deprecation status +Note that this model will soon be migrated to a more general purpose Entity status + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+deprecated
+Boolean! +		 +

Whether the dataset has been deprecated by owner

+
+decommissionTime
+Long +		 +

The time user plan to decommission this dataset

+
+note
+String! +		 +

Additional information about the dataset deprecation plan

+
+actor
+String +		 +

The user who will be credited for modifying this deprecation content

+
+ +## DatasetEditableProperties + +Dataset properties that are editable via the UI This represents logical metadata, +as opposed to technical metadata + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+description
+String +		 +

Description of the Dataset

+
+name
+String +		 +

Editable name of the Dataset

+
+ +## DatasetFieldProfile + +An individual Dataset Field Profile + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+fieldPath
+String! +		 +

The standardized path of the field

+
+uniqueCount
+Long +		 +

The unique value count for the field across the Dataset

+
+uniqueProportion
+Float +		 +

The proportion of rows with unique values across the Dataset

+
+nullCount
+Long +		 +

The number of NULL row values across the Dataset

+
+nullProportion
+Float +		 +

The proportion of rows with NULL values across the Dataset

+
+min
+String +		 +

The min value for the field

+
+max
+String +		 +

The max value for the field

+
+mean
+String +		 +

The mean value for the field

+
+median
+String +		 +

The median value for the field

+
+stdev
+String +		 +

The standard deviation for the field

+
+sampleValues
+[String!] +		 +

A set of sample values for the field

+
+quantiles
+[Quantile!] +		 +

Sorted list of quantile cutoffs for the field, in ascending order +Only for numerical columns

+
+distinctValueFrequencies
+[ValueFrequency!] +		 +

Volume of each column value for a low-cardinality / categorical field

+
+ +## DatasetFilter + +Describes a generic filter on a dataset + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+DatasetFilterType! +		 +

Type of partition

+
+sql
+String +		 +

The raw query if using a SQL FilterType

+
+ +## DatasetProfile + +A Dataset Profile associated with a Dataset, containing profiling statistics about the Dataset + +

Implements

+ +- [TimeSeriesAspect](/docs/graphql/interfaces#timeseriesaspect) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+timestampMillis
+Long! +		 +

The time at which the profile was reported

+
+rowCount
+Long +		 +

An optional row count of the Dataset

+
+columnCount
+Long +		 +

An optional column count of the Dataset

+
+sizeInBytes
+Long +		 +

The storage size in bytes

+
+fieldProfiles
+[DatasetFieldProfile!] +		 +

An optional set of per field statistics obtained in the profile

+
+partitionSpec
+PartitionSpec +		 +

Information about the partition that was profiled

+
+ +## DatasetProperties + +Additional read only properties about a Dataset + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the dataset used in display

+
+qualifiedName
+String +		 +

Fully-qualified name of the Dataset

+
+origin
+FabricType! +		 +

Environment in which the dataset belongs to or where it was generated +Note that this field will soon be deprecated in favor of a more standardized concept of Environment

+
+description
+String +		 +

Read only technical description for dataset

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the Dataset

+
+externalUrl
+String +		 +

External URL associated with the Dataset

+
+created
+Long +		 +

Created timestamp millis associated with the Dataset

+
+createdActor
+String +		 +

Actor associated with the Dataset's created timestamp

+
+lastModified
+AuditStamp! +		 +

Last Modified timestamp millis associated with the Dataset

+
+lastModifiedActor
+String +		 +
Deprecated: No longer supported
+ +

Actor associated with the Dataset's lastModified timestamp. +Deprecated - Use lastModified.actor instead.

+
+ +## DatasetStatsSummary + +Experimental - subject to change. A summary of usage metrics about a Dataset. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+queryCountLast30Days
+Int +		 +

The query count in the past 30 days

+
+uniqueUserCountLast30Days
+Int +		 +

The unique user count in the past 30 days

+
+topUsersLast30Days
+[CorpUser!] +		 +

The top users in the past 30 days

+
+ +## DataTransform + +Information about a transformation applied to data assets + +

Fields

+ + + + + + + + + +
NameDescription
+queryStatement
+QueryStatement +		 +

The transformation may be defined by a query statement

+
+ +## DataTransformLogic + +Information about transformations applied to data assets + +

Fields

+ + + + + + + + + +
NameDescription
+transforms
+[DataTransform!]! +		 +

List of transformations applied

+
+ +## DataTypeEntity + +A data type registered in DataHub + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the Query

+
+type
+EntityType! +		 +

A standard Entity Type

+
+info
+DataTypeInfo! +		 +

Info about this type including its name

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## DataTypeInfo + +Properties about an individual data type + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+StdDataType! +		 +

The standard data type

+
+qualifiedName
+String! +		 +

The fully qualified name of the type. This includes its namespace

+
+displayName
+String +		 +

The display name of this type

+
+description
+String +		 +

The description of this type

+
+ +## DateRange + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+start
+String! +		 + +
+end
+String! +		 + +
+ +## DebugAccessResult + +Experimental API result to debug Access for users. +Backward incompatible changes will be made without notice in the future. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+roles
+[String!]! +		 +

Roles that the user has.

+
+groups
+[String!]! +		 +

Groups that the user belongs to.

+
+groupsWithRoles
+[String!]! +		 +

List of groups that the user is assigned to AND where the group has a role. +This is a subset of the groups property.

+
+rolesViaGroups
+[String!]! +		 +

Final set of roles that are coming through groups. +If not role assigned to groups, then this would be empty.

+
+allRoles
+[String!]! +		 +

Union of roles + rolesViaGroups that the user has.

+
+policies
+[String!]! +		 +

List of Policy that apply to this user directly or indirectly.

+
+privileges
+[String!]! +		 +

List of privileges that this user has directly or indirectly.

+
+ +## Deprecation + +Information about Metadata Entity deprecation status + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+deprecated
+Boolean! +		 +

Whether the entity has been deprecated by owner

+
+decommissionTime
+Long +		 +

The time user plan to decommission this entity

+
+note
+String +		 +

Additional information about the entity deprecation plan

+
+actor
+String +		 +

The user who will be credited for modifying this deprecation content

+
+actorEntity
+Entity +		 +

The hydrated user who will be credited for modifying this deprecation content

+
+replacement
+Entity +		 +

The optional replacement entity

+
+ +## DisplayProperties + +Properties related to how the entity is displayed in the Datahub UI + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+colorHex
+String +		 +

Color associated with the entity in Hex. For example #FFFFFF

+
+icon
+IconProperties +		 +

The icon associated with the entity

+
+ +## DocPropagationSettings + +Global (platform-level) settings related to the doc propagation feature + +

Fields

+ + + + + + + + + +
NameDescription
+docColumnPropagation
+Boolean +		 +

The default doc propagation setting for the platform.

+
+ +## Document + +A Document entity in DataHub + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Document

+
+type
+EntityType! +		 +

A standard Entity Type

+
+info
+DocumentInfo +		 +

Information about the Document

+
+settings
+DocumentSettings +		 +

Settings specific to the Document

+
+subType
+String +		 +

The sub-type of the Document (e.g., "FAQ", "Tutorial", "Reference", etc.)

+
+dataPlatformInstance
+DataPlatformInstance +		 +

Data Platform Instance associated with the Document

+
+platform
+DataPlatform! +		 +

The platform that this Document originates in. For example, "Notion" for representing docs ingested from Notion. +For documents natively authored on DataHub, the platform will be "DataHub".

+
+ownership
+Ownership +		 +

Ownership metadata of the Document

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+tags
+GlobalTags +		 +

Tags applied to the Document

+
+glossaryTerms
+GlossaryTerms +		 +

Glossary terms associated with the Document

+
+domain
+DomainAssociation +		 +

The Domain associated with the Document

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the Document (links)

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+documentation
+Documentation +		 +

Documentation aspect containing editable description for this Document

+
+changeHistory
+[DocumentChange!]! +		 +

Change history for this document. +Returns a chronological list of changes made to the document.

+ +

Arguments

+ + + + + + + + + + + + + + + + + +
NameDescription
+startTimeMillis
+Long +		 +

Start time in milliseconds since epoch (optional). +Defaults to 30 days ago if not specified.

+
+endTimeMillis
+Long +		 +

End time in milliseconds since epoch (optional). +Defaults to current time if not specified.

+
+limit
+Int +		 +

Maximum number of change entries to return. +Defaults to 50.

+
+ +
+parentDocuments
+ParentDocumentsResult +		 +

Recursively get the lineage of parent documents for this document. +Returns parents with direct parent first followed by the parent's parent, etc.

+
+ +## Documentation + +Object containing the documentation aspect for an entity + +

Fields

+ + + + + + + + + +
NameDescription
+documentations
+[DocumentationAssociation!]! +		 +

Structured properties on this entity

+
+ +## DocumentationAssociation + +Object containing the documentation aspect for an entity + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+documentation
+String! +		 +

Structured properties on this entity

+
+attribution
+MetadataAttribution +		 +

Information about who, why, and how this metadata was applied

+
+ +## DocumentChange + +A change made to a document. +Represents a single modification with timestamp, actor, and description. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+changeType
+DocumentChangeType! +		 +

Type of change that occurred

+
+description
+String! +		 +

Human-readable description of what changed

+
+actor
+CorpUser +		 +

User who made the change (optional, may not be available for all changes)

+
+timestamp
+Long! +		 +

When the change occurred (milliseconds since epoch)

+
+details
+[StringMapEntry!] +		 +

Additional context about the change (optional). +For example, if a document was moved, this might contain the old and new parent URNs.

+
+ +## DocumentContent + +The contents of a Document + +

Fields

+ + + + + + + + + +
NameDescription
+text
+String! +		 +

The text contents of the Document

+
+ +## DocumentInfo + +Information about a Document + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+title
+String +		 +

Optional title for the document

+
+source
+DocumentSource +		 +

Information about the external source of this document. +Only populated for third-party documents ingested from external systems. +If null, the document is first-party (created directly in DataHub).

+
+status
+DocumentStatus +		 +

Status of the Document (published, unpublished, etc.)

+
+contents
+DocumentContent! +		 +

Content of the Document

+
+created
+ResolvedAuditStamp! +		 +

The audit stamp for when the document was created

+
+lastModified
+ResolvedAuditStamp! +		 +

The audit stamp for when the document was last modified (any field)

+
+relatedAssets
+[DocumentRelatedAsset!] +		 +

Assets referenced by or related to this Document

+
+relatedDocuments
+[DocumentRelatedDocument!] +		 +

Documents referenced by or related to this Document

+
+parentDocument
+DocumentParentDocument +		 +

The parent document of this Document

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the Document

+
+ +## DocumentParentDocument + +The parent document of the document + +

Fields

+ + + + + + + + + +
NameDescription
+document
+Document! +		 +

The hierarchical parent document for this document

+
+ +## DocumentRelatedAsset + +A data asset referenced by a Document + +

Fields

+ + + + + + + + + +
NameDescription
+asset
+Entity! +		 +

The asset referenced by or related to the document

+
+ +## DocumentRelatedDocument + +A document referenced by or related to another Document + +

Fields

+ + + + + + + + + +
NameDescription
+document
+Document! +		 +

The document referenced by or related to the document

+
+ +## DocumentSettings + +Settings specific to a Document + +

Fields

+ + + + + + + + + +
NameDescription
+showInGlobalContext
+Boolean! +		 +

Whether or not this document should be visible in the global context (e.g., global navigation, knowledge base search). +When false, the document is accessible primarily through the assets it is related to. +When true, the document appears in the global documents space accessible to all users.

+
+ +## DocumentSource + +Information about the external source of a document + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+sourceType
+DocumentSourceType! +		 +

The type of the source

+
+externalUrl
+String +		 +

URL to the external source where this document originated

+
+externalId
+String +		 +

Unique identifier in the external system

+
+ +## DocumentStatus + +Status information for a Document + +

Fields

+ + + + + + + + + +
NameDescription
+state
+DocumentState! +		 +

The current state of the document

+
+ +## Domain + +A domain, or a logical grouping of Metadata Entities + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the domain

+
+type
+EntityType! +		 +

A standard Entity Type

+
+id
+String! +		 +

Id of the domain

+
+properties
+DomainProperties +		 +

Properties about a domain

+
+ownership
+Ownership +		 +

Ownership metadata of the dataset

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dataset

+
+entities
+SearchResults +		 +

Children entities inside of the Domain

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+DomainEntitiesInput +		 + +
+ +
+parentDomains
+ParentDomainsResult +		 +

Recursively get the lineage of parent domains for this entity

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+displayProperties
+DisplayProperties +		 +

Display properties for the domain

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+settings
+AssetSettings +		 +

Settings associated with this asset

+
+ +## DomainAssociation + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+domain
+Domain! +		 +

The domain related to the assocaited urn

+
+associatedUrn
+String! +		 +

Reference back to the tagged urn for tracking purposes e.g. when sibling nodes are merged together

+
+ +## DomainProperties + +Properties about a domain + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Display name of the domain

+
+description
+String +		 +

Description of the Domain

+
+createdOn
+ResolvedAuditStamp +		 +

A Resolved Audit Stamp corresponding to the creation of this resource

+
+ +## DownstreamEntityRelationships + +Deprecated, use relationships query instead + +

Fields

+ + + + + + + + + +
NameDescription
+entities
+[EntityRelationshipLegacy] +		 + +
+ +## EditableSchemaFieldInfo + +Editable schema field metadata ie descriptions, tags, etc + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+fieldPath
+String! +		 +

Flattened name of a field identifying the field the editable info is applied to

+
+description
+String +		 +

Edited description of the field

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use tags field instead +Tags associated with the field

+
+tags
+GlobalTags +		 +

Tags associated with the field

+
+glossaryTerms
+GlossaryTerms +		 +

Glossary terms associated with the field

+
+ +## EditableSchemaMetadata + +Information about schema metadata that is editable via the UI + +

Fields

+ + + + + + + + + +
NameDescription
+editableSchemaFieldInfo
+[EditableSchemaFieldInfo!]! +		 +

Editable schema field metadata

+
+ +## EditableTagProperties + +Additional read write Tag properties +Deprecated! Replaced by TagProperties. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

A display name for the Tag

+
+description
+String +		 +

A description of the Tag

+
+ +## Embed + +Information required to render an embedded version of an asset + +

Fields

+ + + + + + + + + +
NameDescription
+renderUrl
+String +		 +

A URL which can be rendered inside of an iframe.

+
+ +## EmbeddingConfig + +Embedding model configuration for generating semantic search vectors. +Only includes minimal settings required for clients to match server embedding behavior. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+provider
+String! +		 +

Embedding provider type (e.g., "aws-bedrock")

+
+modelId
+String! +		 +

Model identifier (e.g., "cohere.embed-english-v3")

+
+modelEmbeddingKey
+String! +		 +

Embedding storage key for SemanticContent aspects (e.g., "cohere_embed_v3"). +This is the canonical key used in the embeddings map when storing SemanticContent aspects. +Server-provided to ensure exact match between client (writing) and server (querying).

+
+awsProviderConfig
+AwsProviderConfig +		 +

AWS Bedrock-specific configuration (only populated when provider is "aws-bedrock")

+
+ +## EntityAssertionsResult + +A list of Assertions Associated with an Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of assertions in the returned result set

+
+total
+Int! +		 +

The total number of assertions in the result set

+
+assertions
+[Assertion!]! +		 +

The assertions themselves

+
+ +## EntityCountResult + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+entityType
+EntityType! +		 + +
+count
+Int! +		 + +
+ +## EntityCountResults + + + +

Fields

+ + + + + + + + + +
NameDescription
+counts
+[EntityCountResult!] +		 + +
+ +## EntityIncidentsResult + +A list of Incidents Associated with an Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of assertions in the returned result set

+
+total
+Int! +		 +

The total number of assertions in the result set

+
+incidents
+[Incident!]! +		 +

The incidents themselves

+
+ +## EntityLineageResult + +A list of lineage information associated with a source Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

Start offset of the result set

+
+count
+Int +		 +

Number of results in the returned result set

+
+total
+Int +		 +

Total number of results in the result set

+
+filtered
+Int +		 +

The number of results that were filtered out of the page (soft-deleted or non-existent)

+
+relationships
+[LineageRelationship!]! +		 +

Relationships in the result set

+
+ +## EntityPath + +An overview of the field that was matched in the entity search document + +

Fields

+ + + + + + + + + +
NameDescription
+path
+[Entity]! +		 +

Path of entities between source and destination nodes

+
+ +## EntityPrivileges + +Shared privileges object across entities. Not all privileges apply to every entity. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+canManageChildren
+Boolean +		 +

Whether or not a user can create child entities under a parent entity. +For example, can one create Terms/Node sunder a Glossary Node.

+
+canManageEntity
+Boolean +		 +

Whether or not a user can delete or move this entity.

+
+canEditLineage
+Boolean +		 +

Whether or not a user can create or delete lineage edges for an entity.

+
+canEditEmbed
+Boolean +		 +

Whether or not a user update the embed information

+
+canEditQueries
+Boolean +		 +

Whether or not a user can update the Queries for the entity (e.g. dataset)

+
+canEditProperties
+Boolean +		 +

Whether or not a user can update the properties for the entity (e.g. dataset)

+
+canEditTags
+Boolean +		 +

Whether or not a user can update tags for the entity

+
+canEditGlossaryTerms
+Boolean +		 +

Whether or not a user can update glossary terms for the entity

+
+canEditDescription
+Boolean +		 +

Whether or not a user can update the description for the entity

+
+canEditLinks
+Boolean +		 +

Whether or not a user can update the links for the entity

+
+canEditDomains
+Boolean +		 +

Whether or not a user can update the domain(s) for the entity

+
+canEditDataProducts
+Boolean +		 +

Whether or not a user can update the data product(s) that the entity belongs to

+
+canEditOwners
+Boolean +		 +

Whether or not a user can update the owners for the entity

+
+canEditIncidents
+Boolean +		 +

Whether or not a user can update the incidents for an asset

+
+canEditAssertions
+Boolean +		 +

Whether or not a user can update assertions for an asset

+
+canEditDeprecation
+Boolean +		 +

Whether or not a user can update the deprecation status for an entity

+
+canEditSchemaFieldTags
+Boolean +		 +

Whether or not a user can update the schema field tags for a dataset

+
+canEditSchemaFieldGlossaryTerms
+Boolean +		 +

Whether or not a user can update the schema field tags for a dataset

+
+canEditSchemaFieldDescription
+Boolean +		 +

Whether or not a user can update the schema field tags for a dataset

+
+canViewDatasetUsage
+Boolean +		 +

Whether the user can view dataset usage stats

+
+canViewDatasetProfile
+Boolean +		 +

Whether the user can view dataset profiling stats

+
+canViewDatasetOperations
+Boolean +		 +

Whether the user can view dataset operations

+
+canManageAssetSummary
+Boolean +		 +

Whether the user can manage asset summary

+
+ +## EntityProfileConfig + +Configuration for an entity profile + +

Fields

+ + + + + + + + + +
NameDescription
+defaultTab
+String +		 +

The enum value from EntityProfileTab for which tab should be showed by default on +entity profile pages. If null, rely on default sorting from React code.

+
+ +## EntityProfileParams + +Context to define the entity profile page + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the entity being shown

+
+type
+EntityType! +		 +

Type of the enity being displayed

+
+ +## EntityProfilesConfig + +Configuration for different entity profiles + +

Fields

+ + + + + + + + + +
NameDescription
+domain
+EntityProfileConfig +		 +

The configurations for a Domain entity profile

+
+ +## EntityRelationship + +A relationship between two entities TODO Migrate all entity relationships to this more generic model + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+String! +		 +

The type of the relationship

+
+direction
+RelationshipDirection! +		 +

The direction of the relationship relative to the source entity

+
+entity
+Entity +		 +

Entity that is related via lineage

+
+created
+AuditStamp +		 +

An AuditStamp corresponding to the last modification of this relationship

+
+ +## EntityRelationshipLegacy + +Deprecated, use relationships query instead + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+entity
+EntityWithRelationships +		 +

Entity that is related via lineage

+
+created
+AuditStamp +		 +

An AuditStamp corresponding to the last modification of this relationship

+
+ +## EntityRelationshipsResult + +A list of relationship information associated with a source Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

Start offset of the result set

+
+count
+Int +		 +

Number of results in the returned result set

+
+total
+Int +		 +

Total number of results in the result set

+
+relationships
+[EntityRelationship!]! +		 +

Relationships in the result set

+
+ +## EntityTypeEntity + +An entity type registered in DataHub + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the Query

+
+type
+EntityType! +		 +

A standard Entity Type

+
+info
+EntityTypeInfo! +		 +

Info about this type including its name

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## EntityTypeInfo + +Properties about an individual entity type + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+EntityType! +		 +

The standard entity type

+
+qualifiedName
+String! +		 +

The fully qualified name of the entity type. This includes its namespace

+
+displayName
+String +		 +

The display name of this type

+
+description
+String +		 +

The description of this type

+
+ +## ERModelRelationship + +An ERModelRelationship is a high-level abstraction that dictates what datasets fields are erModelRelationshiped. + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the role

+
+type
+EntityType! +		 +

The standard Entity Type

+
+id
+String! +		 +

Unique id for the erModelRelationship

+
+properties
+ERModelRelationshipProperties +		 +

An additional set of read only properties

+
+editableProperties
+ERModelRelationshipEditableProperties +		 +

An additional set of of read write properties

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dataset

+
+ownership
+Ownership +		 +

Ownership metadata of the dataset

+
+status
+Status +		 +

Status of the Dataset

+
+tags
+GlobalTags +		 +

Tags used for searching dataset

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dataset

+
+relationships
+EntityRelationshipsResult +		 +

List of relationships between the source Entity and some destination entities with a given types

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+lineage
+EntityLineageResult +		 +

No-op required for the model

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+ +## ERModelRelationshipEditableProperties + +Additional properties about a ERModelRelationship + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+description
+String +		 +

Documentation of the ERModelRelationship

+
+name
+String +		 +

Display name of the ERModelRelationship

+
+ +## ERModelRelationshipProperties + +Additional properties about a ERModelRelationship + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the ERModelRelationship used in display

+
+source
+Dataset! +		 +

The urn of source

+
+destination
+Dataset! +		 +

The urn of destination

+
+relationshipFieldMappings
+[RelationshipFieldMapping!] +		 +

The relationFieldMappings

+
+cardinality
+ERModelRelationshipCardinality! +		 +

Cardinality of the ERModelRelationship

+
+createdTime
+Long +		 +

Created timestamp millis associated with the ERModelRelationship

+
+createdActor
+Entity +		 +

Created actor urn associated with the ERModelRelationship

+
+ +## EthicalConsiderations + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+data
+[String!] +		 +

Does the model use any sensitive data eg, protected classes

+
+humanLife
+[String!] +		 +

Is the model intended to inform decisions about matters central to human life or flourishing eg, health or safety

+
+mitigations
+[String!] +		 +

What risk mitigation strategies were used during model development

+
+risksAndHarms
+[String!] +		 +

What risks may be present in model usage +Try to identify the potential recipients, likelihood, and magnitude of harms +If these cannot be determined, note that they were considered but remain unknown

+
+useCases
+[String!] +		 +

Are there any known model use cases that are especially fraught +This may connect directly to the intended use section

+
+ +## ExecutionRequest + +Retrieve an ingestion execution request + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Urn of the execution request

+
+type
+EntityType! +		 +

The standard Entity Type

+
+id
+String! +		 +

Unique id for the execution request

+
+input
+ExecutionRequestInput! +		 +

Input provided when creating the Execution Request

+
+result
+ExecutionRequestResult +		 +

Result of the execution request

+
+source
+IngestionSource +		 +

The ingestion source of this execution request

+
+relationships
+EntityRelationshipsResult +		 +

Unused for execution requests

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## ExecutionRequestInput + +Input provided when creating an Execution Request + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+task
+String! +		 +

The type of the task to executed

+
+source
+ExecutionRequestSource! +		 +

The source of the execution request

+
+arguments
+[StringMapEntry!] +		 +

Arguments provided when creating the execution request

+
+requestedAt
+Long! +		 +

The time at which the request was created

+
+actorUrn
+String +		 +
Deprecated: Use actor instead
+ +

Urn of the actor who created this execution request

+
+actor
+CorpUser +		 +

The actor who created this execution request

+
+executorId
+String +		 +

The specific executor to route the request to. If none is provided, a "default" executor is used.

+
+ +## ExecutionRequestResult + +The result of an ExecutionRequest + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+status
+String! +		 +

The result of the request, e.g. either SUCCEEDED or FAILED

+
+startTimeMs
+Long +		 +

Time at which the task began

+
+durationMs
+Long +		 +

Duration of the task

+
+report
+String +		 +

A report about the ingestion run

+
+structuredReport
+StructuredReport +		 +

A structured report for this Execution Request

+
+ +## ExecutionRequestSource + +Information about the source of an execution request + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+String +		 +

The type of the source, e.g. SCHEDULED_INGESTION_SOURCE

+
+ingestionSource
+String +		 +

The urn of the ingestion source, if applicable

+
+ +## ExtraProperty + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Name of the extra property

+
+value
+String! +		 +

Value of the extra property

+
+ +## FacetFilter + +A single filter value + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+field
+String! +		 +

Name of field to filter by

+
+condition
+FilterOperator +		 +

Condition for the values.

+
+values
+[String!]! +		 +

Values, one of which the intended field should match.

+
+negated
+Boolean +		 +

If the filter should or should not be matched

+
+ +## FacetMetadata + +Contains valid fields to filter search results further on + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+field
+String! +		 +

Name of a field present in the search entity

+
+displayName
+String +		 +

Display name of the field

+
+entity
+Entity +		 +

Entity corresponding to the facet

+
+aggregations
+[AggregationMetadata!]! +		 +

Aggregated search result counts by value of the field

+
+ +## FeatureFlagsConfig + +Configurations related to DataHub Views feature + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+readOnlyModeEnabled
+Boolean! +		 +

Whether read only mode is enabled on an instance. +Right now this only affects ability to edit user profile image URL but can be extended.

+
+showSearchFiltersV2
+Boolean! +		 +

Whether search filters V2 should be shown or the default filter side-panel

+
+showBrowseV2
+Boolean! +		 +

Whether browse V2 sidebar should be shown

+
+platformBrowseV2
+Boolean! +		 +

Whether browse v2 is platform mode, which means that platforms are displayed instead of entity types at the root.

+
+lineageGraphV2
+Boolean! +		 +

Whether to show the new lineage visualization.

+
+showAcrylInfo
+Boolean! +		 +

Whether we should show CTAs in the UI related to moving to DataHub Cloud by DataHub.

+
+erModelRelationshipFeatureEnabled
+Boolean! +		 +

Whether ERModelRelationship Tables Feature should be shown.

+
+showAccessManagement
+Boolean! +		 +

Whether we should show AccessManagement tab in the datahub UI.

+
+nestedDomainsEnabled
+Boolean! +		 +

Enables the nested Domains feature that allows users to have sub-Domains. +If this is off, Domains appear "flat" again.

+
+businessAttributeEntityEnabled
+Boolean! +		 +

Whether business attribute entity should be shown

+
+dataContractsEnabled
+Boolean! +		 +

Whether data contracts should be enabled

+
+editableDatasetNameEnabled
+Boolean! +		 +

Whether dataset names are editable

+
+themeV2Enabled
+Boolean! +		 +

Allows the V2 theme to be turned on. +Includes new UX for home page, search, entity profiles, and lineage. +If false, then the V2 experience will be unavailable even if themeV2Default or themeV2Toggleable are true.

+
+themeV2Default
+Boolean! +		 +

Sets the default theme to V2. +If themeV2Toggleable is set, then users can toggle between V1 and V2. +If not, then the default is the only option.

+
+themeV2Toggleable
+Boolean! +		 +

Allows the V2 theme to be toggled by users.

+
+schemaFieldCLLEnabled
+Boolean! +		 +

Enables links to schema field-level lineage on lineage explorer.

+
+showSeparateSiblings
+Boolean! +		 +

If turned on, all siblings will be separated with no way to get to a "combined" sibling view

+
+showManageStructuredProperties
+Boolean! +		 +

If turned on, show the manage structured properties tab in the govern dropdown

+
+hideDbtSourceInLineage
+Boolean! +		 +

If turned on, hides DBT Sources from lineage by: +i) Hiding the source in the lineage graph, if it has no downstreams +ii) Swapping to the source's sibling urn on V2 lineage graph +iii) Representing source sibling as a merged node, with both icons on graph and combined version in sidebar

+
+schemaFieldLineageIgnoreStatus
+Boolean! +		 +

If turned on, schema field lineage will always fetch ghost entities and present them as real

+
+showNavBarRedesign
+Boolean! +		 +

If turned on, show the newly designed nav bar in the V2 experience

+
+showAutoCompleteResults
+Boolean! +		 +

If turned on, we display auto complete results. Otherwise, do not.

+
+entityVersioningEnabled
+Boolean! +		 +

If turned on, exposes the versioning feature by allowing users to link entities in the UI.

+
+showHasSiblingsFilter
+Boolean! +		 +

If turned on, show the "has siblings" filter in search

+
+showSearchBarAutocompleteRedesign
+Boolean! +		 +

If turned on, show the redesigned search bar's autocomplete

+
+showManageTags
+Boolean! +		 +

If enabled, users will be able to view the tags management experience

+
+showIntroducePage
+Boolean! +		 +

If enabled, we will show the introduce page in the V2 UI experience to add a title and select platforms

+
+showIngestionPageRedesign
+Boolean! +		 +

If turned on, show the re-designed Ingestions page

+
+showLineageExpandMore
+Boolean! +		 +

If enabled, show the expand more button (>>) in the lineage graph

+
+showStatsTabRedesign
+Boolean! +		 +

If turned on, show the re-designed Stats tab on the entity page

+
+showHomePageRedesign
+Boolean! +		 +

If turned on, show the re-designed home page

+
+showProductUpdates
+Boolean! +		 +

Whether product updates on the sidebar is enabled. Will go to oss.

+
+lineageGraphV3
+Boolean! +		 +

Enables the redesign of the lineage v2 graph

+
+logicalModelsEnabled
+Boolean! +		 +

Enables logical models feature

+
+showHomepageUserRole
+Boolean! +		 +

Enables displaying the homepage user role underneath the name. Only available for custom home page.

+
+showDefaultExternalLinks
+Boolean! +		 +

If enabled, show the default external links on the entity page

+
+assetSummaryPageV1
+Boolean! +		 +

Enables displaying the asset summary page

+
+datasetSummaryPageV1
+Boolean! +		 +

Enables displaying the dataset summary page

+
+ingestionOnboardingRedesignV1
+Boolean! +		 +

Enables displaying the ingestion onboarding redesign

+
+documentationFileUploadV1
+Boolean! +		 +

If enabled, allows uploading of files for documentation.

+
+multipleDataProductsPerAsset
+Boolean! +		 +

If enabled, allows assets to belong to multiple data products simultaneously.

+
+glossaryBasedPoliciesEnabled
+Boolean! +		 +

If enabled, allows glossary-based policies to be created.

+
+contextDocumentsEnabled
+Boolean! +		 +

If enabled, shows the context documents feature in the sidebar.

+
+hideLineageInSearchCards
+Boolean! +		 +

If enabled, hides lineage information in search result cards

+
+ +## FieldAssertionInfo + +A definition of a Field (Column) assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+FieldAssertionType! +		 +

The type of the field assertion being monitored.

+
+entityUrn
+String! +		 +

The entity targeted by this Field check.

+
+fieldValuesAssertion
+FieldValuesAssertion +		 +

The definition of an assertion that validates individual values of a field / column for a set of rows.

+
+fieldMetricAssertion
+FieldMetricAssertion +		 +

The definition of an assertion that validates a common metric obtained about a field / column for a set of rows.

+
+filter
+DatasetFilter +		 +

A definition of the specific filters that should be applied, when performing monitoring. +If not provided, there is no filter, and the full table is under consideration.

+
+ +## FieldFormPromptAssociation + +An association for field-level form prompts + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+fieldPath
+String! +		 +

The schema field path

+
+lastModified
+ResolvedAuditStamp! +		 +

When and by whom this form field-level prompt has last been modified

+
+ +## FieldMetricAssertion + +A definition of a Field Metric assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+field
+SchemaFieldSpec! +		 +

The field under evaluation

+
+metric
+FieldMetricType! +		 +

The specific metric to assert against.

+
+operator
+AssertionStdOperator! +		 +

The predicate to evaluate against the metric for the field / column.

+
+parameters
+AssertionStdParameters +		 +

Standard parameters required for the assertion.

+
+ +## FieldTransform + +Definition of a transform applied to the values of a column / field. + +

Fields

+ + + + + + + + + +
NameDescription
+type
+FieldTransformType! +		 +

The type of the field transform.

+
+ +## FieldUsageCounts + +The usage for a particular Dataset field + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+fieldName
+String +		 +

The path of the field

+
+count
+Int +		 +

The count of usages

+
+ +## FieldValuesAssertion + +A definition of a Field Values assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+field
+SchemaFieldSpec! +		 +

The field under evaluation.

+
+transform
+FieldTransform +		 +

An optional transform to apply to field values before evaluating the operator.

+
+operator
+AssertionStdOperator! +		 +

The predicate to evaluate against a single value of the field. +Depending on the operator, parameters may be required

+
+parameters
+AssertionStdParameters +		 +

Standard parameters required for the assertion.

+
+failThreshold
+FieldValuesFailThreshold! +		 +

Additional customization about when the assertion should be officially considered failing.

+
+excludeNulls
+Boolean! +		 +

Whether to ignore or allow nulls when running the values assertion.

+
+ +## FieldValuesFailThreshold + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+FieldValuesFailThresholdType! +		 +

The type of failure threshold.

+
+value
+Long! +		 +

The value of the threshold, either representing a count or percentage.

+
+ +## FineGrainedLineage + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+upstreams
+[SchemaFieldRef!] +		 + +
+downstreams
+[SchemaFieldRef!] +		 + +
+query
+String +		 + +
+transformOperation
+String +		 + +
+ +## FixedIntervalSchedule + +A fixed interval schedule. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+unit
+DateInterval! +		 +

Interval unit such as minute/hour/day etc.

+
+multiple
+Int! +		 +

How many units. Defaults to 1.

+
+ +## FloatBox + + + +

Fields

+ + + + + + + + + +
NameDescription
+floatValue
+Float! +		 + +
+ +## ForeignKeyConstraint + +Metadata around a foreign key constraint between two datasets + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

The human-readable name of the constraint

+
+foreignFields
+[SchemaFieldEntity] +		 +

List of fields in the foreign dataset

+
+sourceFields
+[SchemaFieldEntity] +		 +

List of fields in this dataset

+
+foreignDataset
+Dataset +		 +

The foreign dataset for easy reference

+
+ +## Form + +A form that helps with filling out metadata on an entity + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the Form

+
+type
+EntityType! +		 +

A standard Entity Type

+
+info
+FormInfo! +		 +

Information about this form

+
+ownership
+Ownership +		 +

Ownership metadata of the form

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## FormActorAssignment + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+owners
+Boolean! +		 +

Whether the form should be completed by owners of the assets which the form is applied to.

+
+users
+[CorpUser!] +		 +

Urns of the users that the form is assigned to. If null, then no users are specifically targeted.

+
+groups
+[CorpGroup!] +		 +

Groups that the form is assigned to. If null, then no groups are specifically targeted.

+
+isAssignedToMe
+Boolean! +		 +

Whether or not the current actor is universally assigned to this form, either by user or by group. +Note that this does not take into account entity ownership based assignment.

+
+ +## FormAssociation + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+form
+Form! +		 +

The form related to the associated urn

+
+associatedUrn
+String! +		 +

Reference back to the urn with the form on it for tracking purposes e.g. when sibling nodes are merged together

+
+incompletePrompts
+[FormPromptAssociation!] +		 +

The prompt that still need to be completed for this form

+
+completedPrompts
+[FormPromptAssociation!] +		 +

The prompt that are already completed for this form

+
+ +## FormInfo + +Properties about an individual Form + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of this form

+
+description
+String +		 +

The description of this form

+
+type
+FormType! +		 +

The type of this form

+
+prompts
+[FormPrompt!]! +		 +

The prompt for this form

+
+actors
+FormActorAssignment! +		 +

The actors that are assigned to complete the forms for the associated entities.

+
+ +## FormPrompt + +A prompt shown to the user to collect metadata about an entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String! +		 +

The ID of this prompt. This will be globally unique.

+
+title
+String! +		 +

The title of this prompt

+
+formUrn
+String! +		 +

The urn of the parent form that this prompt is part of

+
+description
+String +		 +

The description of this prompt

+
+type
+FormPromptType! +		 +

The description of this prompt

+
+required
+Boolean! +		 +

Whether the prompt is required for the form to be considered completed.

+
+structuredPropertyParams
+StructuredPropertyParams +		 +

The params for this prompt if type is STRUCTURED_PROPERTY

+
+ +## FormPromptAssociation + +A form that helps with filling out metadata on an entity + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+id
+String! +		 +

The unique id of the form prompt

+
+lastModified
+ResolvedAuditStamp! +		 +

When and by whom this form prompt has last been modified

+
+fieldAssociations
+FormPromptFieldAssociations +		 +

Optional information about the field-level prompt associations.

+
+ +## FormPromptFieldAssociations + +Information about the field-level prompt associations. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+completedFieldPrompts
+[FieldFormPromptAssociation!] +		 +

If this form prompt is for fields, this will contain a list of completed associations per field

+
+incompleteFieldPrompts
+[FieldFormPromptAssociation!] +		 +

If this form prompt is for fields, this will contain a list of incomlete associations per field

+
+ +## Forms + +Requirements forms that are assigned to an entity. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+incompleteForms
+[FormAssociation!]! +		 +

Forms that are still incomplete.

+
+completedForms
+[FormAssociation!]! +		 +

Forms that have been completed.

+
+verifications
+[FormVerificationAssociation!]! +		 +

Verifications that have been applied to the entity via completed forms.

+
+ +## FormVerificationAssociation + +Verification object that has been applied to the entity via a completed form. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+form
+Form! +		 +

The form related to the associated urn

+
+lastModified
+ResolvedAuditStamp +		 +

When this verification was applied to this entity

+
+ +## FreshnessAssertionInfo + +Information about an Freshness assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entityUrn
+String! +		 +

The urn of the entity that the Freshness assertion is related to

+
+type
+FreshnessAssertionType! +		 +

The type of the Freshness Assertion

+
+schedule
+FreshnessAssertionSchedule! +		 +

Produce FAIL Assertion Result if the asset is not updated on the cadence and within the time range described by the schedule.

+
+filter
+DatasetFilter +		 +

A filter applied when querying an external Dataset or Table

+
+ +## FreshnessAssertionSchedule + +Attributes defining a single Freshness schedule. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+FreshnessAssertionScheduleType! +		 +

The type of schedule

+
+cron
+FreshnessCronSchedule +		 +

A cron schedule. This is populated if the type is CRON.

+
+fixedInterval
+FixedIntervalSchedule +		 +

A fixed interval schedule. This is populated if the type is FIXED_INTERVAL.

+
+ +## FreshnessContract + + + +

Fields

+ + + + + + + + + +
NameDescription
+assertion
+Assertion! +		 +

The assertion representing the Freshness contract.

+
+ +## FreshnessCronSchedule + +A cron-formatted schedule + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+cron
+String! +		 +

A cron-formatted execution interval, as a cron string, e.g. 1 * * * *

+
+timezone
+String! +		 +

Timezone in which the cron interval applies, e.g. America/Los Angeles

+
+windowStartOffsetMs
+Long +		 +

An optional offset in milliseconds to SUBTRACT from the timestamp generated by the cron schedule +to generate the lower bounds of the "Freshness window", or the window of time in which an event must have occurred in order for the Freshness +to be considering passing. +If left empty, the start of the Freshness window will be the end of the previously evaluated Freshness window.

+
+ +## FreshnessStats + +Freshness stats for a query result. +Captures whether the query was served out of a cache, what the staleness was, etc. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+cached
+Boolean +		 +

Whether a cache was used to respond to this query

+
+systemFreshness
+[SystemFreshness] +		 +

The latest timestamp in millis of the system that was used to respond to this query +In case a cache was consulted, this reflects the freshness of the cache +In case an index was consulted, this reflects the freshness of the index

+
+ +## GetPresignedUploadUrlResponse + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+url
+String! +		 + +
+fileId
+String! +		 + +
+ +## GetQuickFiltersResult + +The result object when fetching quick filters + +

Fields

+ + + + + + + + + +
NameDescription
+quickFilters
+[QuickFilter]! +		 +

The list of quick filters to render in the UI

+
+ +## GetRootGlossaryNodesResult + +The result when getting Glossary entities + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+nodes
+[GlossaryNode!]! +		 +

A list of Glossary Nodes without a parent node

+
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of nodes in the returned result

+
+total
+Int! +		 +

The total number of nodes in the result set

+
+ +## GetRootGlossaryTermsResult + +The result when getting root GlossaryTerms + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+terms
+[GlossaryTerm!]! +		 +

A list of Glossary Terms without a parent node

+
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of terms in the returned result

+
+total
+Int! +		 +

The total number of terms in the result set

+
+ +## GetSchemaBlameResult + +Schema changes computed at a specific version. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+version
+SemanticVersionStruct +		 +

Selected semantic version

+
+schemaFieldBlameList
+[SchemaFieldBlame!] +		 +

List of schema blame. Absent when there are no fields to return history for.

+
+ +## GetSchemaVersionListResult + +Schema changes computed at a specific version. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+latestVersion
+SemanticVersionStruct +		 +

Latest and current semantic version

+
+version
+SemanticVersionStruct +		 +

Selected semantic version

+
+semanticVersionList
+[SemanticVersionStruct!] +		 +

All semantic versions. Absent when there are no versions.

+
+ +## GetTimelineResult + +Result of getting timeline from a specific version. + +

Fields

+ + + + + + + + + +
NameDescription
+changeTransactions
+[ChangeTransaction!]! +		 + +
+ +## GlobalHomePageSettings + +Global settings related to the home page for an instance + +

Fields

+ + + + + + + + + +
NameDescription
+defaultTemplate
+DataHubPageTemplate +		 +

The default page template for the home page for this instance

+
+ +## GlobalTags + +Tags attached to a particular Metadata Entity + +

Fields

+ + + + + + + + + +
NameDescription
+tags
+[TagAssociation!] +		 +

The set of tags attached to the Metadata Entity

+
+ +## GlobalViewsSettings + +Global (platform-level) settings related to the Views feature + +

Fields

+ + + + + + + + + +
NameDescription
+defaultView
+String +		 +

The global default View. If a user does not have a personal default, then +this will be the default view.

+
+ +## GlossaryNode + +A Glossary Node, or a directory in a Business Glossary represents a container of +Glossary Terms or other Glossary Nodes + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the glossary term

+
+ownership
+Ownership +		 +

Ownership metadata of the glossary term

+
+type
+EntityType! +		 +

A standard Entity Type

+
+properties
+GlossaryNodeProperties +		 +

Additional properties associated with the Glossary Term

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+parentNodes
+ParentNodesResult +		 +

Recursively get the lineage of glossary nodes for this entity

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the Glossary Node

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+displayProperties
+DisplayProperties +		 +

Display properties for the glossary node

+
+childrenCount
+GlossaryNodeChildrenCount +		 +

Carries information about where an entity originated from.

+
+glossaryChildrenSearch
+ScrollResults +		 +

Executes a search on the children of this glossary node

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ScrollAcrossEntitiesInput! +		 + +
+ +
+settings
+AssetSettings +		 +

Settings associated with this asset

+
+ +## GlossaryNodeChildrenCount + +All of the parent nodes for GlossaryTerms and GlossaryNodes + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+termsCount
+Int! +		 +

The number of child glossary terms

+
+nodesCount
+Int! +		 +

The number of child glossary nodes

+
+ +## GlossaryNodeProperties + +Additional read only properties about a Glossary Node + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the Glossary Term

+
+description
+String +		 +

Description of the glossary term

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the Glossary Node

+
+createdOn
+ResolvedAuditStamp +		 +

A Resolved Audit Stamp corresponding to the creation of this resource

+
+ +## GlossaryTerm + +A Glossary Term, or a node in a Business Glossary representing a standardized domain +data type + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the glossary term

+
+ownership
+Ownership +		 +

Ownership metadata of the glossary term

+
+domain
+DomainAssociation +		 +

The Domain associated with the glossary term

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the glossary term

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the glossary term

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the Glossary Term

+
+type
+EntityType! +		 +

A standard Entity Type

+
+name
+String! +		 +
Deprecated: No longer supported
+ +

A unique identifier for the Glossary Term. Deprecated - Use properties.name field instead.

+
+hierarchicalName
+String! +		 +

hierarchicalName of glossary term

+
+properties
+GlossaryTermProperties +		 +

Additional properties associated with the Glossary Term

+
+glossaryTermInfo
+GlossaryTermInfo +		 +

Deprecated, use properties field instead +Details of the Glossary Term

+
+deprecation
+Deprecation +		 +

The deprecation status of the Glossary Term

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+schemaMetadata
+SchemaMetadata +		 +

Schema metadata of the dataset

+ +

Arguments

+ + + + + + + + + +
NameDescription
+version
+Long +		 + +
+ +
+parentNodes
+ParentNodesResult +		 +

Recursively get the lineage of glossary nodes for this entity

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+settings
+AssetSettings +		 +

Settings associated with this asset

+
+ +## GlossaryTermAssociation + +An edge between a Metadata Entity and a Glossary Term Modeled as a struct to permit +additional attributes +TODO Consider whether this query should be serviced by the relationships field + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+term
+GlossaryTerm! +		 +

The glossary term itself

+
+actor
+CorpUser +		 +

The actor who is responsible for the term being added"

+
+associatedUrn
+String! +		 +

Reference back to the associated urn for tracking purposes e.g. when sibling nodes are merged together

+
+context
+String +		 +

The context of how/why this term is associated

+
+attribution
+MetadataAttribution +		 +

Information about who, why, and how this metadata was applied

+
+ +## GlossaryTermInfo + +Deprecated, use GlossaryTermProperties instead +Information about a glossary term + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

The name of the Glossary Term

+
+description
+String +		 +

Description of the glossary term

+
+definition
+String! +		 +
Deprecated: No longer supported
+ +

Definition of the glossary term. Deprecated - Use 'description' instead.

+
+termSource
+String! +		 +

Term Source of the glossary term

+
+sourceRef
+String +		 +

Source Ref of the glossary term

+
+sourceUrl
+String +		 +

Source Url of the glossary term

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Properties of the glossary term

+
+rawSchema
+String +		 +

Schema definition of glossary term

+
+ +## GlossaryTermProperties + +Additional read only properties about a Glossary Term + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the Glossary Term

+
+description
+String +		 +

Description of the glossary term

+
+definition
+String! +		 +
Deprecated: No longer supported
+ +

Definition of the glossary term. Deprecated - Use 'description' instead.

+
+termSource
+String! +		 +

Term Source of the glossary term

+
+sourceRef
+String +		 +

Source Ref of the glossary term

+
+sourceUrl
+String +		 +

Source Url of the glossary term

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Properties of the glossary term

+
+rawSchema
+String +		 +

Schema definition of glossary term

+
+createdOn
+ResolvedAuditStamp +		 +

A Resolved Audit Stamp corresponding to the creation of this resource

+
+ +## GlossaryTerms + +Glossary Terms attached to a particular Metadata Entity + +

Fields

+ + + + + + + + + +
NameDescription
+terms
+[GlossaryTermAssociation!] +		 +

The set of glossary terms attached to the Metadata Entity

+
+ +## Health + +The resolved Health of an Asset + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+HealthStatusType! +		 +

An enum representing the type of health indicator

+
+reportedAt
+Long +		 +

The timestamp when the health was reported

+
+status
+HealthStatus! +		 +

An enum representing the resolved Health status of an Asset

+
+message
+String +		 +

An optional message describing the resolved health status

+
+activeIncidentHealthDetails
+ActiveIncidentHealthDetails +		 +

If type=INCIDENTS and status=FAIL, populate the details of the latest incident.

+
+latestAssertionStatusByType
+[AssertionHealthStatusByType!] +		 +

If type=ASSERTIONS, populate a breakdown of the assertion statuses by type.

+
+causes
+[String!] +		 +

NOTE: @deprecated +The causes responsible for the health status +I.e., the assertion urns that are failing

+
+ +## HierarchyViewModuleParams + +The params required if the module is type HIERARCHY_VIEW + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+assetUrns
+[String!]! +		 +

The list of assets to show in the module

+
+showRelatedEntities
+Boolean! +		 +

Whether to show related entities in the module

+
+relatedEntitiesFilterJson
+String +		 +

Optional filters to filter relatedEntities (assetUrns) out

+

The stringified json representing the logical predicate built in the UI to select assets. +This predicate is turned into orFilters to send through graphql since graphql doesn't support +arbitrary nesting. This string is used to restore the UI for this logical predicate.

+
+ +## Highlight + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+value
+Int! +		 + +
+title
+String! +		 + +
+body
+String! +		 + +
+ +## HomePageConfig + +Configurations related to the Search bar + +

Fields

+ + + + + + + + + +
NameDescription
+firstInPersonalSidebar
+PersonalSidebarSection! +		 +

The section that comes first on the personal sidebar on the homepage

+
+ +## HyperParameterMap + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+key
+String! +		 + +
+value
+HyperParameterValueType! +		 + +
+ +## IconProperties + +Properties describing an icon associated with an entity + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+iconLibrary
+IconLibrary +		 +

The source of the icon: e.g. Antd, Material, etc

+
+name
+String +		 +

The name of the icon

+
+style
+String +		 +

Any modifier for the icon, this will be library-specific, e.g. filled/outlined, etc

+
+ +## IdentityManagementConfig + +Configurations related to Identity Management + +

Fields

+ + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether identity management screen is able to be shown in the UI

+
+ +## Incident + +An incident represents an active issue on a data asset. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Incident

+
+type
+EntityType! +		 +

The standard Entity Type

+
+incidentType
+IncidentType! +		 +

The type of incident

+
+customType
+String +		 +

A custom type of incident. Present only if type is 'CUSTOM'

+
+title
+String +		 +

An optional title associated with the incident

+
+description
+String +		 +

An optional description associated with the incident

+
+incidentStatus
+IncidentStatus! +		 +

The status of an incident

+
+status
+IncidentStatus! +		 +

The status of an incident +@deprecated, use incidentStatus instead

+
+priority
+IncidentPriority +		 +

Optional priority of the incident.

+
+assignees
+[OwnerType!] +		 +

The users or groups are assigned to resolve the incident

+
+entity
+Entity! +		 +

The entity that the incident is associated with.

+
+source
+IncidentSource +		 +

The source of the incident, i.e. how it was generated

+
+startedAt
+Long +		 +

An optional time at which the incident actually started (may be before the date it was raised).

+
+created
+AuditStamp! +		 +

The time at which the incident was initially created

+
+tags
+GlobalTags +		 +

The standard tags for the Incident

+
+relationships
+EntityRelationshipsResult +		 +

List of relationships between the source Entity and some destination entities with a given types

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## IncidentSource + +Details about the source of an incident, e.g. how it was created. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+IncidentSourceType! +		 +

The type of the incident source

+
+source
+Entity +		 +

The source of the incident. If the source type is ASSERTION_FAILURE, this will have the assertion that generated the incident.

+
+ +## IncidentStatus + +Details about the status of an asset incident + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+state
+IncidentState! +		 +

The state of the incident

+
+stage
+IncidentStage +		 +

The lifecycle stage of the incident. Null means that no stage has been assigned.

+
+message
+String +		 +

An optional message associated with the status

+
+lastUpdated
+AuditStamp! +		 +

The time that the status last changed

+
+ +## IncrementingSegmentFieldTransformer + +The definition of the transformer function that should be applied to a given field / column value in a dataset +in order to determine the segment or bucket that it belongs to, which in turn is used to evaluate +volume assertions. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+IncrementingSegmentFieldTransformerType! +		 +

The 'standard' operator type. Note that not all source systems will support all operators.

+
+nativeType
+String +		 +

The 'native' transformer type, useful as a back door if a custom transformer is required. +This field is required if the type is NATIVE.

+
+ +## IncrementingSegmentRowCountChange + +Attributes defining an INCREMENTING_SEGMENT_ROW_COUNT_CHANGE volume assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+segment
+IncrementingSegmentSpec! +		 +

A specification of how the 'segment' can be derived using a column and an optional transformer function.

+
+type
+AssertionValueChangeType! +		 +

The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage.

+
+operator
+AssertionStdOperator! +		 +

The operator you'd like to apply to the row count value +Note that only numeric operators are valid inputs: +GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, +BETWEEN.

+
+parameters
+AssertionStdParameters! +		 +

The parameters you'd like to provide as input to the operator. +Note that only numeric parameter types are valid inputs: NUMBER.

+
+ +## IncrementingSegmentRowCountTotal + +Attributes defining an INCREMENTING_SEGMENT_ROW_COUNT_TOTAL volume assertion. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+segment
+IncrementingSegmentSpec! +		 +

A specification of how the 'segment' can be derived using a column and an optional transformer function.

+
+operator
+AssertionStdOperator! +		 +

The operator you'd like to apply. +Note that only numeric operators are valid inputs: +GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, +BETWEEN.

+
+parameters
+AssertionStdParameters! +		 +

The parameters you'd like to provide as input to the operator. +Note that only numeric parameter types are valid inputs: NUMBER.

+
+ +## IncrementingSegmentSpec + +Core attributes required to identify an incrementing segment in a table. This type is mainly useful +for tables that constantly increase with new rows being added on a particular cadence (e.g. fact or event tables). + +An incrementing segment represents a logical chunk of data which is INSERTED +into a dataset on a regular interval, along with the presence of a constantly-incrementing column +value such as an event time, date partition, or last modified column. + +An incrementing segment is principally identified by 2 key attributes combined: + +1. A field or column that represents the incrementing value. New rows that are inserted will be identified using this column. + Note that the value of this column may not by itself represent the "bucket" or the "segment" in which the row falls. + +2. [Optional] An transformer function that may be applied to the selected column value in order + to obtain the final "segment identifier" or "bucket identifier". Rows that have the same value after applying the transformation + will be grouped into the same segment, using which the final value (e.g. row count) will be determined. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+field
+SchemaFieldSpec! +		 +

The field to use to generate segments. It must be constantly incrementing as new rows are inserted.

+
+transformer
+IncrementingSegmentFieldTransformer +		 +

Optional transformer function to apply to the field in order to obtain the final segment or bucket identifier. +If not provided, then no operator will be applied to the field. (identity function)

+
+ +## IngestionConfig + +A set of configurations for an Ingestion Source + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+recipe
+String! +		 +

The JSON-encoded recipe to use for ingestion

+
+executorId
+String! +		 +

Advanced: The specific executor that should handle the execution request. Defaults to 'default'.

+
+version
+String +		 +

Advanced: The version of the ingestion framework to use

+
+debugMode
+Boolean +		 +

Advanced: Whether or not to run ingestion in debug mode

+
+extraArgs
+[StringMapEntry!] +		 +

Advanced: Extra arguments for the ingestion run.

+
+ +## IngestionRun + +The runs associated with an Ingestion Source managed by DataHub + +

Fields

+ + + + + + + + + +
NameDescription
+executionRequestUrn
+String +		 +

The urn of the execution request associated with the user

+
+ +## IngestionSchedule + +A schedule associated with an Ingestion Source + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+timezone
+String +		 +

Time Zone abbreviation (e.g. GMT, EDT). Defaults to UTC.

+
+interval
+String! +		 +

The cron-formatted interval to execute the ingestion source on

+
+ +## IngestionSource + +An Ingestion Source Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Ingestion Source

+
+type
+String! +		 +

The type of the source itself, e.g. mysql, bigquery, bigquery-usage. Should match the recipe.

+
+name
+String! +		 +

The display name of the Ingestion Source

+
+schedule
+IngestionSchedule +		 +

An optional schedule associated with the Ingestion Source

+
+platform
+DataPlatform +		 +

The data platform associated with this ingestion source

+
+config
+IngestionConfig! +		 +

An type-specific set of configurations for the ingestion source

+
+executions
+IngestionSourceExecutionRequests +		 +

Previous requests to execute the ingestion source

+ +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+start
+Int +		 + +
+count
+Int +		 + +
+ +
+ownership
+Ownership +		 +

Ownership metadata of the ingestion source

+
+latestSuccessfulExecution
+ExecutionRequest +		 +

The latest successful execution request for this source

+
+source
+IngestionSourceSource +		 +

The source or origin of the Ingestion Source

+
+ +## IngestionSourceExecutionRequests + +Requests for execution associated with an ingestion source + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+total
+Int +		 +

The total number of results in the result set

+
+executionRequests
+[ExecutionRequest!]! +		 +

The execution request objects comprising the result set

+
+ +## IngestionSourceSource + +Source information for an ingestion source + +

Fields

+ + + + + + + + + +
NameDescription
+type
+IngestionSourceSourceType! +		 +

The source type of the ingestion source

+
+ +## InputField + +Input field of the chart + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+schemaFieldUrn
+String +		 + +
+schemaField
+SchemaField +		 + +
+ +## InputFields + +Input fields of the chart + +

Fields

+ + + + + + + + + +
NameDescription
+fields
+[InputField] +		 + +
+ +## InstitutionalMemory + +Institutional memory metadata, meaning internal links and pointers related to an Entity + +

Fields

+ + + + + + + + + +
NameDescription
+elements
+[InstitutionalMemoryMetadata!]! +		 +

List of records that represent the institutional memory or internal documentation of an entity

+
+ +## InstitutionalMemoryMetadata + +An institutional memory resource about a particular Metadata Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+url
+String! +		 +

Link to a document or wiki page or another internal resource

+
+label
+String! +		 +

Label associated with the URL

+
+author
+CorpUser! +		 +
Deprecated: Use `actor`
+ +

The author of this metadata +Deprecated! Use actor instead for users or groups.

+
+actor
+ResolvedActor! +		 +

The author of this metadata

+
+created
+AuditStamp! +		 +

An AuditStamp corresponding to the creation of this resource

+
+updated
+AuditStamp +		 +

An AuditStamp corresponding to the updating of this resource

+
+description
+String! +		 +
Deprecated: No longer supported
+ +

Deprecated, use label instead +Description of the resource

+
+associatedUrn
+String! +		 +

Reference back to the owned urn for tracking purposes e.g. when sibling nodes are merged together

+
+settings
+InstitutionalMemoryMetadataSettings +		 +

Settings for this record of institutional memory

+
+ +## InstitutionalMemoryMetadataSettings + +Settings for an institutional memory record + +

Fields

+ + + + + + + + + +
NameDescription
+showInAssetPreview
+Boolean! +		 +

Show record in asset preview like on entity header and search previews

+
+ +## IntBox + + + +

Fields

+ + + + + + + + + +
NameDescription
+intValue
+Int! +		 + +
+ +## IntendedUse + + + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+primaryUses
+[String!] +		 +

Primary Use cases for the model

+
+primaryUsers
+[IntendedUserType!] +		 +

Primary Intended Users

+
+outOfScopeUses
+[String!] +		 +

Out of scope uses of the MLModel

+
+ +## IntMapEntry + +An entry in a integer string map represented as a tuple + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+key
+String! +		 +

The key of the map entry

+
+value
+Int +		 +

The value for the map entry

+
+ +## InviteToken + +Token that allows users to sign up as a native user + +

Fields

+ + + + + + + + + +
NameDescription
+inviteToken
+String! +		 +

The invite token

+
+ +## KeyValueSchema + +Information about a raw Key Value Schema + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+keySchema
+String! +		 +

Raw key schema

+
+valueSchema
+String! +		 +

Raw value schema

+
+ +## LineageConfig + +Configurations related to Lineage + +

Fields

+ + + + + + + + + +
NameDescription
+supportsImpactAnalysis
+Boolean! +		 +

Whether the backend support impact analysis feature

+
+ +## LineageRelationship + +Metadata about a lineage relationship between two entities + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+String! +		 +

The type of the relationship

+
+entity
+Entity +		 +

Entity that is related via lineage

+
+degree
+Int! +		 +

Degree of relationship (number of hops to get to entity)

+
+createdOn
+Long +		 +

Timestamp for when this lineage relationship was created. Could be null.

+
+createdActor
+Entity +		 +

The actor who created this lineage relationship. Could be null.

+
+updatedOn
+Long +		 +

Timestamp for when this lineage relationship was last updated. Could be null.

+
+updatedActor
+Entity +		 +

The actor who last updated this lineage relationship. Could be null.

+
+isManual
+Boolean +		 +

Whether this edge is a manual edge. Could be null.

+
+paths
+[EntityPath] +		 +

The paths traversed for this relationship

+
+ +## LinkModuleParams + +The params required if the module is type LINK + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+linkUrl
+String! +		 +

The URL of the link

+
+imageUrl
+String +		 +

The image URL of the link

+
+description
+String +		 +

The description of the link

+
+ +## LinkParams + +Parameters required to specify the page to land once clicked + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+searchParams
+SearchParams +		 +

Context to define the search page

+
+entityProfileParams
+EntityProfileParams +		 +

Context to define the entity profile page

+
+ +## ListAccessTokenResult + +Results returned when listing access tokens + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set

+
+count
+Int! +		 +

The number of results to be returned

+
+total
+Int! +		 +

The total number of results in the result set

+
+tokens
+[AccessTokenMetadata!]! +		 +

The token metadata themselves

+
+ +## ListBusinessAttributesResult + +The result obtained when listing Business Attribute + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Business Attributes in the returned result set

+
+total
+Int! +		 +

The total number of Business Attributes in the result set

+
+businessAttributes
+[BusinessAttribute!]! +		 +

The Business Attributes

+
+ +## ListDomainsResult + +The result obtained when listing DataHub Domains + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Domains in the returned result set

+
+total
+Int! +		 +

The total number of Domains in the result set

+
+domains
+[Domain!]! +		 +

The Domains themselves

+
+ +## ListGroupsResult + +The result obtained when listing DataHub Groups + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Policies in the returned result set

+
+total
+Int! +		 +

The total number of Policies in the result set

+
+groups
+[CorpGroup!]! +		 +

The groups themselves

+
+ +## ListIngestionSourcesResult + +Results returned when listing ingestion sources + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set

+
+count
+Int! +		 +

The number of results to be returned

+
+total
+Int! +		 +

The total number of results in the result set

+
+ingestionSources
+[IngestionSource!]! +		 +

The Ingestion Sources themselves

+
+ +## ListOwnershipTypesResult + +Results when listing custom ownership types. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set

+
+count
+Int! +		 +

The number of results to be returned

+
+total
+Int! +		 +

The total number of results in the result set

+
+ownershipTypes
+[OwnershipTypeEntity!]! +		 +

The Custom Ownership Types themselves

+
+ +## ListPoliciesResult + +The result obtained when listing DataHub Access Policies + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Policies in the returned result set

+
+total
+Int! +		 +

The total number of Policies in the result set

+
+policies
+[Policy!]! +		 +

The Policies themselves

+
+ +## ListPostsResult + +The result obtained when listing Posts + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Roles in the returned result set

+
+total
+Int! +		 +

The total number of Roles in the result set

+
+posts
+[Post!]! +		 +

The Posts themselves

+
+ +## ListQueriesResult + +Results when listing entity queries + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set

+
+count
+Int! +		 +

The number of results to be returned

+
+total
+Int! +		 +

The total number of results in the result set

+
+queries
+[QueryEntity!]! +		 +

The Queries themselves

+
+ +## ListRecommendationsResult + +Results returned by the ListRecommendations query + +

Fields

+ + + + + + + + + +
NameDescription
+modules
+[RecommendationModule!]! +		 +

List of modules to show

+
+ +## ListRolesResult + +The result obtained when listing DataHub Roles + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Roles in the returned result set

+
+total
+Int! +		 +

The total number of Roles in the result set

+
+roles
+[DataHubRole!]! +		 +

The Roles themselves

+
+ +## ListSecretsResult + +Input for listing DataHub Secrets + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int +		 +

The starting offset of the result set

+
+count
+Int +		 +

The number of results to be returned

+
+total
+Int +		 +

The total number of results in the result set

+
+secrets
+[Secret!]! +		 +

The secrets themselves

+
+ +## ListServiceAccountsResult + +Results returned when listing service accounts + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set

+
+count
+Int! +		 +

The number of results to be returned

+
+total
+Int! +		 +

The total number of results in the result set

+
+serviceAccounts
+[ServiceAccount!]! +		 +

The service accounts

+
+ +## ListTestsResult + +The result obtained when listing DataHub Tests + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Tests in the returned result set

+
+total
+Int! +		 +

The total number of Tests in the result set

+
+tests
+[Test!]! +		 +

The Tests themselves

+
+ +## ListUsersResult + +The result obtained when listing DataHub Users + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Policies in the returned result set

+
+total
+Int! +		 +

The total number of Policies in the result set

+
+users
+[CorpUser!]! +		 +

The users themselves

+
+ +## ListViewsResult + +The result obtained when listing DataHub Views + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Views in the returned result set

+
+total
+Int! +		 +

The total number of Views in the result set

+
+views
+[DataHubView!]! +		 +

The Views themselves

+
+ +## ManagedIngestionConfig + +Configurations related to managed, UI based ingestion + +

Fields

+ + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether ingestion screen is enabled in the UI

+
+ +## MatchedField + +An overview of the field that was matched in the entity search document + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Name of the field that matched

+
+value
+String! +		 +

Value of the field that matched

+
+entity
+Entity +		 +

Entity if the value is an urn

+
+ +## Media + +Media content + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+MediaType! +		 +

The type of media

+
+location
+String! +		 +

The location of the media (a URL)

+
+ +## MetadataAttribution + +Information about who, why, and how this metadata was applied + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+time
+Long! +		 +

The time this metadata was applied

+
+actor
+Entity! +		 +

The actor responsible for this metadata application

+
+source
+Entity +		 +

The source of this metadata application. If propagated, this will be an action.

+
+sourceDetail
+[StringMapEntry!] +		 +

Extra details about how this metadata was applied

+
+ +## Metrics + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+performanceMeasures
+[String!] +		 +

Measures of ML Model performance

+
+decisionThreshold
+[String!] +		 +

Decision Thresholds used if any

+
+ +## MLFeature + +An ML Feature Metadata Entity Note that this entity is incubating + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the ML Feature

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+name
+String! +		 +

The display name for the ML Feature

+
+featureNamespace
+String! +		 +

MLFeature featureNamespace

+
+description
+String +		 +

The description about the ML Feature

+
+dataType
+MLFeatureDataType +		 +

MLFeature data type

+
+ownership
+Ownership +		 +

Ownership metadata of the MLFeature

+
+featureProperties
+MLFeatureProperties +		 +
Deprecated: No longer supported
+ +

ModelProperties metadata of the MLFeature

+
+properties
+MLFeatureProperties +		 +

ModelProperties metadata of the MLFeature

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the MLFeature

+
+status
+Status +		 +

Status metadata of the MLFeature

+
+deprecation
+Deprecation +		 +

Deprecation

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+tags
+GlobalTags +		 +

Tags applied to entity

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the entity

+
+domain
+DomainAssociation +		 +

The Domain associated with the entity

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+editableProperties
+MLFeatureEditableProperties +		 +

An additional set of of read write properties

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## MLFeatureEditableProperties + + + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

The edited description

+
+ +## MLFeatureProperties + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+description
+String +		 + +
+dataType
+MLFeatureDataType +		 + +
+version
+VersionTag +		 + +
+sources
+[Dataset] +		 + +
+customProperties
+[CustomPropertiesEntry!] +		 + +
+ +## MLFeatureTable + +An ML Feature Table Entity Note that this entity is incubating + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the ML Feature Table

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+name
+String! +		 +

The display name

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the MLFeatureTable is defined

+
+description
+String +		 +

MLFeatureTable description

+
+ownership
+Ownership +		 +

Ownership metadata of the MLFeatureTable

+
+properties
+MLFeatureTableProperties +		 +

Additional read only properties associated the the ML Feature Table

+
+featureTableProperties
+MLFeatureTableProperties +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +ModelProperties metadata of the MLFeature

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the MLFeature

+
+status
+Status +		 +

Status metadata of the MLFeatureTable

+
+deprecation
+Deprecation +		 +

Deprecation

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the ML Feature Table. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+tags
+GlobalTags +		 +

Tags applied to entity

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the entity

+
+domain
+DomainAssociation +		 +

The Domain associated with the entity

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+editableProperties
+MLFeatureTableEditableProperties +		 +

An additional set of of read write properties

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## MLFeatureTableEditableProperties + + + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

The edited description

+
+ +## MLFeatureTableProperties + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+description
+String +		 + +
+mlFeatures
+[MLFeature] +		 + +
+mlPrimaryKeys
+[MLPrimaryKey] +		 + +
+customProperties
+[CustomPropertiesEntry!] +		 + +
+ +## MLHyperParam + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 + +
+description
+String +		 + +
+value
+String +		 + +
+createdAt
+Long +		 + +
+ +## MLMetric + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

Name of the metric (e.g. accuracy, precision, recall)

+
+description
+String +		 +

Description of what this metric measures

+
+value
+String +		 +

The computed value of the metric

+
+createdAt
+Long +		 +

Timestamp when this metric was recorded

+
+ +## MLModel + +An ML Model Metadata Entity Note that this entity is incubating + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) +- [SupportsVersions](/docs/graphql/interfaces#supportsversions) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the ML model

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+name
+String! +		 +

ML model display name

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the MLModel is defined

+
+origin
+FabricType! +		 +

Fabric type where mlmodel belongs to or where it was generated

+
+description
+String +		 +

Human readable description for mlmodel

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use tags field instead +The standard tags for the ML Model

+
+tags
+GlobalTags +		 +

The standard tags for the ML Model

+
+ownership
+Ownership +		 +

Ownership metadata of the mlmodel

+
+properties
+MLModelProperties +		 +

Additional read only information about the ML Model

+
+intendedUse
+IntendedUse +		 +

Intended use of the mlmodel

+
+factorPrompts
+MLModelFactorPrompts +		 +

Factors metadata of the mlmodel

+
+metrics
+Metrics +		 +

Metrics metadata of the mlmodel

+
+evaluationData
+[BaseData!] +		 +

Evaluation Data of the mlmodel

+
+trainingData
+[BaseData!] +		 +

Training Data of the mlmodel

+
+quantitativeAnalyses
+QuantitativeAnalyses +		 +

Quantitative Analyses of the mlmodel

+
+ethicalConsiderations
+EthicalConsiderations +		 +

Ethical Considerations of the mlmodel

+
+caveatsAndRecommendations
+CaveatsAndRecommendations +		 +

Caveats and Recommendations of the mlmodel

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the mlmodel

+
+sourceCode
+SourceCode +		 +

Source Code

+
+status
+Status +		 +

Status metadata of the mlmodel

+
+cost
+Cost +		 +

Cost Aspect of the mlmodel

+
+deprecation
+Deprecation +		 +

Deprecation

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the ML Model. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the entity

+
+domain
+DomainAssociation +		 +

The Domain associated with the entity

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+editableProperties
+MLModelEditableProperties +		 +

An additional set of of read write properties

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+versionProperties
+VersionProperties +		 + +
+ +## MLModelEditableProperties + + + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

The edited description

+
+ +## MLModelFactorPrompts + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+relevantFactors
+[MLModelFactors!] +		 +

What are foreseeable salient factors for which MLModel performance may vary, and how were these determined

+
+evaluationFactors
+[MLModelFactors!] +		 +

Which factors are being reported, and why were these chosen

+
+ +## MLModelFactors + + + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+groups
+[String!] +		 +

Distinct categories with similar characteristics that are present in the evaluation data instances

+
+instrumentation
+[String!] +		 +

Instrumentation used for MLModel

+
+environment
+[String!] +		 +

Environment in which the MLModel is deployed

+
+ +## MLModelGroup + +An ML Model Group Metadata Entity +Note that this entity is incubating + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the ML Model Group

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+name
+String! +		 +

The display name for the Entity

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the MLModelGroup is defined

+
+origin
+FabricType! +		 +

Fabric type where MLModelGroup belongs to or where it was generated

+
+description
+String +		 +

Human readable description for MLModelGroup

+
+properties
+MLModelGroupProperties +		 +

Additional read only properties about the ML Model Group

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the ml model group

+
+ownership
+Ownership +		 +

Ownership metadata of the MLModelGroup

+
+status
+Status +		 +

Status metadata of the MLModelGroup

+
+deprecation
+Deprecation +		 +

Deprecation

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the ML Model Group. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+tags
+GlobalTags +		 +

Tags applied to entity

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the entity

+
+domain
+DomainAssociation +		 +

The Domain associated with the entity

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+editableProperties
+MLModelGroupEditableProperties +		 +

An additional set of of read write properties

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## MLModelGroupEditableProperties + + + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

The edited description

+
+ +## MLModelGroupProperties + +Properties describing a group of related ML models + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

Display name of the model group

+
+description
+String +		 +

Detailed description of the model group's purpose and contents

+
+created
+AuditStamp +		 +

When this model group was created

+
+lastModified
+AuditStamp +		 +

When this model group was last modified

+
+version
+VersionTag +		 +

Version identifier for this model group

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom key-value properties for the model group

+
+externalUrl
+String +		 +

URL to view this model group in the external system

+
+createdAt
+Long +		 +
Deprecated: Use `created` instead
+ +

Deprecated creation timestamp +@deprecated Use the 'created' field instead

+
+mlModelLineageInfo
+MLModelLineageInfo +		 +

Information related to lineage to this model group

+
+ +## MLModelLineageInfo + +Represents lineage information for ML entities. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+trainingJobs
+[String!] +		 +

List of jobs or processes used to train the model.

+
+downstreamJobs
+[String!] +		 +

List of jobs or processes that use this model.

+
+ +## MLModelProperties + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String +		 +

The display name of the model used in the UI

+
+description
+String +		 +

Detailed description of the model's purpose and characteristics

+
+lastModified
+AuditStamp +		 +

When the model was last modified

+
+version
+String +		 +

Version identifier for this model

+
+type
+String +		 +

The type/category of ML model (e.g. classification, regression)

+
+hyperParameters
+HyperParameterMap +		 +

Mapping of hyperparameter configurations

+
+hyperParams
+[MLHyperParam] +		 +

List of hyperparameter settings used to train this model

+
+trainingMetrics
+[MLMetric] +		 +

Performance metrics from model training

+
+mlFeatures
+[String!] +		 +

Names of ML features used by this model

+
+tags
+[String!] +		 +

Tags for categorizing and searching models

+
+groups
+[MLModelGroup] +		 +

Model groups this model belongs to

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Additional custom properties specific to this model

+
+externalUrl
+String +		 +

URL to view this model in external system

+
+created
+AuditStamp +		 +

When this model was created

+
+date
+Long +		 +
Deprecated: Use `created` instead
+ +

Deprecated timestamp for model creation +@deprecated Use 'created' field instead

+
+mlModelLineageInfo
+MLModelLineageInfo +		 +

Information related to lineage to this model group

+
+ +## MLPrimaryKey + +An ML Primary Key Entity Note that this entity is incubating + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the ML Primary Key

+
+type
+EntityType! +		 +

A standard Entity Type

+
+lastIngested
+Long +		 +

The timestamp for the last time this entity was ingested

+
+name
+String! +		 +

The display name

+
+featureNamespace
+String! +		 +

MLPrimaryKey featureNamespace

+
+description
+String +		 +

MLPrimaryKey description

+
+dataType
+MLFeatureDataType +		 +

MLPrimaryKey data type

+
+properties
+MLPrimaryKeyProperties +		 +

Additional read only properties of the ML Primary Key

+
+primaryKeyProperties
+MLPrimaryKeyProperties +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties field instead +MLPrimaryKeyProperties

+
+ownership
+Ownership +		 +

Ownership metadata of the MLPrimaryKey

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the MLPrimaryKey

+
+status
+Status +		 +

Status metadata of the MLPrimaryKey

+
+deprecation
+Deprecation +		 +

Deprecation

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+tags
+GlobalTags +		 +

Tags applied to entity

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the entity

+
+domain
+DomainAssociation +		 +

The Domain associated with the entity

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+editableProperties
+MLPrimaryKeyEditableProperties +		 +

An additional set of of read write properties

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+structuredProperties
+StructuredProperties +		 +

Structured properties about this asset

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+privileges
+EntityPrivileges +		 +

Privileges given to a user relevant to this entity

+
+ +## MLPrimaryKeyEditableProperties + + + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

The edited description

+
+ +## MLPrimaryKeyProperties + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+description
+String +		 + +
+dataType
+MLFeatureDataType +		 + +
+version
+VersionTag +		 + +
+sources
+[Dataset] +		 + +
+customProperties
+[CustomPropertiesEntry!] +		 + +
+ +## MLTrainingRunProperties + +Properties specific to an ML model training run instance + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+id
+String +		 +

Unique identifier for this training run

+
+outputUrls
+[String] +		 +

List of URLs to access training run outputs (e.g. model artifacts, logs)

+
+hyperParams
+[MLHyperParam] +		 +

Hyperparameters used in this training run

+
+trainingMetrics
+[MLMetric] +		 +

Performance metrics recorded during this training run

+
+ +## NamedBar + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 + +
+segments
+[BarSegment!]! +		 + +
+ +## NamedLine + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 + +
+data
+[NumericDataPoint!]! +		 + +
+ +## Notebook + +A Notebook Metadata Entity + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) +- [BrowsableEntity](/docs/graphql/interfaces#browsableentity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Notebook

+
+type
+EntityType! +		 +

A standard Entity Type

+
+tool
+String! +		 +

The Notebook tool name

+
+notebookId
+String! +		 +

An id unique within the Notebook tool

+
+info
+NotebookInfo +		 +

Additional read only information about the Notebook

+
+editableProperties
+NotebookEditableProperties +		 +

Additional read write properties about the Notebook

+
+ownership
+Ownership +		 +

Ownership metadata of the Notebook

+
+status
+Status +		 +

Status metadata of the Notebook

+
+content
+NotebookContent! +		 +

The content of this Notebook

+
+tags
+GlobalTags +		 +

The tags associated with the Notebook

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the Notebook

+
+domain
+DomainAssociation +		 +

The Domain associated with the Notebook

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+dataPlatformInstance
+DataPlatformInstance +		 +

The specific instance of the data platform that this entity belongs to

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+subTypes
+SubTypes +		 +

Sub Types that this entity implements

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the notebook

+
+platform
+DataPlatform! +		 +

Standardized platform.

+
+browsePaths
+[BrowsePath!] +		 +

The browse paths corresponding to the Notebook. If no Browse Paths have been generated before, this will be null.

+
+browsePathV2
+BrowsePathV2 +		 +

The browse path V2 corresponding to an entity. If no Browse Paths V2 have been generated before, this will be null.

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+ +## NotebookCell + +The Union of every NotebookCell + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+chartCell
+ChartCell +		 +

The chart cell content. The will be non-null only when all other cell field is null.

+
+textCell
+TextCell +		 +

The text cell content. The will be non-null only when all other cell field is null.

+
+queryChell
+QueryCell +		 +

The query cell content. The will be non-null only when all other cell field is null.

+
+type
+NotebookCellType! +		 +

The type of this Notebook cell

+
+ +## NotebookContent + +The actual content in a Notebook + +

Fields

+ + + + + + + + + +
NameDescription
+cells
+[NotebookCell!]! +		 +

The content of a Notebook which is composed by a list of NotebookCell

+
+ +## NotebookEditableProperties + +Notebook properties that are editable via the UI This represents logical metadata, +as opposed to technical metadata + +

Fields

+ + + + + + + + + +
NameDescription
+description
+String +		 +

Description of the Notebook

+
+ +## NotebookInfo + +Additional read only information about a Notebook + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+title
+String +		 +

Display of the Notebook

+
+description
+String +		 +

Description of the Notebook

+
+externalUrl
+String +		 +

Native platform URL of the Notebook

+
+customProperties
+[CustomPropertiesEntry!] +		 +

A list of platform specific metadata tuples

+
+changeAuditStamps
+ChangeAuditStamps +		 +

Captures information about who created/last modified/deleted this Notebook and when

+
+ +## NumberValue + +Numeric property value + +

Fields

+ + + + + + + + + +
NameDescription
+numberValue
+Float! +		 +

The value of a number type property

+
+ +## NumericDataPoint + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+x
+String! +		 + +
+y
+Int! +		 + +
+ +## Operation + +Operational info for an entity. + +

Implements

+ +- [TimeSeriesAspect](/docs/graphql/interfaces#timeseriesaspect) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+timestampMillis
+Long! +		 +

The time at which the operation was reported

+
+actor
+String +		 +

Actor who issued this operation.

+
+operationType
+OperationType! +		 +

Operation type of change.

+
+customOperationType
+String +		 +

A custom operation type

+
+sourceType
+OperationSourceType +		 +

Source of the operation

+
+numAffectedRows
+Long +		 +

How many rows were affected by this operation.

+
+affectedDatasets
+[String!] +		 +

Which other datasets were affected by this operation.

+
+lastUpdatedTimestamp
+Long! +		 +

When time at which the asset was actually updated

+
+partition
+String +		 +

Optional partition identifier

+
+customProperties
+[StringMapEntry!] +		 +

Custom operation properties

+
+ +## OperationsAggregation + +An aggregation of Dataset operations statistics + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+bucket
+Long +		 +

The time window start time

+
+duration
+WindowDuration +		 +

The time window span

+
+resource
+String +		 +

The resource urn associated with the operations information, eg a Dataset urn

+
+aggregations
+OperationsAggregationsResult +		 +

The rolled up operations metrics

+
+ +## OperationsAggregationsResult + +Rolled up metrics about Dataset operations over time + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+totalOperations
+Int +		 +

The total number of operations performed within the queried time range

+
+totalInserts
+Int +		 +

The total number of INSERT operations performed within the queried time range

+
+totalUpdates
+Int +		 +

The total number of UPDATE operations performed within the queried time range

+
+totalDeletes
+Int +		 +

The total number of DELETE operations performed within the queried time range

+
+totalCreates
+Int +		 +

The total number of CREATE operations performed within the queried time range

+
+totalAlters
+Int +		 +

The total number of ALTER operations performed within the queried time range

+
+totalDrops
+Int +		 +

The total number of DROP operations performed within the queried time range

+
+totalCustoms
+Int +		 +

The total number of CUSTOM operations performed within the queried time range

+
+customOperationsMap
+[IntMapEntry!] +		 +

A map from each custom operation type to the total count for that type

+
+ +## OperationsQueryResult + +The result of a Dataset operations query + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+buckets
+[OperationsAggregation] +		 +

A set of relevant time windows for use in displaying operations

+
+aggregations
+OperationsAggregationsResult +		 +

A set of rolled up aggregations about the Dataset operations

+
+ +## Origin + +Carries information about where an entity originated from. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+OriginType! +		 +

Where an entity originated from. Either NATIVE or EXTERNAL

+
+externalType
+String +		 +

Only populated if type is EXTERNAL. The externalType of the entity, such as the name of the identity provider.

+
+ +## Owner + +An owner of a Metadata Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+owner
+OwnerType! +		 +

Owner object

+
+type
+OwnershipType +		 +
Deprecated: No longer supported
+ +

The type of the ownership. Deprecated - Use ownershipType field instead.

+
+ownershipType
+OwnershipTypeEntity +		 +

Ownership type information

+
+source
+OwnershipSource +		 +

Source information for the ownership

+
+associatedUrn
+String! +		 +

Reference back to the owned urn for tracking purposes e.g. when sibling nodes are merged together

+
+attribution
+MetadataAttribution +		 +

Information about who, why, and how this metadata was applied

+
+ +## Ownership + +Ownership information about a Metadata Entity + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+owners
+[Owner!] +		 +

List of owners of the entity

+
+lastModified
+AuditStamp! +		 +

Audit stamp containing who last modified the record and when

+
+ +## OwnershipSource + +Information about the source of Ownership metadata about a Metadata Entity + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+OwnershipSourceType! +		 +

The type of the source

+
+url
+String +		 +

An optional reference URL for the source

+
+ +## OwnershipTypeEntity + +A single Custom Ownership Type + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the custom ownership type.

+
+type
+EntityType! +		 +

A standard Entity Type

+
+info
+OwnershipTypeInfo +		 +

Information about the Custom Ownership Type

+
+status
+Status +		 +

Status of the Custom Ownership Type

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from the Custom Ownership Type

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## OwnershipTypeInfo + +Properties about an individual Custom Ownership Type. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the Custom Ownership Type

+
+description
+String +		 +

The description of the Custom Ownership Type

+
+created
+AuditStamp +		 +

An Audit Stamp corresponding to the creation of this resource

+
+lastModified
+AuditStamp +		 +

An Audit Stamp corresponding to the update of this resource

+
+ +## ParentContainersResult + +All of the parent containers for a given entity. Returns parents with direct parent first followed by the parent's parent etc. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+count
+Int! +		 +

The number of containers bubbling up for this entity

+
+containers
+[Container!]! +		 +

A list of parent containers in order from direct parent, to parent's parent etc. If there are no containers, return an emty list

+
+ +## ParentDocumentsResult + +All of the parent documents for a given document. Returns parents with direct parent first followed by the parent's parent, etc. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+count
+Int! +		 +

The number of parent documents bubbling up for this document

+
+documents
+[Document!]! +		 +

The ordered list of parent documents, starting with the direct parent

+
+ +## ParentDomainsResult + +All of the parent domains starting from a single Domain through all of its ancestors + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+count
+Int! +		 +

The number of parent domains bubbling up for this entity

+
+domains
+[Entity!]! +		 +

A list of parent domains in order from direct parent, to parent's parent etc. If there are no parents, return an empty list

+
+ +## ParentNodesResult + +All of the parent nodes for GlossaryTerms and GlossaryNodes + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+count
+Int! +		 +

The number of parent nodes bubbling up for this entity

+
+nodes
+[GlossaryNode!]! +		 +

A list of parent nodes in order from direct parent, to parent's parent etc. If there are no nodes, return an empty list

+
+ +## PartitionSpec + +Information about the partition being profiled + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+PartitionType! +		 +

The partition type

+
+partition
+String +		 +

The partition identifier

+
+timePartition
+TimeWindow +		 +

The optional time window partition information - required if type is TIMESTAMP_FIELD.

+
+ +## PatchEntityResult + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+name
+String +		 + +
+success
+Boolean! +		 + +
+error
+String +		 + +
+ +## PlatformPrivileges + +The platform privileges that the currently authenticated user has + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+viewAnalytics
+Boolean! +		 +

Whether the user should be able to view analytics

+
+managePolicies
+Boolean! +		 +

Whether the user should be able to manage policies

+
+manageIdentities
+Boolean! +		 +

Whether the user should be able to manage users & groups

+
+generatePersonalAccessTokens
+Boolean! +		 +

Whether the user should be able to generate personal access tokens

+
+createDomains
+Boolean! +		 +

Whether the user should be able to create new Domains

+
+manageDomains
+Boolean! +		 +

Whether the user should be able to manage Domains

+
+manageIngestion
+Boolean! +		 +

Whether the user is able to manage UI-based ingestion

+
+manageSecrets
+Boolean! +		 +

Whether the user is able to manage UI-based secrets

+
+manageTokens
+Boolean! +		 +

Whether the user should be able to manage tokens on behalf of other users.

+
+viewTests
+Boolean! +		 +

Whether the user is able to view Tests

+
+manageTests
+Boolean! +		 +

Whether the user is able to manage Tests

+
+manageGlossaries
+Boolean! +		 +

Whether the user should be able to manage Glossaries

+
+manageDocuments
+Boolean! +		 +

Whether the user should be able to manage Documents (Knowledge Articles)

+
+manageUserCredentials
+Boolean! +		 +

Whether the user is able to manage user credentials

+
+createTags
+Boolean! +		 +

Whether the user should be able to create new Tags

+
+manageTags
+Boolean! +		 +

Whether the user should be able to create and delete all Tags

+
+viewManageTags
+Boolean! +		 +

Whether the user should be able to view the tags management page.

+
+manageGlobalViews
+Boolean! +		 +

Whether the user should be able to create, update, and delete global views.

+
+manageOwnershipTypes
+Boolean! +		 +

Whether the user should be able to create, update, and delete ownership types.

+
+manageGlobalAnnouncements
+Boolean! +		 +

Whether the user can create and delete posts pinned to the home page.

+
+createBusinessAttributes
+Boolean! +		 +

Whether the user can create Business Attributes.

+
+manageBusinessAttributes
+Boolean! +		 +

Whether the user can manage Business Attributes.

+
+manageStructuredProperties
+Boolean! +		 +

Whether the user can create, edit, and delete structured properties.

+
+viewStructuredPropertiesPage
+Boolean! +		 +

Whether the user can view the manage structured properties page.

+
+manageApplications
+Boolean! +		 +

Whether the user can manage applications.

+
+manageFeatures
+Boolean! +		 +

Whether the user can manage platform features.

+
+manageHomePageTemplates
+Boolean! +		 +

Whether the user can manage default home page template.

+
+manageServiceAccounts
+Boolean! +		 +

Whether the user can create, update, and delete service accounts.

+
+ +## PoliciesConfig + +Configurations related to the Policies Feature + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether the policies feature is enabled and should be displayed in the UI

+
+platformPrivileges
+[Privilege!]! +		 +

A list of platform privileges to display in the Policy Builder experience

+
+resourcePrivileges
+[ResourcePrivileges!]! +		 +

A list of resource privileges to display in the Policy Builder experience

+
+ +## Policy + +DEPRECATED +TODO: Eventually get rid of this in favor of DataHub Policy +An DataHub Platform Access Policy Access Policies determine who can perform what actions against which resources on the platform + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Policy

+
+type
+PolicyType! +		 +

The type of the Policy

+
+name
+String! +		 +

The name of the Policy

+
+state
+PolicyState! +		 +

The present state of the Policy

+
+description
+String +		 +

The description of the Policy

+
+resources
+ResourceFilter +		 +

The resources that the Policy privileges apply to

+
+privileges
+[String!]! +		 +

The privileges that the Policy grants

+
+actors
+ActorFilter! +		 +

The actors that the Policy grants privileges to

+
+editable
+Boolean! +		 +

Whether the Policy is editable, ie system policies, or not

+
+ +## PolicyEvaluationDetail + +Details about how a policy was evaluated for a given actor and resource + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+policyName
+String! +		 +

The policy that was evaluated

+
+reason
+String! +		 +

The reason for deny for this policy

+
+ +## PolicyMatchCriterion + +Criterion to define relationship between field and values + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+field
+String! +		 +

The name of the field that the criterion refers to +e.g. entity_type, entity_urn, domain

+
+values
+[PolicyMatchCriterionValue!]! +		 +

Values. Matches criterion if any one of the values matches condition (OR-relationship)

+
+condition
+PolicyMatchCondition! +		 +

The name of the field that the criterion refers to

+
+ +## PolicyMatchCriterionValue + +Value in PolicyMatchCriterion with hydrated entity if value is urn + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+value
+String! +		 +

The value of the field to match

+
+entity
+Entity +		 +

Hydrated entities of the above values. Only set if the value is an urn

+
+ +## PolicyMatchFilter + +Filter object that encodes a complex filter logic with OR + AND + +

Fields

+ + + + + + + + + +
NameDescription
+criteria
+[PolicyMatchCriterion!] +		 +

List of criteria to apply

+
+ +## Post + +Input provided when creating a Post + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Post

+
+type
+EntityType! +		 +

The standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from the Post

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+postType
+PostType! +		 +

The type of post

+
+content
+PostContent! +		 +

The content of the post

+
+lastModified
+AuditStamp! +		 +

When the post was last modified

+
+ +## PostContent + +Post content + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+contentType
+PostContentType! +		 +

The type of post content

+
+title
+String! +		 +

The title of the post

+
+description
+String +		 +

Optional content of the post

+
+link
+String +		 +

Optional link that the post is associated with

+
+media
+Media +		 +

Optional media contained in the post

+
+ +## Privilege + +An individual DataHub Access Privilege + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+String! +		 +

Standardized privilege type, serving as a unique identifier for a privilege eg EDIT_ENTITY

+
+displayName
+String +		 +

The name to appear when displaying the privilege, eg Edit Entity

+
+description
+String +		 +

A description of the privilege to display

+
+ +## Privileges + +Object that encodes the privileges the actor has for a given resource + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+privileges
+[String!]! +		 +

Granted Privileges

+
+evaluationDetails
+[PolicyEvaluationDetail] +		 +

Details about how each policy was evaluated

+
+ +## ProductUpdate + +Product update information fetched from remote JSON source + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether this update is enabled and should be shown

+
+id
+String! +		 +

Unique identifier for this update (e.g., "v1.2.1") +Version changes trigger re-display for users who dismissed previous versions

+
+title
+String! +		 +

Title of the update notification

+
+header
+String +		 +

Optional header text (displayed instead of title if provided)

+
+requiredVersion
+String +		 +

Optional minimum required version for this update

+
+image
+String +		 +

Optional URL to an image to display with the update

+
+description
+String +		 +

Optional description text for the update

+
+primaryCtaText
+String +		 +

Primary call-to-action button text (required if primaryCtaLink is provided)

+
+primaryCtaLink
+String +		 +

Primary call-to-action link URL, with telemetry client ID appended (required if primaryCtaText is provided) +Relative URLs will be prefixed with baseUrl

+
+secondaryCtaText
+String +		 +

Secondary call-to-action button text (optional)

+
+secondaryCtaLink
+String +		 +

Secondary call-to-action link URL, with telemetry client ID appended (optional)

+
+ctaText
+String! +		 +

Call-to-action button text (deprecated, use primaryCtaText instead) +Kept for backward compatibility

+
+ctaLink
+String! +		 +

Call-to-action link URL, with telemetry client ID appended (deprecated, use primaryCtaLink instead) +Kept for backward compatibility

+
+features
+[ProductUpdateFeature!] +		 +

Optional list of features (up to 3) to display with icons and descriptions

+
+ +## ProductUpdateFeature + +Product update feature information + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+title
+String! +		 +

Title of the feature (subheader)

+
+description
+String! +		 +

Description text for the feature (bullet text)

+
+icon
+String +		 +

Phosphor icon name (PascalCase, e.g., "Lightning", "Sparkle", "Settings", "Domain") +Optional - if not provided, a default bullet will be shown

+
+availability
+String +		 +

Optional availability text (e.g., "Available in DataHub Cloud")

+
+ +## Quantile + +A quantile along with its corresponding value + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+quantile
+String! +		 +

Quantile. E.g. "0.25" for the 25th percentile

+
+value
+String! +		 +

The value of the quantile

+
+ +## QuantitativeAnalyses + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+unitaryResults
+ResultsType +		 +

Link to a dashboard with results showing how the model performed with respect to each factor

+
+intersectionalResults
+ResultsType +		 +

Link to a dashboard with results showing how the model performed with respect to the intersection of evaluated factors

+
+ +## QueriesTabConfig + +Configuration for the queries tab + +

Fields

+ + + + + + + + + +
NameDescription
+queriesTabResultSize
+Int +		 +

Number of queries to show in the queries tab

+
+ +## QueryCell + +A Notebook cell which contains Query as content + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+cellTitle
+String! +		 +

Title of the cell

+
+cellId
+String! +		 +

Unique id for the cell.

+
+changeAuditStamps
+ChangeAuditStamps +		 +

Captures information about who created/last modified/deleted this TextCell and when

+
+rawQuery
+String! +		 +

Raw query to explain some specific logic in a Notebook

+
+lastExecuted
+AuditStamp +		 +

Captures information about who last executed this query cell and when

+
+ +## QueryEntity + +An individual Query + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the Query

+
+type
+EntityType! +		 +

A standard Entity Type

+
+properties
+QueryProperties +		 +

Properties about the Query

+
+subjects
+[QuerySubject!] +		 +

Subjects for the query

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+platform
+DataPlatform +		 +

Platform from which the Query was detected

+
+ +## QueryProperties + +Properties about an individual Query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+statement
+QueryStatement! +		 +

The Query statement itself

+
+source
+QuerySource! +		 +

The source of the Query

+
+name
+String +		 +

The name of the Query

+
+description
+String +		 +

The description of the Query

+
+created
+AuditStamp! +		 +

An Audit Stamp corresponding to the creation of this resource

+
+createdOn
+ResolvedAuditStamp! +		 +

A Resolved Audit Stamp corresponding to the creation of this resource

+
+lastModified
+AuditStamp! +		 +

An Audit Stamp corresponding to the update of this resource

+
+origin
+Entity +		 +

The asset that this query originated from, e.g. a View, a dbt Model, etc.

+
+customProperties
+[CustomPropertiesEntry!] +		 +

Custom properties of the Data Product

+
+ +## QueryStatement + +An individual Query Statement + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+value
+String! +		 +

The query statement value

+
+language
+QueryLanguage! +		 +

The language for the Query Statement

+
+ +## QuerySubject + +The subject for a Query + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+dataset
+Dataset! +		 +

The dataset which is the subject of the Query

+
+schemaField
+SchemaFieldEntity +		 +

The schema field which is the subject of the Query. +This will be populated if the subject is specifically a schema field.

+
+ +## QuickFilter + +A quick filter in search and auto-complete + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+field
+String! +		 +

Name of field to filter by

+
+value
+String! +		 +

Value to filter on

+
+entity
+Entity +		 +

Entity that the value maps to if any

+
+ +## RawAspect + +Payload representing data about a single aspect + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+aspectName
+String! +		 +

The name of the aspect

+
+payload
+String +		 +

JSON string containing the aspect's payload

+
+renderSpec
+AspectRenderSpec +		 +

Details for the frontend on how the raw aspect should be rendered

+
+ +## RecommendationContent + +Content to display within each recommendation module + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+value
+String! +		 +

String representation of content

+
+entity
+Entity +		 +

Entity being recommended. Empty if the content being recommended is not an entity

+
+params
+RecommendationParams +		 +

Additional context required to generate the the recommendation

+
+ +## RecommendationModule + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+title
+String! +		 +

Title of the module to display

+
+moduleId
+String! +		 +

Unique id of the module being recommended

+
+renderType
+RecommendationRenderType! +		 +

Type of rendering that defines how the module should be rendered

+
+content
+[RecommendationContent!]! +		 +

List of content to display inside the module

+
+ +## RecommendationParams + +Parameters required to render a recommendation of a given type + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+searchParams
+SearchParams +		 +

Context to define the search recommendations

+
+entityProfileParams
+EntityProfileParams +		 +

Context to define the entity profile page

+
+contentParams
+ContentParams +		 +

Context about the recommendation

+
+ +## RelatedDocumentsResult + +Result containing context documents related to an entity. +Same structure as SearchDocumentsResult. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Documents in the returned result set

+
+total
+Int! +		 +

The total number of Documents in the result set

+
+documents
+[Document!]! +		 +

The Documents themselves

+
+facets
+[FacetMetadata!] +		 +

Facets for filtering search results

+
+ +## RelationshipFieldMapping + +ERModelRelationship FieldMap + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+sourceField
+String! +		 +

left field

+
+destinationField
+String! +		 +

bfield

+
+ +## ResetToken + +Token that allows native users to reset their credentials + +

Fields

+ + + + + + + + + +
NameDescription
+resetToken
+String! +		 +

The reset token

+
+ +## ResolvedAuditStamp + +Audit stamp containing a resolved actor + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+time
+Long! +		 +

When the audited action took place

+
+actor
+CorpUser +		 +

Who performed the audited action

+
+ +## ResourceFilter + +The resources that a DataHub Access Policy applies to + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+String +		 +
Deprecated: No longer supported
+ +

Deprecated, use filter instead +The type of the resource the policy should apply to Not required because in the future we want to support filtering by type OR by domain

+
+resources
+[String!] +		 +
Deprecated: No longer supported
+ +

Deprecated, use filter instead +A list of specific resource urns to apply the filter to

+
+allResources
+Boolean +		 +
Deprecated: No longer supported
+ +

Deprecated, use filter instead +Whether of not to apply the filter to all resources of the type

+
+filter
+PolicyMatchFilter +		 +

Whether of not to apply the filter to all resources of the type

+
+privilegeConstraints
+PolicyMatchFilter +		 +

Constraints on what subresources can be acted upon

+
+ +## ResourcePrivileges + +A privilege associated with a particular resource type +A resource is most commonly a DataHub Metadata Entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+resourceType
+String! +		 +

Resource type associated with the Access Privilege, eg dataset

+
+resourceTypeDisplayName
+String +		 +

The name to used for displaying the resourceType

+
+entityType
+EntityType +		 +

An optional entity type to use when performing search and navigation to the entity

+
+privileges
+[Privilege!]! +		 +

A list of privileges that are supported against this resource

+
+ +## Restricted + +A restricted entity that the user does not have full permissions to view. +This entity type does not relate to an entity type in the database. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the restricted entity

+
+type
+EntityType! +		 +

The standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

Edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+ +## RichTextModuleParams + +The params required if the module is type RICH_TEXT + +

Fields

+ + + + + + + + + +
NameDescription
+content
+String! +		 +

The content of the rich text module

+
+ +## Role + + + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key of the Metadata Entity

+
+type
+EntityType! +		 +

A standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

List of relationships between the source Entity and some destination entities with a given types

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+id
+String! +		 +

Id of the Role

+
+properties
+RoleProperties +		 +

Role properties to include Request Access Url

+
+actors
+Actor +		 +

A standard Entity Type

+
+isAssignedToMe
+Boolean! +		 + +
+ +## RoleAssociation + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+role
+Role! +		 +

The Role entity itself

+
+associatedUrn
+String! +		 +

Reference back to the tagged urn for tracking purposes e.g. when sibling nodes are merged together

+
+ +## RoleGroup + + + +

Fields

+ + + + + + + + + +
NameDescription
+group
+CorpGroup! +		 +

Linked corp group of a role

+
+ +## RoleProperties + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

Name of the Role in an organisation

+
+description
+String +		 +

Description about the role

+
+type
+String +		 +

Role type can be READ, WRITE or ADMIN

+
+requestUrl
+String +		 +

Url to request a role for a user in an organisation

+
+ +## RoleUser + + + +

Fields

+ + + + + + + + + +
NameDescription
+user
+CorpUser! +		 +

Linked corp user of a role

+
+ +## Row + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+values
+[String!]! +		 + +
+cells
+[Cell!] +		 + +
+ +## RowCountChange + +Attributes defining an ROW_COUNT_CHANGE volume assertion. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+AssertionValueChangeType! +		 +

The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage.

+
+operator
+AssertionStdOperator! +		 +

The operator you'd like to apply. +Note that only numeric operators are valid inputs: +GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, +BETWEEN.

+
+parameters
+AssertionStdParameters! +		 +

The parameters you'd like to provide as input to the operator. +Note that only numeric parameter types are valid inputs: NUMBER.

+
+ +## RowCountTotal + +Attributes defining an ROW_COUNT_TOTAL volume assertion. + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+operator
+AssertionStdOperator! +		 +

The operator you'd like to apply. +Note that only numeric operators are valid inputs: +GREATER_THAN, GREATER_THAN_OR_EQUAL_TO, EQUAL_TO, LESS_THAN, LESS_THAN_OR_EQUAL_TO, +BETWEEN.

+
+parameters
+AssertionStdParameters! +		 +

The parameters you'd like to provide as input to the operator. +Note that only numeric parameter types are valid inputs: NUMBER.

+
+ +## Schema + +Deprecated, use SchemaMetadata instead +Metadata about a Dataset schema + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+datasetUrn
+String +		 +

Dataset this schema metadata is associated with

+
+name
+String! +		 +

Schema name

+
+platformUrn
+String! +		 +

Platform this schema metadata is associated with

+
+version
+Long! +		 +

The version of the GMS Schema metadata

+
+cluster
+String +		 +

The cluster this schema metadata is derived from

+
+hash
+String! +		 +

The SHA1 hash of the schema content

+
+platformSchema
+PlatformSchema +		 +

The native schema in the datasets platform, schemaless if it was not provided

+
+fields
+[SchemaField!]! +		 +

Client provided a list of fields from value schema

+
+primaryKeys
+[String!] +		 +

Client provided list of fields that define primary keys to access record

+
+foreignKeys
+[ForeignKeyConstraint] +		 +

Client provided list of foreign key constraints

+
+createdAt
+Long +		 +

The time at which the schema metadata information was created

+
+lastObserved
+Long +		 +

The time at which the schema metadata information was last ingested

+
+ +## SchemaAssertionField + +Defines a schema field, each with a specified path and type. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+path
+String! +		 +

The standard V1 path of the field within the schema.

+
+type
+SchemaFieldDataType! +		 +

The std type of the field

+
+nativeType
+String +		 +

Optional: The specific native or standard type of the field.

+
+ +## SchemaAssertionInfo + +Information about an Schema assertion + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entityUrn
+String! +		 +

The entity targeted by this schema assertion.

+
+fields
+[SchemaAssertionField!]! +		 +

A single field in the schema assertion.

+
+schema
+SchemaMetadata +		 +

A definition of the expected structure for the asset +Deprecated! Use the simpler 'fields' instead.

+
+compatibility
+SchemaAssertionCompatibility! +		 +

The compatibility level required for the assertion to pass.

+
+ +## SchemaContract + + + +

Fields

+ + + + + + + + + +
NameDescription
+assertion
+Assertion! +		 +

The assertion representing the schema contract.

+
+ +## SchemaField + +Information about an individual field in a Dataset schema + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+fieldPath
+String! +		 +

Flattened name of the field computed from jsonPath field

+
+jsonPath
+String +		 +

Flattened name of a field in JSON Path notation

+
+label
+String +		 +

Human readable label for the field. Not supplied by all data sources

+
+nullable
+Boolean! +		 +

Indicates if this field is optional or nullable

+
+description
+String +		 +

Description of the field

+
+type
+SchemaFieldDataType! +		 +

Platform independent field type of the field

+
+nativeDataType
+String +		 +

The native type of the field in the datasets platform as declared by platform schema

+
+recursive
+Boolean! +		 +

Whether the field references its own type recursively

+
+globalTags
+GlobalTags +		 +
Deprecated: No longer supported
+ +

Deprecated, use tags field instead +Tags associated with the field

+
+tags
+GlobalTags +		 +

Tags associated with the field

+
+glossaryTerms
+GlossaryTerms +		 +

Glossary terms associated with the field

+
+isPartOfKey
+Boolean +		 +

Whether the field is part of a key schema

+
+isPartitioningKey
+Boolean +		 +

Whether the field is part of a partitioning key schema

+
+jsonProps
+String +		 +

For schema fields that have other properties that are not modeled explicitly, represented as a JSON string.

+
+schemaFieldEntity
+SchemaFieldEntity +		 +

Schema field entity that exist in the database for this schema field

+
+ +## SchemaFieldBlame + +Blame for a single field + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+fieldPath
+String! +		 +

Flattened name of a schema field

+
+schemaFieldChange
+SchemaFieldChange! +		 +

Attributes identifying a field change

+
+ +## SchemaFieldChange + +Attributes identifying a field change + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+timestampMillis
+Long! +		 +

The time at which the schema was updated

+
+lastSemanticVersion
+String! +		 +

The last semantic version that this schema was changed in

+
+versionStamp
+String! +		 +

Version stamp of the change

+
+changeType
+ChangeOperationType! +		 +

The type of the change

+
+lastSchemaFieldChange
+String +		 +

Last column update, such as Added/Modified/Removed in v1.2.3.

+
+ +## SchemaFieldEntity + +Standalone schema field entity. Differs from the SchemaField struct because it is not directly nested inside a +schema field + +

Implements

+ +- [EntityWithRelationships](/docs/graphql/interfaces#entitywithrelationships) +- [Entity](/docs/graphql/interfaces#entity) +- [HasLogicalParent](/docs/graphql/interfaces#haslogicalparent) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

Primary key of the schema field

+
+type
+EntityType! +		 +

A standard Entity Type

+
+fieldPath
+String! +		 +

Field path identifying the field in its dataset

+
+parent
+Entity! +		 +

The field's parent.

+
+structuredProperties
+StructuredProperties +		 +

Structured properties on this schema field

+
+forms
+Forms +		 +

The forms associated with the Dataset

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+relatedDocuments
+RelatedDocumentsResult +		 +

Get context documents related to this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelatedDocumentsInput! +		 + +
+ +
+lineage
+EntityLineageResult +		 +

Edges extending from this entity grouped by direction in the lineage graph

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+LineageInput! +		 + +
+ +
+businessAttributes
+BusinessAttributes +		 +

Business Attribute associated with the field

+
+documentation
+Documentation +		 +

Documentation aspect for this schema field

+
+status
+Status +		 +

The status of the SchemaFieldEntity

+
+deprecation
+Deprecation +		 +

deprecation status of the schema field

+
+tags
+GlobalTags +		 +

Tags associated with the field

+
+glossaryTerms
+GlossaryTerms +		 +

Glossary terms associated with the field

+
+logicalParent
+Entity +		 +

If this entity represents a physical asset, this is its logical parent, from which metadata can propagate.

+
+ +## SchemaFieldRef + +A Dataset schema field (i.e. column) + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A schema field urn

+
+path
+String! +		 +

A schema field path

+
+ +## SchemaFieldSpec + +Information about the field to use in an assertion + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+path
+String! +		 +

The field path

+
+type
+String! +		 +

The DataHub standard schema field type.

+
+nativeType
+String! +		 +

The native field type

+
+ +## SchemaMetadata + +Metadata about a Dataset schema + +

Implements

+ +- [Aspect](/docs/graphql/interfaces#aspect) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+aspectVersion
+Long +		 +

The logical version of the schema metadata, where zero represents the latest version +with otherwise monotonic ordering starting at one

+
+datasetUrn
+String +		 +

Dataset this schema metadata is associated with

+
+name
+String! +		 +

Schema name

+
+platformUrn
+String! +		 +

Platform this schema metadata is associated with

+
+version
+Long! +		 +

The version of the GMS Schema metadata

+
+cluster
+String +		 +

The cluster this schema metadata is derived from

+
+hash
+String! +		 +

The SHA1 hash of the schema content

+
+platformSchema
+PlatformSchema +		 +

The native schema in the datasets platform, schemaless if it was not provided

+
+fields
+[SchemaField!]! +		 +

Client provided a list of fields from value schema

+
+primaryKeys
+[String!] +		 +

Client provided list of fields that define primary keys to access record

+
+foreignKeys
+[ForeignKeyConstraint] +		 +

Client provided list of foreign key constraints

+
+createdAt
+Long +		 +

The time at which the schema metadata information was created

+
+ +## ScrollAcrossLineageResults + +Results returned by issuing a search across relationships query using scroll API + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+nextScrollId
+String +		 +

Opaque ID to pass to the next request to the server

+
+count
+Int! +		 +

The number of entities included in the result set

+
+total
+Int! +		 +

The total number of search results matching the query and filters

+
+searchResults
+[SearchAcrossLineageResult!]! +		 +

The search result entities

+
+facets
+[FacetMetadata!] +		 +

Candidate facet aggregations used for search filtering

+
+freshness
+FreshnessStats +		 +

Optional freshness characteristics of this query (cached, staleness etc.)

+
+isPartial
+Boolean +		 +

Indicates whether the results are partial due to reaching the maxRelations limit or timeout

+
+ +## ScrollResults + +Results returned by issuing a search query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+nextScrollId
+String +		 +

Opaque ID to pass to the next request to the server

+
+count
+Int! +		 +

The number of entities included in the result set

+
+total
+Int! +		 +

The total number of search results matching the query and filters

+
+searchResults
+[SearchResult!]! +		 +

The search result entities for a scroll request

+
+facets
+[FacetMetadata!] +		 +

Candidate facet aggregations used for search filtering

+
+ +## SearchAcrossLineageResult + +Individual search result from a search across relationships query (has added metadata about the path) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entity
+Entity! +		 +

The resolved DataHub Metadata Entity matching the search query

+
+insights
+[SearchInsight!] +		 +

Insights about why the search result was matched

+
+matchedFields
+[MatchedField!]! +		 +

Matched field hint

+
+paths
+[EntityPath] +		 +

Optional list of entities between the source and destination node

+
+degree
+Int! +		 +

Degree of relationship (number of hops to get to entity)

+
+degrees
+[Int!] +		 +

Degrees of relationship (for entities discoverable at multiple degrees)

+
+explored
+Boolean! +		 +

Marks whether or not this entity was explored further for lineage

+
+truncatedChildren
+Boolean! +		 +

Indicates this destination node has additional unexplored child relationships

+
+ignoredAsHop
+Boolean! +		 +

Whether this relationship was ignored as a hop

+
+ +## SearchAcrossLineageResults + +Results returned by issuing a search across relationships query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The offset of the result set

+
+count
+Int! +		 +

The number of entities included in the result set

+
+total
+Int! +		 +

The total number of search results matching the query and filters

+
+searchResults
+[SearchAcrossLineageResult!]! +		 +

The search result entities

+
+facets
+[FacetMetadata!] +		 +

Candidate facet aggregations used for search filtering

+
+freshness
+FreshnessStats +		 +

Optional freshness characteristics of this query (cached, staleness etc.)

+
+lineageSearchPath
+LineageSearchPath +		 +

The path taken when doing search across lineage

+
+isPartial
+Boolean +		 +

Indicates whether the results are partial due to reaching the maxRelations limit or timeout

+
+ +## SearchBarConfig + +Configurations related to the Search bar + +

Fields

+ + + + + + + + + +
NameDescription
+apiVariant
+SearchBarAPI! +		 +

API variant

+
+ +## SearchCardConfig + +Configurations related to the Search card + +

Fields

+ + + + + + + + + +
NameDescription
+showDescription
+Boolean! +		 +

Whether the search card should show description

+
+ +## SearchDocumentsResult + +The result obtained when searching Documents + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The starting offset of the result set returned

+
+count
+Int! +		 +

The number of Documents in the returned result set

+
+total
+Int! +		 +

The total number of Documents in the result set

+
+documents
+[Document!]! +		 +

The Documents themselves

+
+facets
+[FacetMetadata!] +		 +

Facets for filtering search results

+
+ +## SearchFlagsConfig + +Configurations related the Search Flags + +

Fields

+ + + + + + + + + +
NameDescription
+defaultSkipHighlighting
+Boolean! +		 +

Default value for skipHighlighting search flag. Currently used in Search Page and Search Bar

+
+ +## SearchInsight + +Insights about why a search result was returned or ranked in the way that it was + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+text
+String! +		 +

The insight to display

+
+icon
+String +		 +

An optional emoji to display in front of the text

+
+ +## SearchParams + +Context to define the search recommendations + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+types
+[EntityType!] +		 +

Entity types to be searched. If this is not provided, all entities will be searched.

+
+query
+String! +		 +

Search query

+
+filters
+[FacetFilter!] +		 +

Filters

+
+ +## SearchResult + +An individual search result hit + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entity
+Entity! +		 +

The resolved DataHub Metadata Entity matching the search query

+
+insights
+[SearchInsight!] +		 +

Insights about why the search result was matched

+
+matchedFields
+[MatchedField!]! +		 +

Matched field hint

+
+extraProperties
+[ExtraProperty!] +		 +

Additional properties about the search result. Used for rendering in the UI

+
+ +## SearchResults + +Results returned by issuing a search query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+start
+Int! +		 +

The offset of the result set

+
+count
+Int! +		 +

The number of entities included in the result set

+
+total
+Int! +		 +

The total number of search results matching the query and filters

+
+searchResults
+[SearchResult!]! +		 +

The search result entities

+
+facets
+[FacetMetadata!] +		 +

Candidate facet aggregations used for search filtering

+
+suggestions
+[SearchSuggestion!] +		 +

Search suggestions based on the query provided for alternate query texts

+
+ +## SearchResultsVisualConfig + +Configuration for a search result + +

Fields

+ + + + + + + + + +
NameDescription
+enableNameHighlight
+Boolean +		 +

Whether a search result should highlight the name/description if it was matched on those fields.

+
+ +## SearchSuggestion + +A suggestion for an alternate search query given an original query compared to all +of the entity names in our search index. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+text
+String! +		 +

The suggested text based on the provided query text compared to +the entity name field in the search index.

+
+score
+Float +		 +

The "edit distance" for this suggestion. The closer this number is to 1, the +closer the suggested text is to the original text. The closer it is to 0, the +further from the original text it is.

+
+frequency
+Int +		 +

The number of entities that would match on the name field given the suggested text

+
+ +## Secret + +A referencible secret stored in DataHub's system. Notice that we do not return the actual secret value. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The urn of the secret

+
+name
+String! +		 +

The name of the secret

+
+description
+String +		 +

An optional description for the secret

+
+ +## SecretValue + +A plaintext secret value + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

The name of the secret

+
+value
+String! +		 +

The plaintext value of the secret.

+
+ +## SemanticSearchConfig + +Configuration for semantic search including embedding generation settings + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether semantic search is enabled on the server

+
+enabledEntities
+[String!]! +		 +

Entity types that support semantic search (e.g., ["document"])

+
+embeddingConfig
+EmbeddingConfig +		 +

Embedding generation configuration (nullable when semantic search is disabled)

+
+ +## SemanticVersionStruct + +Properties identify a semantic version + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+semanticVersion
+String +		 +

Semantic version of the change

+
+semanticVersionTimestamp
+Long +		 +

Semantic version timestamp

+
+versionStamp
+String +		 +

Version stamp of the change

+
+ +## ServiceAccount + +A service account represents a non-human identity for programmatic API access. + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the service account

+
+type
+EntityType! +		 +

The standard Entity Type

+
+name
+String! +		 +

The unique name/identifier of the service account.

+
+displayName
+String +		 +

The display name of the service account.

+
+description
+String +		 +

An optional description of the service account.

+
+createdBy
+String +		 +

The actor URN that created this service account.

+
+createdAt
+Long +		 +

The time when the service account was created. +May be null in list operations where full entity details are not fetched.

+
+updatedAt
+Long +		 +

The time when the service account was last updated.

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## SiblingProperties + +Metadata about the entity's siblings + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+isPrimary
+Boolean +		 +

If this entity is the primary sibling among the sibling set

+
+siblings
+[Entity] +		 +

The sibling entities

+
+ +## SourceCode + + + +

Fields

+ + + + + + + + + +
NameDescription
+sourceCode
+[SourceCodeUrl!] +		 +

Source Code along with types

+
+ +## SourceCodeUrl + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+type
+SourceCodeUrlType! +		 +

Source Code Url Types

+
+sourceCodeUrl
+String! +		 +

Source Code Url

+
+ +## SqlAssertionInfo + +Attributes defining a SQL Assertion + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+type
+SqlAssertionType! +		 +

The type of the SQL assertion being monitored.

+
+entityUrn
+String! +		 +

The entity targeted by this SQL check.

+
+statement
+String! +		 +

The SQL statement to be executed when evaluating the assertion.

+
+changeType
+AssertionValueChangeType +		 +

The type of the value used to evaluate the assertion: a fixed absolute value or a relative percentage. +Required if the type is METRIC_CHANGE.

+
+operator
+AssertionStdOperator! +		 +

The operator you'd like to apply to the result of the SQL query.

+
+parameters
+AssertionStdParameters! +		 +

The parameters you'd like to provide as input to the operator.

+
+ +## Status + +The status of a particular Metadata Entity + +

Fields

+ + + + + + + + + +
NameDescription
+removed
+Boolean! +		 +

Whether the entity is removed or not

+
+ +## StepStateResult + +A single step state + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+id
+String! +		 +

Unique id of the step

+
+properties
+[StringMapEntry!]! +		 +

The properties for the step state

+
+ +## StringBox + + + +

Fields

+ + + + + + + + + +
NameDescription
+stringValue
+String! +		 + +
+ +## StringMapEntry + +An entry in a string string map represented as a tuple + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+key
+String! +		 +

The key of the map entry

+
+value
+String +		 +

The value fo the map entry

+
+ +## StringValue + +String property value + +

Fields

+ + + + + + + + + +
NameDescription
+stringValue
+String! +		 +

The value of a string type property

+
+ +## StructuredProperties + +Object containing structured properties for an entity + +

Fields

+ + + + + + + + + +
NameDescription
+properties
+[StructuredPropertiesEntry!] +		 +

Structured properties on this entity

+
+ +## StructuredPropertiesEntry + +An entry in an structured properties list represented as a tuple + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+structuredProperty
+StructuredPropertyEntity! +		 +

The key of the map entry

+
+values
+[PropertyValue]! +		 +

The values of the structured property for this entity

+
+valueEntities
+[Entity] +		 +

The optional entities associated with the values if the values are entity urns

+
+associatedUrn
+String! +		 +

The urn of the entity this property came from for tracking purposes e.g. when sibling nodes are merged together

+
+attribution
+MetadataAttribution +		 +

Information about who, why, and how this metadata was applied

+
+ +## StructuredPropertyDefinition + +Properties about an individual Query + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+qualifiedName
+String! +		 +

The fully qualified name of the property. This includes its namespace

+
+displayName
+String +		 +

The display name of this structured property

+
+description
+String +		 +

The description of this property

+
+cardinality
+PropertyCardinality +		 +

The cardinality of a Structured Property determining whether one or multiple values +can be applied to the entity from this property.

+
+allowedValues
+[AllowedValue!] +		 +

A list of allowed values that the property is allowed to take.

+
+valueType
+DataTypeEntity! +		 +

The type of this structured property

+
+typeQualifier
+TypeQualifier +		 +

Allows for type specialization of the valueType to be more specific about which +entity types are allowed, for example.

+
+entityTypes
+[EntityTypeEntity!]! +		 +

Entity types that this structured property can be applied to

+
+immutable
+Boolean! +		 +

Whether or not this structured property is immutable

+
+created
+ResolvedAuditStamp +		 +

Audit stamp for when this structured property was created

+
+lastModified
+ResolvedAuditStamp +		 +

Audit stamp for when this structured property was last modified

+
+ +## StructuredPropertyEntity + +A structured property that can be shared between different entities + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

A primary key associated with the structured property

+
+type
+EntityType! +		 +

A standard Entity Type

+
+exists
+Boolean +		 +

Whether or not this entity exists on DataHub

+
+definition
+StructuredPropertyDefinition! +		 +

Definition of this structured property including its name

+
+settings
+StructuredPropertySettings +		 +

Definition of this structured property including its name

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## StructuredPropertyParams + +A prompt shown to the user to collect metadata about an entity + +

Fields

+ + + + + + + + + +
NameDescription
+structuredProperty
+StructuredPropertyEntity! +		 +

The structured property required for the prompt on this entity

+
+ +## StructuredPropertySettings + +Settings specific to a structured property entity + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+isHidden
+Boolean! +		 +

Whether or not this asset should be hidden in the main application

+
+showInSearchFilters
+Boolean! +		 +

Whether or not this asset should be displayed as a search filter

+
+showInAssetSummary
+Boolean! +		 +

Whether or not this asset should be displayed in the asset sidebar

+
+hideInAssetSummaryWhenEmpty
+Boolean! +		 +

Whether or not this asset should be hidden in the asset sidebar (showInAssetSummary should be enabled) +when its value is empty

+
+showAsAssetBadge
+Boolean! +		 +

Whether or not this asset should be displayed as an asset badge on other asset's headers

+
+showInColumnsTable
+Boolean! +		 +

Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab.

+
+ +## StructuredReport + +A flexible carrier for structured results of an execution request. + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+type
+String! +		 +

The type of the structured report. (e.g. INGESTION_REPORT, TEST_CONNECTION_REPORT, etc.)

+
+serializedValue
+String! +		 +

The serialized value of the structured report

+
+contentType
+String! +		 +

The content-type of the serialized value (e.g. application/json, application/json;gzip etc.)

+
+ +## SubTypes + + + +

Fields

+ + + + + + + + + +
NameDescription
+typeNames
+[String!] +		 +

The sub-types that this entity implements. e.g. Datasets that are views will implement the "view" subtype

+
+ +## SummaryElement + +Info for a given asset summary element + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+elementType
+SummaryElementType! +		 +

The type of element/property

+
+structuredProperty
+StructuredPropertyEntity +		 +

The structured property associated with this summary element if it is a STRUCTURED_PROPERTY type

+
+ +## SystemFreshness + + + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+systemName
+String! +		 +

Name of the system

+
+freshnessMillis
+Long! +		 +

The latest timestamp in millis of the system that was used to respond to this query +In case a cache was consulted, this reflects the freshness of the cache +In case an index was consulted, this reflects the freshness of the index

+
+ +## TableChart + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+title
+String! +		 + +
+columns
+[String!]! +		 + +
+rows
+[Row!]! +		 + +
+ +## TableSchema + +Information about a raw Table Schema + +

Fields

+ + + + + + + + + +
NameDescription
+schema
+String! +		 +

Raw table schema

+
+ +## Tag + +A Tag Entity, which can be associated with other Metadata Entities and subresources + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the TAG

+
+type
+EntityType! +		 +

A standard Entity Type

+
+name
+String! +		 +
Deprecated: No longer supported
+ +

A unique identifier for the Tag. Deprecated - Use properties.name field instead.

+
+properties
+TagProperties +		 +

Additional properties about the Tag

+
+editableProperties
+EditableTagProperties +		 +
Deprecated: No longer supported
+ +

Additional read write properties about the Tag +Deprecated! Use 'properties' field instead.

+
+ownership
+Ownership +		 +

Ownership metadata of the dataset

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+description
+String +		 +
Deprecated: No longer supported
+ +

Deprecated, use properties.description field instead

+
+aspects
+[RawAspect!] +		 +

Experimental API. +For fetching extra entities that do not have custom UI code yet

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AspectParams +		 + +
+ +
+ +## TagAssociation + +An edge between a Metadata Entity and a Tag Modeled as a struct to permit +additional attributes +TODO Consider whether this query should be serviced by the relationships field + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+tag
+Tag! +		 +

The tag itself

+
+associatedUrn
+String! +		 +

Reference back to the tagged urn for tracking purposes e.g. when sibling nodes are merged together

+
+context
+String +		 +

The context of how/why this tag is associated

+
+attribution
+MetadataAttribution +		 +

Information about who, why, and how this metadata was applied

+
+ +## TagProperties + +Properties for a DataHub Tag + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+name
+String! +		 +

A display name for the Tag

+
+description
+String +		 +

A description of the Tag

+
+colorHex
+String +		 +

An optional RGB hex code for a Tag color, e.g. #FFFFFF

+
+ +## TelemetryConfig + +Configurations related to tracking users in the app + +

Fields

+ + + + + + + + + +
NameDescription
+enableThirdPartyLogging
+Boolean +		 +

Env variable for whether or not third party logging should be enabled for this instance

+
+ +## Test + +A metadata entity representing a DataHub Test + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Test itself

+
+type
+EntityType! +		 +

The standard Entity Type

+
+name
+String! +		 +

The name of the Test

+
+category
+String! +		 +

The category of the Test (user defined)

+
+description
+String +		 +

Description of the test

+
+definition
+TestDefinition! +		 +

Definition for the test

+
+relationships
+EntityRelationshipsResult +		 +

Unused for tests

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## TestDefinition + +Definition of the test + +

Fields

+ + + + + + + + + +
NameDescription
+json
+String +		 +

JSON-based def for the test

+
+ +## TestResult + +The result of running a test + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+test
+Test +		 +

The test itself, or null if the test has been deleted

+
+type
+TestResultType! +		 +

The final result, e.g. either SUCCESS or FAILURE.

+
+ +## TestResults + +A set of test results + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+passing
+[TestResult!]! +		 +

The tests passing

+
+failing
+[TestResult!]! +		 +

The tests failing

+
+ +## TestsConfig + +Configurations related to DataHub Tests feature + +

Fields

+ + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether Tests feature is enabled

+
+ +## TextCell + +A Notebook cell which contains text as content + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+cellTitle
+String! +		 +

Title of the cell

+
+cellId
+String! +		 +

Unique id for the cell.

+
+changeAuditStamps
+ChangeAuditStamps +		 +

Captures information about who created/last modified/deleted this TextCell and when

+
+text
+String! +		 +

The actual text in a TextCell in a Notebook

+
+ +## ThemeConfig + +Configuration for any custom theme-ing + +

Fields

+ + + + + + + + + +
NameDescription
+themeId
+String +		 +

The optional custom theme ID to determine which theme config we use in the frontend

+
+ +## TimelineParameterEntry + +A timeline parameter entry + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+key
+String +		 +

The key of the parameter

+
+value
+String +		 +

The value of the parameter

+
+ +## TimeseriesCapabilitiesResult + +A set of capabilities regarding our timerseries indices + +

Fields

+ + + + + + + + + +
NameDescription
+assetStats
+AssetStatsResult +		 +

Information regarding asset stats

+
+ +## TimeSeriesChart + +For consumption by UI only + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+title
+String! +		 + +
+lines
+[NamedLine!]! +		 + +
+dateRange
+DateRange! +		 + +
+interval
+DateInterval! +		 + +
+ +## TimeWindow + +A time window with a finite start and end time + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+startTimeMillis
+Long! +		 +

The start time of the time window

+
+durationMillis
+Long! +		 +

The end time of the time window

+
+ +## TypeQualifier + +Allows for type specialization of the valueType to be more specific about which +entity types are allowed, for example. + +

Fields

+ + + + + + + + + +
NameDescription
+allowedTypes
+[EntityTypeEntity!] +		 +

The list of allowed entity types

+
+ +## UpdateStepStateResult + +Result returned when fetching step state + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+id
+String! +		 +

Id of the step

+
+succeeded
+Boolean! +		 +

Whether the update succeeded.

+
+ +## UpstreamEntityRelationships + +Deprecated, use relationships query instead + +

Fields

+ + + + + + + + + +
NameDescription
+entities
+[EntityRelationshipLegacy] +		 + +
+ +## UsageAggregation + +An aggregation of Dataset usage statistics + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+bucket
+Long +		 +

The time window start time

+
+duration
+WindowDuration +		 +

The time window span

+
+resource
+String +		 +

The resource urn associated with the usage information, eg a Dataset urn

+
+metrics
+UsageAggregationMetrics +		 +

The rolled up usage metrics

+
+ +## UsageAggregationMetrics + +Rolled up metrics about Dataset usage over time + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+uniqueUserCount
+Int +		 +

The unique number of users who have queried the dataset within the time range

+
+users
+[UserUsageCounts] +		 +

Usage statistics within the time range by user

+
+totalSqlQueries
+Int +		 +

The total number of queries issued against the dataset within the time range

+
+topSqlQueries
+[String] +		 +

A set of common queries issued against the dataset within the time range

+
+fields
+[FieldUsageCounts] +		 +

Per field usage statistics within the time range

+
+ +## UsageQueryResult + +The result of a Dataset usage query + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+buckets
+[UsageAggregation] +		 +

A set of relevant time windows for use in displaying usage statistics

+
+aggregations
+UsageQueryResultAggregations +		 +

A set of rolled up aggregations about the Dataset usage

+
+ +## UsageQueryResultAggregations + +A set of rolled up aggregations about the Dataset usage + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+uniqueUserCount
+Int +		 +

The count of unique Dataset users within the queried time range

+
+users
+[UserUsageCounts] +		 +

The specific per user usage counts within the queried time range

+
+fields
+[FieldUsageCounts] +		 +

The specific per field usage counts within the queried time range

+
+totalSqlQueries
+Int +		 +

The total number of queries executed within the queried time range +Note that this field will likely be deprecated in favor of a totalQueries field

+
+ +## UserUsageCounts + +Information about individual user usage of a Dataset + +

Fields

+ + + + + + + + + + + + + + + + + +
NameDescription
+user
+CorpUser +		 +

The user of the Dataset

+
+count
+Int +		 +

The number of queries issued by the user

+
+userEmail
+String +		 +

The extracted user email +Note that this field will soon be deprecated and merged with user

+
+ +## ValueFrequency + +A frequency distribution of a specific value within a dataset + +

Fields

+ + + + + + + + + + + + + +
NameDescription
+value
+String! +		 +

Specific value. For numeric columns, the value will contain a stringified value

+
+frequency
+Long! +		 +

Volume of the value

+
+ +## VersionedDataset + +A Dataset entity, which encompasses Relational Tables, Document store collections, streaming topics, and other sets of data having an independent lifecycle + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the Dataset

+
+type
+EntityType! +		 +

The standard Entity Type

+
+platform
+DataPlatform! +		 +

Standardized platform urn where the dataset is defined

+
+container
+Container +		 +

The parent container in which the entity resides

+
+parentContainers
+ParentContainersResult +		 +

Recursively get the lineage of containers for this entity

+
+name
+String! +		 +

Unique guid for dataset +No longer to be used as the Dataset display name. Use properties.name instead

+
+properties
+DatasetProperties +		 +

An additional set of read only properties

+
+editableProperties
+DatasetEditableProperties +		 +

An additional set of of read write properties

+
+ownership
+Ownership +		 +

Ownership metadata of the dataset

+
+deprecation
+Deprecation +		 +

The deprecation status of the dataset

+
+institutionalMemory
+InstitutionalMemory +		 +

References to internal resources related to the dataset

+
+editableSchemaMetadata
+EditableSchemaMetadata +		 +

Editable schema metadata of the dataset

+
+status
+Status +		 +

Status of the Dataset

+
+tags
+GlobalTags +		 +

Tags used for searching dataset

+
+glossaryTerms
+GlossaryTerms +		 +

The structured glossary terms associated with the dataset

+
+domain
+DomainAssociation +		 +

The Domain associated with the Dataset

+
+applications
+[ApplicationAssociation!] +		 +

The applications associated with the entity

+
+application
+ApplicationAssociation +		 +
Deprecated: Use applications instead
+ +

Deprecated, use applications instead +The application associated with the entity

+
+health
+[Health!] +		 +

Experimental! The resolved health status of the asset

+
+schema
+Schema +		 +

Schema metadata of the dataset

+
+subTypes
+SubTypes +		 +

Sub Types that this entity implements

+
+viewProperties
+ViewProperties +		 +

View related properties. Only relevant if subtypes field contains view.

+
+origin
+FabricType! +		 +
Deprecated: No longer supported
+ +

Deprecated, see the properties field instead +Environment in which the dataset belongs to or where it was generated +Note that this field will soon be deprecated in favor of a more standardized concept of Environment

+
+relationships
+EntityRelationshipsResult +		 +

No-op, has to be included due to model

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+ +## VersionProperties + + + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+versionSet
+VersionSet! +		 +

The linked Version Set entity that ties multiple versioned assets together

+
+version
+VersionTag! +		 +

Label for this versioned asset, should be unique within a version set (not enforced)

+
+aliases
+[VersionTag!]! +		 +

Additional version identifiers for this versioned asset.

+
+comment
+String +		 +

Comment documenting what this version was created for, changes, or represents

+
+isLatest
+Boolean! +		 +

Whether this version is currently the latest in its verison set

+
+created
+ResolvedAuditStamp +		 +

Timestamp reflecting when the metadata for this version was created in DataHub

+
+createdInSource
+ResolvedAuditStamp +		 +

Timestamp reflecting when the metadata for this version was created in DataHub

+
+ +## VersionSet + + + +

Implements

+ +- [Entity](/docs/graphql/interfaces#entity) + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 +

The primary key of the VersionSet

+
+type
+EntityType! +		 +

The standard Entity Type

+
+relationships
+EntityRelationshipsResult +		 +

Granular API for querying edges extending from this entity

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+RelationshipsInput! +		 + +
+ +
+latestVersion
+Entity +		 +

The latest versioned entity linked to in this version set

+
+versionsSearch
+SearchResults +		 +

Executes a search on all versioned entities linked to this version set +By default sorts by sortId in descending order

+ +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchAcrossEntitiesInput! +		 + +
+ +
+ +## VersionTag + +The technical version associated with a given Metadata Entity + +

Fields

+ + + + + + + + + +
NameDescription
+versionTag
+String +		 + +
+ +## ViewProperties + +Properties about a Dataset of type view + +

Fields

+ + + + + + + + + + + + + + + + + + + + + +
NameDescription
+materialized
+Boolean! +		 +

Whether the view is materialized or not

+
+logic
+String! +		 +

The logic associated with the view, most commonly a SQL statement

+
+formattedLogic
+String +		 +

A formatted version of the logic associated with the view. +For dbt, this contains the compiled SQL.

+
+language
+String! +		 +

The language in which the view logic is written, for example SQL

+
+ +## ViewsConfig + +Configurations related to DataHub Views feature + +

Fields

+ + + + + + + + + +
NameDescription
+enabled
+Boolean! +		 +

Whether Views feature is enabled

+
+ +## VisualConfig + +Configurations related to visual appearance of the app + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+logoUrl
+String +		 +

Custom logo url for the homepage & top banner

+
+faviconUrl
+String +		 +

Custom favicon url for the homepage & top banner

+
+appTitle
+String +		 +

Custom app title to show in the browser tab

+
+hideGlossary
+Boolean +		 +

Boolean flag disabling viewing the Business Glossary page for users without the 'Manage Glossaries' privilege

+
+queriesTab
+QueriesTabConfig +		 +

Configuration for the queries tab

+
+entityProfiles
+EntityProfilesConfig +		 +

Configuration for the queries tab

+
+searchResult
+SearchResultsVisualConfig +		 +

Configuration for search results

+
+showFullTitleInLineage
+Boolean +		 +

Show full title in lineage view by default

+
+theme
+ThemeConfig +		 +

Configuration for custom theme-ing

+
+application
+ApplicationConfig +		 +

Configuration for the application sidebar section

+
+ +## VolumeAssertionInfo + +A definition of a Volume (row count) assertion. + +

Fields

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDescription
+entityUrn
+String! +		 +

The entity targeted by this Volume check.

+
+type
+VolumeAssertionType! +		 +

The type of the freshness assertion being monitored.

+
+rowCountTotal
+RowCountTotal +		 +

Produce FAILURE Assertion Result if the row count of the asset does not meet specific requirements. +Required if type is 'ROW_COUNT_TOTAL'.

+
+rowCountChange
+RowCountChange +		 +

Produce FAILURE Assertion Result if the row count delta of the asset does not meet specific requirements. +Required if type is 'ROW_COUNT_CHANGE'.

+
+incrementingSegmentRowCountTotal
+IncrementingSegmentRowCountTotal +		 +

Produce FAILURE Assertion Result if the latest incrementing segment row count total of the asset +does not meet specific requirements. Required if type is 'INCREMENTING_SEGMENT_ROW_COUNT_TOTAL'.

+
+incrementingSegmentRowCountChange
+IncrementingSegmentRowCountChange +		 +

Produce FAILURE Assertion Result if the incrementing segment row count delta of the asset +does not meet specific requirements. Required if type is 'INCREMENTING_SEGMENT_ROW_COUNT_CHANGE'.

+
+filter
+DatasetFilter +		 +

A definition of the specific filters that should be applied, when performing monitoring. +If not provided, there is no filter, and the full table is under consideration.

+
+ diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/queries.md b/docs-archive/versioned_docs/version-1.5.0/graphql/queries.md new file mode 100644 index 00000000..aac856c1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/queries.md @@ -0,0 +1,2259 @@ +--- +id: queries +title: Queries +slug: queries +sidebar_position: 1 +--- + +## aggregateAcrossEntities + +**Type:** [AggregateResults](/docs/graphql/objects#aggregateresults) + +Aggregate across DataHub entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AggregateAcrossEntitiesInput! +		 + +
+ +## appConfig + +**Type:** [AppConfig](/docs/graphql/objects#appconfig) + +Fetch configurations +Used by DataHub UI + +## application + +**Type:** [Application](/docs/graphql/objects#application) + +Fetch an Application by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## assertion + +**Type:** [Assertion](/docs/graphql/objects#assertion) + +Fetch an Assertion by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## autoComplete + +**Type:** [AutoCompleteResults](/docs/graphql/objects#autocompleteresults) + +Autocomplete a search query against a specific DataHub Entity Type + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AutoCompleteInput! +		 + +
+ +## autoCompleteForMultiple + +**Type:** [AutoCompleteMultipleResults](/docs/graphql/objects#autocompletemultipleresults) + +Autocomplete a search query against a specific set of DataHub Entity Types + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+AutoCompleteMultipleInput! +		 + +
+ +## batchGetStepStates + +**Type:** [BatchGetStepStatesResult!](/docs/graphql/objects#batchgetstepstatesresult) + +Batch fetch the state for a set of steps. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BatchGetStepStatesInput! +		 + +
+ +## browse + +**Type:** [BrowseResults](/docs/graphql/objects#browseresults) + +Hierarchically browse a specific type of DataHub Entity by path +Used by explore in the UI + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BrowseInput! +		 + +
+ +## browsePaths + +**Type:** [[BrowsePath!]](/docs/graphql/objects#browsepath) + +Retrieve the browse paths corresponding to an entity + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BrowsePathsInput! +		 + +
+ +## browseV2 + +**Type:** [BrowseResultsV2](/docs/graphql/objects#browseresultsv2) + +Browse for different entities by getting organizational groups and their +aggregated counts + content. Uses browsePathsV2 aspect and replaces our old +browse endpoint. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+BrowseV2Input! +		 + +
+ +## businessAttribute + +**Type:** [BusinessAttribute](/docs/graphql/objects#businessattribute) + +Fetch a Business Attribute by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## chart + +**Type:** [Chart](/docs/graphql/objects#chart) + +Fetch a Chart by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## connection + +**Type:** [DataHubConnection](/docs/graphql/objects#datahubconnection) + +Get a set of connection details by URN. +This requires the 'Manage Connections' platform privilege. +Returns null if a connection with the provided urn does not exist. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## container + +**Type:** [Container](/docs/graphql/objects#container) + +Fetch an Entity Container by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## corpGroup + +**Type:** [CorpGroup](/docs/graphql/objects#corpgroup) + +Fetch a CorpGroup, representing a DataHub platform group by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## corpUser + +**Type:** [CorpUser](/docs/graphql/objects#corpuser) + +Fetch a CorpUser, representing a DataHub platform user, by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dashboard + +**Type:** [Dashboard](/docs/graphql/objects#dashboard) + +Fetch a Dashboard by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dataFlow + +**Type:** [DataFlow](/docs/graphql/objects#dataflow) + +Fetch a Data Flow (or Data Pipeline) by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dataJob + +**Type:** [DataJob](/docs/graphql/objects#datajob) + +Fetch a Data Job (or Data Task) by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dataPlatform + +**Type:** [DataPlatform](/docs/graphql/objects#dataplatform) + +Fetch a Data Platform by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dataPlatformInstance + +**Type:** [DataPlatformInstance](/docs/graphql/objects#dataplatforminstance) + +Fetch a Data Platform Instance by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dataProcessInstance + +**Type:** [DataProcessInstance](/docs/graphql/objects#dataprocessinstance) + +Fetch a Data Process Instance by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dataProduct + +**Type:** [DataProduct](/docs/graphql/objects#dataproduct) + +Fetch a DataProduct by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## dataset + +**Type:** [Dataset](/docs/graphql/objects#dataset) + +Fetch a Dataset by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## debugAccess + +**Type:** [DebugAccessResult!](/docs/graphql/objects#debugaccessresult) + +Experimental API to debug Access for users. +Backward incompatible changes will be made without notice in the future. +Do not build on top of this API. + +

Arguments

+ + + + + + + + + +
NameDescription
+userUrn
+String! +		 + +
+ +## docPropagationSettings + +**Type:** [DocPropagationSettings](/docs/graphql/objects#docpropagationsettings) + +Fetch the global settings related to the docs propagation feature. + +## document + +**Type:** [Document](/docs/graphql/objects#document) + +Get a Document by URN. +Requires the GET_ENTITY privilege for the document or MANAGE_DOCUMENTS platform privilege. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## domain + +**Type:** [Domain](/docs/graphql/objects#domain) + +Fetch a Domain by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## entities + +**Type:** [[Entity]](/docs/graphql/interfaces#entity) + +Gets entities based on their urns +checkForExistence will do n+1 sql calls to check for existence of each entity requested if set to true + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urns
+[String!]! +		 + +
+checkForExistence
+Boolean +		 + +
+ +## entity + +**Type:** [Entity](/docs/graphql/interfaces#entity) + +Gets an entity based on its urn + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## entityExists + +**Type:** [Boolean](/docs/graphql/scalars#boolean) + +Get whether or not not an entity exists + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## erModelRelationship + +**Type:** [ERModelRelationship](/docs/graphql/objects#ermodelrelationship) + +Fetch a ERModelRelationship by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## executionRequest + +**Type:** [ExecutionRequest](/docs/graphql/objects#executionrequest) + +Get an execution request +urn: The primary key associated with the execution request. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## form + +**Type:** [Form](/docs/graphql/objects#form) + +Fetch a Form by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## getAccessToken + +**Type:** [AccessToken](/docs/graphql/objects#accesstoken) + +Generates an access token for DataHub APIs for a particular user & of a particular type +Deprecated, use createAccessToken instead + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetAccessTokenInput! +		 + +
+ +## getAccessTokenMetadata + +**Type:** [AccessTokenMetadata!](/docs/graphql/objects#accesstokenmetadata) + +Fetches the metadata of an access token. +This is useful to debug when you have a raw token but don't know the actor. + +

Arguments

+ + + + + + + + + +
NameDescription
+token
+String! +		 + +
+ +## getAnalyticsCharts + +**Type:** [[AnalyticsChartGroup!]!](/docs/graphql/objects#analyticschartgroup) + +Retrieves a set of server driven Analytics Charts to render in the UI + +## getEntityCounts + +**Type:** [EntityCountResults](/docs/graphql/objects#entitycountresults) + +Fetches the number of entities ingested by type + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+EntityCountInput +		 + +
+ +## getGrantedPrivileges + +**Type:** [Privileges](/docs/graphql/objects#privileges) + +Get all granted privileges for the given actor and resource + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetGrantedPrivilegesInput! +		 + +
+ +## getHighlights + +**Type:** [[Highlight!]!](/docs/graphql/objects#highlight) + +Retrieves a set of server driven Analytics Highlight Cards to render in the UI + +## getInviteToken + +**Type:** [InviteToken](/docs/graphql/objects#invitetoken) + +Get invite token + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetInviteTokenInput! +		 + +
+ +## getMetadataAnalyticsCharts + +**Type:** [[AnalyticsChartGroup!]!](/docs/graphql/objects#analyticschartgroup) + +Retrieves a set of charts regarding the ingested metadata + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+MetadataAnalyticsInput! +		 + +
+ +## getPresignedUploadUrl + +**Type:** [GetPresignedUploadUrlResponse!](/docs/graphql/objects#getpresigneduploadurlresponse) + +Generates a presigned url to upload files + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetPresignedUploadUrlInput! +		 + +
+ +## getQuickFilters + +**Type:** [GetQuickFiltersResult](/docs/graphql/objects#getquickfiltersresult) + +Get quick filters to display in auto-complete + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetQuickFiltersInput! +		 + +
+ +## getRootGlossaryNodes + +**Type:** [GetRootGlossaryNodesResult](/docs/graphql/objects#getrootglossarynodesresult) + +Get all GlossaryNodes without a parentNode + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetRootGlossaryEntitiesInput! +		 + +
+ +## getRootGlossaryTerms + +**Type:** [GetRootGlossaryTermsResult](/docs/graphql/objects#getrootglossarytermsresult) + +Get all GlossaryTerms without a parentNode + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetRootGlossaryEntitiesInput! +		 + +
+ +## getSchemaBlame + +**Type:** [GetSchemaBlameResult](/docs/graphql/objects#getschemablameresult) + +Returns the most recent changes made to each column in a dataset at each dataset version. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetSchemaBlameInput! +		 + +
+ +## getSchemaVersionList + +**Type:** [GetSchemaVersionListResult](/docs/graphql/objects#getschemaversionlistresult) + +Returns the list of schema versions for a dataset. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetSchemaVersionListInput! +		 + +
+ +## getSecretValues + +**Type:** [[SecretValue!]](/docs/graphql/objects#secretvalue) + +Fetch the values of a set of secrets. The caller must have the MANAGE_SECRETS +privilege to use. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetSecretValuesInput! +		 + +
+ +## getServiceAccount + +**Type:** [ServiceAccount](/docs/graphql/objects#serviceaccount) + +Get a service account by URN. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## getTimeline + +**Type:** [GetTimelineResult](/docs/graphql/objects#gettimelineresult) + +Returns a list of timeline actions for an entity based on the filter criteria + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+GetTimelineInput! +		 + +
+ +## globalHomePageSettings + +**Type:** [GlobalHomePageSettings](/docs/graphql/objects#globalhomepagesettings) + +Global settings related to the home page for an instance + +## globalViewsSettings + +**Type:** [GlobalViewsSettings](/docs/graphql/objects#globalviewssettings) + +Fetch the Global Settings related to the Views feature. +Requires the 'Manage Global Views' Platform Privilege. + +## glossaryNode + +**Type:** [GlossaryNode](/docs/graphql/objects#glossarynode) + +Fetch a Glossary Node by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## glossaryTerm + +**Type:** [GlossaryTerm](/docs/graphql/objects#glossaryterm) + +Fetch a Glossary Term by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## ingestionSource + +**Type:** [IngestionSource](/docs/graphql/objects#ingestionsource) + +Fetch a specific ingestion source +urn: The primary key associated with the ingestion source. + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## isAnalyticsEnabled + +**Type:** [Boolean!](/docs/graphql/scalars#boolean) + +Deprecated, use appConfig Query instead +Whether the analytics feature is enabled in the UI + +## latestProductUpdate + +**Type:** [ProductUpdate](/docs/graphql/objects#productupdate) + +Fetch the latest product update information from configured JSON source. +Returns null if the JSON source is unavailable or feature is disabled. + +

Arguments

+ + + + + + + + + +
NameDescription
+refreshCache
+Boolean +		 + +
+ +## listAccessTokens + +**Type:** [ListAccessTokenResult!](/docs/graphql/objects#listaccesstokenresult) + +List access tokens stored in DataHub. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListAccessTokenInput! +		 + +
+ +## listApplicationAssets + +**Type:** [SearchResults](/docs/graphql/objects#searchresults) + +List Application assets for a given urn + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+SearchAcrossEntitiesInput! +		 + +
+ +## listBusinessAttributes + +**Type:** [ListBusinessAttributesResult](/docs/graphql/objects#listbusinessattributesresult) + +Fetch all Business Attributes + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListBusinessAttributesInput! +		 + +
+ +## listDataProductAssets + +**Type:** [SearchResults](/docs/graphql/objects#searchresults) + +List Data Product assets for a given urn + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+input
+SearchAcrossEntitiesInput! +		 + +
+ +## listDomains + +**Type:** [ListDomainsResult](/docs/graphql/objects#listdomainsresult) + +Lists all DataHub Domains belonging to the specified parent. If no parent is specified, lists root domains. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListDomainsInput! +		 + +
+ +## listExecutionRequests + +**Type:** [IngestionSourceExecutionRequests](/docs/graphql/objects#ingestionsourceexecutionrequests) + +List all execution requests + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListExecutionRequestsInput! +		 + +
+ +## listGlobalViews + +**Type:** [ListViewsResult](/docs/graphql/objects#listviewsresult) + +List Global DataHub Views + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListGlobalViewsInput! +		 + +
+ +## listGroups + +**Type:** [ListGroupsResult](/docs/graphql/objects#listgroupsresult) + +List all DataHub Groups + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListGroupsInput! +		 + +
+ +## listIngestionSources + +**Type:** [ListIngestionSourcesResult](/docs/graphql/objects#listingestionsourcesresult) + +List all ingestion sources + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListIngestionSourcesInput! +		 + +
+ +## listMyViews + +**Type:** [ListViewsResult](/docs/graphql/objects#listviewsresult) + +List DataHub Views owned by the current user + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListMyViewsInput! +		 + +
+ +## listOwnershipTypes + +**Type:** [ListOwnershipTypesResult!](/docs/graphql/objects#listownershiptypesresult) + +List Custom Ownership Types + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListOwnershipTypesInput! +		 +

Input required for listing custom ownership types

+
+ +## listPolicies + +**Type:** [ListPoliciesResult](/docs/graphql/objects#listpoliciesresult) + +List all DataHub Access Policies + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListPoliciesInput! +		 + +
+ +## listPosts + +**Type:** [ListPostsResult](/docs/graphql/objects#listpostsresult) + +List all Posts + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListPostsInput! +		 + +
+ +## listQueries + +**Type:** [ListQueriesResult](/docs/graphql/objects#listqueriesresult) + +List Dataset Queries + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListQueriesInput! +		 +

Input required for listing queries

+
+ +## listRecommendations + +**Type:** [ListRecommendationsResult](/docs/graphql/objects#listrecommendationsresult) + +Fetch recommendations for a particular scenario + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListRecommendationsInput! +		 + +
+ +## listRoles + +**Type:** [ListRolesResult](/docs/graphql/objects#listrolesresult) + +List all DataHub Roles + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListRolesInput! +		 + +
+ +## listSecrets + +**Type:** [ListSecretsResult](/docs/graphql/objects#listsecretsresult) + +List all secrets stored in DataHub (no values) + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListSecretsInput! +		 + +
+ +## listServiceAccounts + +**Type:** [ListServiceAccountsResult!](/docs/graphql/objects#listserviceaccountsresult) + +List service accounts stored in DataHub. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListServiceAccountsInput! +		 + +
+ +## listTests + +**Type:** [ListTestsResult](/docs/graphql/objects#listtestsresult) + +List all DataHub Tests + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListTestsInput! +		 + +
+ +## listUsers + +**Type:** [ListUsersResult](/docs/graphql/objects#listusersresult) + +List all DataHub Users + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ListUsersInput! +		 + +
+ +## me + +**Type:** [AuthenticatedUser](/docs/graphql/objects#authenticateduser) + +Fetch details associated with the authenticated user, provided via an auth cookie or header + +## mlFeature + +**Type:** [MLFeature](/docs/graphql/objects#mlfeature) + +Incubating: Fetch a ML Feature by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## mlFeatureTable + +**Type:** [MLFeatureTable](/docs/graphql/objects#mlfeaturetable) + +Incubating: Fetch a ML Feature Table by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## mlModel + +**Type:** [MLModel](/docs/graphql/objects#mlmodel) + +Incubating: Fetch an ML Model by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## mlModelGroup + +**Type:** [MLModelGroup](/docs/graphql/objects#mlmodelgroup) + +Incubating: Fetch an ML Model Group by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## mlPrimaryKey + +**Type:** [MLPrimaryKey](/docs/graphql/objects#mlprimarykey) + +Incubating: Fetch a ML Primary Key by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## notebook + +**Type:** [Notebook](/docs/graphql/objects#notebook) + +Fetch a Notebook by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## role + +**Type:** [Role](/docs/graphql/objects#role) + +Fetch a Role by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## scrollAcrossEntities + +**Type:** [ScrollResults](/docs/graphql/objects#scrollresults) + +Search DataHub entities by providing a pointer reference for scrolling through results. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ScrollAcrossEntitiesInput! +		 + +
+ +## scrollAcrossLineage + +**Type:** [ScrollAcrossLineageResults](/docs/graphql/objects#scrollacrosslineageresults) + +Search across the results of a graph query on a node, uses scroll API + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+ScrollAcrossLineageInput! +		 + +
+ +## search + +**Type:** [SearchResults](/docs/graphql/objects#searchresults) + +Full text search against a specific DataHub Entity Type + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchInput! +		 + +
+ +## searchAcrossEntities + +**Type:** [SearchResults](/docs/graphql/objects#searchresults) + +Search DataHub entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchAcrossEntitiesInput! +		 + +
+ +## searchAcrossLineage + +**Type:** [SearchAcrossLineageResults](/docs/graphql/objects#searchacrosslineageresults) + +Search across the results of a graph query on a node + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchAcrossLineageInput! +		 + +
+ +## searchDocuments + +**Type:** [SearchDocumentsResult!](/docs/graphql/objects#searchdocumentsresult) + +Search Documents with hybrid semantic search and filtering support. +Supports filtering by parent document, types, domains, and query. + +Results are automatically filtered based on document status: +- PUBLISHED documents are shown to all users +- UNPUBLISHED documents are only shown if owned by the current calling user or a group they belong to + +This ensures that unpublished documents remain private to their owners while published +documents are discoverable by everyone. + +Additionally, this API does NOT return documents that have should not be shown in "global" context via +their document settings. + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchDocumentsInput! +		 + +
+ +## searchUsers + +**Type:** [SearchResults](/docs/graphql/objects#searchresults) + +Search DataHub Users + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchAcrossEntitiesInput! +		 + +
+ +## semanticSearch + +**Type:** [SearchResults](/docs/graphql/objects#searchresults) + +Full text semantic search against a specific DataHub Entity Type + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchInput! +		 + +
+ +## semanticSearchAcrossEntities + +**Type:** [SearchResults](/docs/graphql/objects#searchresults) + +Semantic search across DataHub entities + +

Arguments

+ + + + + + + + + +
NameDescription
+input
+SearchAcrossEntitiesInput! +		 + +
+ +## structuredProperty + +**Type:** [StructuredPropertyEntity](/docs/graphql/objects#structuredpropertyentity) + +Fetch a Structured Property by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## tag + +**Type:** [Tag](/docs/graphql/objects#tag) + +Fetch a Tag by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## test + +**Type:** [Test](/docs/graphql/objects#test) + +Fetch a Test by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## versionedDataset + +**Type:** [VersionedDataset](/docs/graphql/objects#versioneddataset) + +Fetch a Dataset by primary key (urn) at a point in time based on aspect versions (versionStamp) + +

Arguments

+ + + + + + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+versionStamp
+String +		 + +
+ +## versionSet + +**Type:** [VersionSet](/docs/graphql/objects#versionset) + +Fetch a Version Set by its URN + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ +## view + +**Type:** [DataHubView](/docs/graphql/objects#datahubview) + +Fetch a View by primary key (urn) + +

Arguments

+ + + + + + + + + +
NameDescription
+urn
+String! +		 + +
+ diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/scalars.md b/docs-archive/versioned_docs/version-1.5.0/graphql/scalars.md new file mode 100644 index 00000000..cbaf9991 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/scalars.md @@ -0,0 +1,27 @@ +--- +id: scalars +title: Scalars +slug: scalars +sidebar_position: 8 +--- + +## Boolean + +The `Boolean` scalar type represents `true` or `false`. + +## Float + +The `Float` scalar type represents signed double-precision fractional values as specified by [IEEE 754](https://en.wikipedia.org/wiki/IEEE_floating_point). + +## Int + +The `Int` scalar type represents non-fractional signed whole numeric values. Int can represent values between -(2^31) and 2^31 - 1. + +## Long + + + +## String + +The `String` scalar type represents textual data, represented as UTF-8 character sequences. The String type is most often used by GraphQL to represent free-form human-readable text. + diff --git a/docs-archive/versioned_docs/version-1.5.0/graphql/unions.md b/docs-archive/versioned_docs/version-1.5.0/graphql/unions.md new file mode 100644 index 00000000..adf65ba6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/graphql/unions.md @@ -0,0 +1,72 @@ +--- +id: unions +title: Unions +slug: unions +sidebar_position: 6 +--- + +## AnalyticsChart + +For consumption by UI only + +

Possible types

+ +- [TimeSeriesChart](/docs/graphql/objects#timeserieschart) +- [BarChart](/docs/graphql/objects#barchart) +- [TableChart](/docs/graphql/objects#tablechart) + +## HyperParameterValueType + + + +

Possible types

+ +- [StringBox](/docs/graphql/objects#stringbox) +- [IntBox](/docs/graphql/objects#intbox) +- [FloatBox](/docs/graphql/objects#floatbox) +- [BooleanBox](/docs/graphql/objects#booleanbox) + +## OwnerType + +An owner of a Metadata Entity, either a user or group + +

Possible types

+ +- [CorpUser](/docs/graphql/objects#corpuser) +- [CorpGroup](/docs/graphql/objects#corpgroup) + +## PlatformSchema + +A type of Schema, either a table schema or a key value schema + +

Possible types

+ +- [TableSchema](/docs/graphql/objects#tableschema) +- [KeyValueSchema](/docs/graphql/objects#keyvalueschema) + +## PropertyValue + +The value of a property + +

Possible types

+ +- [StringValue](/docs/graphql/objects#stringvalue) +- [NumberValue](/docs/graphql/objects#numbervalue) + +## ResolvedActor + + + +

Possible types

+ +- [CorpUser](/docs/graphql/objects#corpuser) +- [CorpGroup](/docs/graphql/objects#corpgroup) + +## ResultsType + + + +

Possible types

+ +- [StringBox](/docs/graphql/objects#stringbox) + diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/AIRFLOW_3_MIGRATION.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/AIRFLOW_3_MIGRATION.md new file mode 100644 index 00000000..9cb02877 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/AIRFLOW_3_MIGRATION.md @@ -0,0 +1,1209 @@ +--- +title: Airflow 3.x Migration Guide +slug: /metadata-ingestion-modules/airflow-plugin/airflow_3_migration +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/AIRFLOW_3_MIGRATION.md +--- +# Airflow 3.x Migration Guide + +This document outlines the changes made to support Apache Airflow 3.x and known limitations. + +## Summary of Major Changes + +Apache Airflow 3.0 introduced significant breaking changes. The DataHub Airflow plugin has been fully updated to support both Airflow 2.4+ and 3.x with backward compatibility. + +### 🏗️ Plugin Architecture: Separate Version-Specific Implementations + +**The plugin now uses separate implementations for Airflow 2.x and 3.x** to achieve clean type safety and maintainability: + +| Component | Airflow 2.x Implementation | Airflow 3.x Implementation | +| ----------------------- | ---------------------------------- | -------------------------------------------------------- | +| **Main Module** | `airflow2/datahub_listener.py` | `airflow3/datahub_listener.py` | +| **Shims/Imports** | `airflow2/_shims.py` | `airflow3/_shims.py` | +| **Lineage Extraction** | Extractor-based (`_extractors.py`) | OpenLineage native (`_airflow3_sql_parser_patch.py`) | +| **OpenLineage Package** | `openlineage-airflow>=1.2.0` | Native provider (`apache-airflow-providers-openlineage`) | +| **Type Checking** | ✅ Clean Airflow 2.x types | ✅ Clean Airflow 3.x types | + +**Version Dispatcher:** The main `datahub_listener.py` automatically imports the correct implementation at runtime: + +```python +from datahub_airflow_plugin._airflow_version_specific import IS_AIRFLOW_3_OR_HIGHER + +if IS_AIRFLOW_3_OR_HIGHER: + from datahub_airflow_plugin.airflow3.datahub_listener import ( + DataHubListener, + get_airflow_plugin_listener, + ) +else: + from datahub_airflow_plugin.airflow2.datahub_listener import ( + DataHubListener, + get_airflow_plugin_listener, + ) +``` + +**Benefits:** + +- ✅ **Type Safety** - Each version can be properly type-checked against its respective Airflow API without conflicts +- ✅ **Maintainability** - No complex version conditionals scattered throughout the code +- ✅ **Clarity** - Clear separation of version-specific logic +- ✅ **Testing** - Each version can be tested independently with its specific type requirements + +**Package Installation:** + +```bash +# For Airflow 2.x (uses standalone openlineage-airflow package) +pip install 'acryl-datahub-airflow-plugin[airflow2]' + +# For Airflow 3.x (uses native OpenLineage provider) +pip install 'acryl-datahub-airflow-plugin[airflow3]' + +# For compatibility across versions (installs base package with native provider) +pip install 'acryl-datahub-airflow-plugin' # Works with both Airflow 2.7+ and 3.x +``` + +### 🎯 Key Architectural Change: Moved Away from Operator-Specific Overrides + +**The most significant change** is how lineage extraction works: + +| Aspect | Airflow 2.x | Airflow 3.x | +| ----------------------- | ------------------------------- | -------------------------------------- | +| **Lineage Extraction** | Operator-specific extractors | Unified SQLParser patch | +| **Customization Point** | Custom extractor per operator | Single SQL parser integration | +| **Column Lineage** | Extractor-dependent | ✅ Consistent across all SQL operators | +| **Maintenance** | Multiple extractors to maintain | Single integration point | + +**In Airflow 2.x**, we used operator-specific extractors: + +```python +# Different extractor for each SQL operator +SnowflakeExtractor, PostgresExtractor, MySQLExtractor, etc. +``` + +**In Airflow 3.x**, we use a **unified SQLParser patch**: + +```python +# Single patch point for ALL SQL operators +SQLParser.generate_openlineage_metadata_from_sql = datahub_enhanced_version +``` + +**Benefits:** + +- ✅ **Better consistency** - All SQL operators use the same lineage extraction logic +- ✅ **Easier maintenance** - One integration point instead of many extractors +- ✅ **Column-level lineage for all** - Works across all SQL operators uniformly +- ✅ **Future-proof** - New SQL operators automatically get DataHub lineage + +### TaskInstance Attribute Changes + +Airflow 3.0 introduced `RuntimeTaskInstance` which has a different structure than Airflow 2.x's `TaskInstance`: + +| Attribute | Airflow 2.x | Airflow 3.0 RuntimeTaskInstance | Status | +| ---------------------- | -------------------------- | ------------------------------- | ------------------------------------- | +| `run_id` | ✅ Database field | ✅ Base class | Available in both | +| `start_date` | ✅ Database field | ✅ RuntimeTI field | Available in both | +| `try_number` | ✅ Database field | ✅ Base class | Available in both | +| `state` | ✅ Database field | ✅ RuntimeTI field | Available in both | +| `task_id` | ✅ Database field | ✅ Base class | Available in both | +| `dag_id` | ✅ Database field | ✅ Base class | Available in both | +| `max_tries` | ✅ Database field | ✅ RuntimeTI field | Available in both | +| `end_date` | ✅ Database field | ✅ RuntimeTI field | Available in both | +| `log_url` | ✅ Property | ✅ RuntimeTI field | Available in both | +| `execution_date` | ✅ Database field | ❌ Renamed | **Renamed to `logical_date`** | +| `duration` | ✅ Database field | ❌ Not available | **Missing - must be calculated** | +| `external_executor_id` | ✅ Database field | ❌ Not available | **Missing in Airflow 3.0** | +| `operator` | ✅ Database field (string) | ⚠️ Different | **Has `task` (BaseOperator) instead** | +| `priority_weight` | ✅ Database field | ❌ Not available | **Missing in Airflow 3.0** | + +**Key Changes:** + +1. **RuntimeTaskInstance is not database-backed** - It's a Pydantic model created at runtime +2. **Minimal base attributes** - Base `TaskInstance` only has: `id`, `task_id`, `dag_id`, `run_id`, `try_number`, `map_index`, `hostname` +3. **execution_date → logical_date** - The familiar `execution_date` was renamed +4. **Missing fields** - Several metadata fields are not available in the runtime context +5. **task vs operator** - Airflow 3.0 provides direct access to the task object, not just its string name + +**Plugin Compatibility:** + +The plugin uses `hasattr()` checks to gracefully handle missing attributes: + +```python +def get_task_instance_attributes(ti: "TaskInstance") -> Dict[str, str]: + attributes = {} + + # Safe attribute access - works in both versions + if hasattr(ti, "run_id"): + attributes["run_id"] = str(ti.run_id) + + # Handle renamed attribute + if hasattr(ti, "execution_date"): + attributes["execution_date"] = str(ti.execution_date) + elif hasattr(ti, "logical_date"): + attributes["logical_date"] = str(ti.logical_date) + + # Handle missing attributes gracefully + if hasattr(ti, "duration"): + attributes["duration"] = str(ti.duration) + + return attributes +``` + +This approach ensures the plugin works correctly with both Airflow 2.x and 3.x task instances. + +**Files Updated:** + +- `src/datahub_airflow_plugin/_airflow_version_specific.py:21-74` - Version-compatible attribute extraction + +### Other Major Changes + +1. **Import Path Updates** - Airflow 3.x reorganized modules under new SDK structure +2. **API Changes** - JWT authentication instead of Basic Auth; v1 API removed +3. **DAG Parameters** - `schedule_interval` → `schedule`; `default_view` removed +4. **Listener Hook Signatures** - Session parameter removed; error parameter added +5. **Database Access Restrictions** - No Variable.get() in hooks (use env vars instead) +6. **Template Rendering** - Skip deepcopy for RuntimeTaskInstance (already rendered) +7. **SubDAG Removal** - SubDAGs completely removed (use TaskGroups) +8. **Log URL Format** - Simplified URL structure and different config key +9. **Threading Support** - ✅ Works in Airflow 3.x (contrary to initial concerns) + +### Compatibility Status + +| Feature | Airflow 2.x | Airflow 3.x | Status | +| ------------------ | ----------- | ----------- | ---------------------- | +| Task lineage | ✅ | ✅ | Fully working | +| Column lineage | ✅ | ✅ | Fully working | +| DAG metadata | ✅ | ✅ | Fully working | +| Execution tracking | ✅ | ✅ | Fully working | +| Threading | ✅ | ✅ | Fully working | +| SubDAG support | ✅ | ❌ | Removed in Airflow 3.x | + +## Detailed Changes + +This section provides in-depth information about each change made for Airflow 3.x compatibility. + +### 1. Import Path Updates + +Airflow 3.x reorganized many modules under a new SDK structure. The plugin now uses conditional imports with fallbacks. + +#### BaseOperator + +```python +# Airflow 3.x (preferred) +from airflow.sdk.bases.operator import BaseOperator + +# Airflow 2.x (fallback) +from airflow.models.baseoperator import BaseOperator +``` + +#### Operator Type Alias + +```python +# Airflow 3.x (preferred) +from airflow.sdk.types import Operator + +# Airflow 2.x (fallback) +from airflow.models.operator import Operator +``` + +#### EmptyOperator + +```python +# Airflow 3.x (preferred) +from airflow.providers.standard.operators.empty import EmptyOperator + +# Airflow 2.x (fallback) +from airflow.operators.empty import EmptyOperator + +# Airflow <2.2 (fallback) +from airflow.operators.dummy import DummyOperator +``` + +#### ExternalTaskSensor + +```python +# Airflow 3.x (preferred) +from airflow.providers.standard.sensors.external_task import ExternalTaskSensor + +# Airflow 2.x (fallback) +from airflow.sensors.external_task import ExternalTaskSensor +``` + +**Files Updated:** + +- `src/datahub_airflow_plugin/airflow2/_shims.py` - Clean Airflow 2.x imports +- `src/datahub_airflow_plugin/airflow3/_shims.py` - Clean Airflow 3.x imports +- `src/datahub_airflow_plugin/_airflow_shims.py` - Pure dispatcher (no logic, just routes to version-specific shims) +- `src/datahub_airflow_plugin/client/airflow_generator.py` - Import from shims + +### 1b. OpenLineage Provider Changes + +Airflow 2.7+ introduced native OpenLineage support, and Airflow 3.x completely removed support for the old `openlineage-airflow` package. The native provider has a different API and doesn't include SQL extractors. + +#### Import Changes + +```python +# Airflow 2.7+ and 3.x (native provider) +from airflow.providers.openlineage.extractors import OperatorLineage as TaskMetadata +from airflow.providers.openlineage.extractors.base import BaseExtractor +from airflow.providers.openlineage.extractors.manager import ExtractorManager + +# Airflow < 2.7 (old openlineage-airflow package) +from openlineage.airflow.extractors import TaskMetadata, BaseExtractor, ExtractorManager +from openlineage.airflow.extractors.sql_extractor import SqlExtractor +``` + +#### API Differences + +The native provider's `ExtractorManager` has a different API: + +| Feature | Old Package (< 2.7) | Native Provider (2.7+) | +| ------------------ | ----------------------------------- | --------------------------------------------- | +| Extractor registry | `self.task_to_extractor.extractors` | `self.extractors` | +| Add extractor | Direct dict assignment | Direct dict assignment or `add_extractor()` | +| SQL extractor | ✅ Included | ❌ Not included | +| Task metadata type | `TaskMetadata` | `OperatorLineage` (aliased as `TaskMetadata`) | + +#### Compatibility Layer + +The plugin implements a compatibility layer that: + +1. Detects which OpenLineage implementation is available +2. Uses the appropriate API for registering extractors +3. Implements custom SQL extraction for Airflow 2.7+ (since native provider doesn't include `SqlExtractor`) + +```python +# Compatibility check in ExtractorManager.__init__ +if hasattr(self, 'task_to_extractor'): + # Old openlineage-airflow (Airflow < 2.7) + extractors_dict = self.task_to_extractor.extractors +else: + # Native provider (Airflow 2.7+) + extractors_dict = self.extractors +``` + +**Impact:** + +- The plugin automatically selects the correct OpenLineage implementation based on Airflow version +- SQL lineage extraction works seamlessly across all supported Airflow versions +- No user action required - the plugin handles version differences internally + +**Files Updated:** + +- `src/datahub_airflow_plugin/_extractors.py` - Conditional OpenLineage imports and API compatibility layer + +**Known Limitations:** + +- The native provider (Airflow 2.7+) doesn't include SQL extractors, so the plugin provides its own implementation +- Some advanced OpenLineage features from the old package may not be available in the native provider + +### 2. DAG Parameter Changes + +#### 2a. Schedule Parameter + +Airflow 3.x removed the `schedule_interval` parameter in favor of `schedule`. + +```python +# Airflow 3.x (required) +DAG("my_dag", schedule=None, ...) + +# Airflow 2.4+ (deprecated but supported) +DAG("my_dag", schedule_interval=None, ...) + +# Airflow <2.4 (only option) +DAG("my_dag", schedule_interval=None, ...) +``` + +**Note:** The `schedule` parameter was introduced in Airflow 2.4.0, so test DAGs use `schedule=` which works in both Airflow 2.4+ and 3.x. + +**Files Updated:** + +- All test DAG files in `tests/integration/dags/*.py` + +#### 2b. Default View Parameter + +Airflow 3.x removed the `default_view` parameter from the DAG constructor. + +```python +# Airflow 2.x (supported) +DAG("my_dag", default_view="tree", ...) # Set default UI view + +# Airflow 3.x (removed) +DAG("my_dag", ...) # default_view parameter no longer accepted +``` + +**Reason for Removal:** + +- **User preferences are now persistent** - The Airflow UI remembers each user's preferred view per DAG +- **Separation of concerns** - DAG definition (pipeline logic) should be separate from UI presentation +- **Cleaner API** - Removes UI-specific parameters from the core DAG class + +**Valid `default_view` values in Airflow 2.x:** + +- `"tree"` - Tree view (hierarchical task structure) +- `"graph"` - Graph view (visual DAG graph) +- `"duration"` - Duration view +- `"gantt"` - Gantt chart view +- `"landing_times"` - Landing times view + +**Migration:** Simply remove the `default_view` parameter from DAG definitions when upgrading to Airflow 3.x. + +**Files Updated:** + +- `tests/integration/dags/airflow3/datahub_emitter_operator_jinja_template_dag.py` - Removed `default_view="tree"` + +### 3. API Changes + +#### REST API Version + +Airflow 3.x removed the v1 API and only supports v2. + +```python +# Airflow 3.x +api_version = "v2" + +# Airflow 2.x +api_version = "v1" +``` + +#### API Authentication + +Airflow 3.x uses JWT token-based authentication instead of HTTP Basic Auth. + +```python +# Airflow 3.x - Get JWT token +response = requests.post( + f"{airflow_url}/auth/token", + data={"username": username, "password": password} +) +token = response.json()["access_token"] +session.headers["Authorization"] = f"Bearer {token}" + +# Airflow 2.x - HTTP Basic Auth +session.auth = (username, password) +``` + +#### Configuration + +Airflow 3.x moved some configuration options: + +```bash +# Airflow 3.x +AIRFLOW__API__PORT=8080 +AIRFLOW__API__BASE_URL=http://airflow.example.com # Used for log URLs + +# Airflow 2.x +AIRFLOW__WEBSERVER__WEB_SERVER_PORT=8080 +AIRFLOW__WEBSERVER__BASE_URL=http://airflow.example.com # Used for log URLs +``` + +**Log URL Format Changes:** + +Airflow 3.x changed the log URL format and configuration: + +| Aspect | Airflow 2.x | Airflow 3.x | +| ---------- | ------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------- | +| Config key | `webserver.base_url` | `api.base_url` | +| URL format | `http://host/dags/{dag_id}/grid?dag_run_id={run_id}&task_id={task_id}&base_date={date}&tab=logs` | `http://host/dags/{dag_id}/runs/{run_id}/tasks/{task_id}?try_number={try_number}` | +| Source | `TaskInstance.log_url` property | `TaskInstance.log_url` property | + +**Example Log URLs:** + +```python +# Airflow 2.x +"http://airflow.example.com/dags/my_dag/grid?dag_run_id=manual_run&task_id=my_task&base_date=2023-09-27T21%3A34%3A38%2B0000&tab=logs" + +# Airflow 3.x +"http://airflow.example.com/dags/my_dag/runs/manual_run/tasks/my_task?try_number=1" +``` + +**Impact on DataHub:** + +- The `log_url` in DataHub's DataProcessInstance will reflect the Airflow version's format +- Both formats link correctly to the Airflow UI task logs +- The plugin automatically uses the correct configuration key for each version + +**Files Updated:** + +- `tests/integration/test_plugin.py` - Authentication, API version logic, and base URL configuration +- `src/datahub_airflow_plugin/_airflow_version_specific.py` - Task instance attribute extraction including log_url + +### 4. CLI Command Changes + +#### DAG Trigger + +Airflow 3.x renamed the `--exec-date` parameter to `--logical-date`. + +```bash +# Airflow 3.x +airflow dags trigger --logical-date "2023-09-27T21:34:38+00:00" my_dag + +# Airflow 2.x +airflow dags trigger --exec-date "2023-09-27T21:34:38+00:00" my_dag +``` + +**Files Updated:** + +- `tests/integration/test_plugin.py` - Conditional CLI parameter + +### 5. Listener Hook Signature Changes + +Airflow 3.x changed the signatures of listener hooks to remove the `session` parameter and add new parameters. + +#### Task Instance Hooks + +**Airflow 3.x signatures:** + +```python +on_task_instance_running(previous_state, task_instance) +on_task_instance_success(previous_state, task_instance) +on_task_instance_failed(previous_state, task_instance, error) +``` + +**Airflow 2.x signatures:** + +```python +on_task_instance_running(previous_state, task_instance, session) +on_task_instance_success(previous_state, task_instance, session) +on_task_instance_failed(previous_state, task_instance, session) +``` + +**Compatibility Fix:** +The plugin uses `**kwargs` to handle both versions without breaking pluggy's hook matching: + +```python +@hookimpl +def on_task_instance_running(self, previous_state, task_instance, **kwargs): + # Extract session if present (Airflow 2.x) + session = kwargs.get("session") + ... + +@hookimpl +def on_task_instance_failed(self, previous_state, task_instance, **kwargs): + # Extract error and session from kwargs (Airflow 3.x passes error, 2.x passes session) + session = kwargs.get("session") + error = kwargs.get("error") + ... +``` + +**Important:** Using default parameters like `session=None` in Airflow 3.0 causes pluggy to fail to match the hook spec, preventing hooks from being called. + +**Files Updated:** + +- `src/datahub_airflow_plugin/datahub_listener.py:559-772` - Listener hook signatures + +### 6. SubDAG Removal + +Airflow 3.x completely removed SubDAGs (deprecated since Airflow 2.0). + +#### Affected Attributes + +- `dag.is_subdag` - ❌ Removed +- `dag.parent_dag` - ❌ Removed +- `task.subdag` - ❌ Removed +- `SubDagOperator` - ❌ Removed + +#### Compatibility Fix + +The plugin uses defensive attribute access: + +```python +# Safe for both Airflow 2.x and 3.x +if getattr(dag, "is_subdag", False) and dag.parent_dag is not None: + # Handle subdag (only executes in Airflow 2.x) + ... +``` + +**Files Updated:** + +- `src/datahub_airflow_plugin/client/airflow_generator.py:76` - SubDAG handling + +### 7. Database Commit Restrictions in Listener Hooks + +Airflow listener hooks are called during SQLAlchemy's `after_flush` event, before the main transaction commits. Any database operations that create new sessions and commit them can interfere with the outer transaction and cause data loss. + +#### Problem + +The kill switch feature originally used `Variable.get()` to check if the plugin should be disabled: + +```python +# This causes issues: +# - Airflow 3.x: RuntimeError: UNEXPECTED COMMIT - THIS WILL BREAK HA LOCKS! +# - Airflow 2.x: Can cause TaskInstanceHistory records to not be persisted +# (see: https://github.com/apache/airflow/pull/48780) +if Variable.get("datahub_airflow_plugin_disable_listener", "false").lower() == "true": + return True +``` + +`Variable.get()` uses the `@provide_session` decorator which creates a new database session and commits it. When called from listener hooks (which execute during `after_flush`, before the main transaction commits), this nested commit can corrupt the outer transaction state and cause data loss. + +#### Solution + +Both Airflow 2.x and 3.x versions now use environment variables instead of database queries: + +**Both versions** (`airflow2/datahub_listener.py` and `airflow3/datahub_listener.py`): + +```python +def check_kill_switch(self) -> bool: + """ + Check kill switch using environment variable. + + We use os.getenv() instead of Variable.get() because Variable.get() + creates a new database session and commits it. When called from listener + hooks (which execute during SQLAlchemy's after_flush event, before the + main transaction commits), this nested commit can corrupt the outer + transaction state and cause data loss. + """ + if ( + os.getenv( + f"AIRFLOW_VAR_{KILL_SWITCH_VARIABLE_NAME}".upper(), "false" + ).lower() + == "true" + ): + logger.debug("DataHub listener disabled by kill switch (env var)") + return True + return False +``` + +**To disable the plugin (both Airflow 2.x and 3.x):** + +```bash +export AIRFLOW_VAR_DATAHUB_AIRFLOW_PLUGIN_DISABLE_LISTENER=true +``` + +**Files Updated:** + +- `src/datahub_airflow_plugin/airflow2/datahub_listener.py` - Kill switch using env var +- `src/datahub_airflow_plugin/airflow3/datahub_listener.py` - Kill switch using env var + +### 8. Threading Support in Airflow 3.x + +#### Status: ✅ Fully Working + +Threading is **enabled by default** in both Airflow 2.x and 3.x. Initial concerns about `RuntimeTaskInstance` unpickleable objects were unfounded. + +#### Why Threading Works + +**Key insight:** `threading.Thread` does **NOT** pickle its arguments - it passes object references directly within the same process. Only `multiprocessing.Process` requires pickling. + +```python +# This works fine - no pickling required +thread = threading.Thread(target=f, args=(task_instance,)) +thread.start() +``` + +**Verification:** Tests pass successfully with `DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD=true` in Airflow 3.0. + +#### Configuration + +Threading is enabled by default for performance benefits: + +```python +# Default: threading enabled +_RUN_IN_THREAD = os.getenv( + "DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD", "true" +).lower() in ("true", "1") +``` + +**To disable threading** (if needed): + +```bash +export DATAHUB_AIRFLOW_PLUGIN_RUN_IN_THREAD=false +``` + +**Benefits of Threading:** + +- Prevents slow lineage extraction from blocking task completion +- Non-blocking metadata emission to DataHub +- Better performance for complex SQL parsing and lineage extraction + +**Files Updated:** + +- `src/datahub_airflow_plugin/datahub_listener.py:147-152` - Threading configuration + +### 9. Template Rendering + +Airflow 3.x's `RuntimeTaskInstance` cannot be deep-copied due to unpickleable objects. + +#### Problem + +The plugin previously deep-copied task instances to render Jinja templates: + +```python +# This fails in Airflow 3.x: TypeError: cannot pickle '_thread.lock' object +task_instance_copy = copy.deepcopy(task_instance) +task_instance_copy.render_templates() +``` + +#### Solution + +The plugin now has separate template rendering implementations for each version: + +**Airflow 2.x** (`airflow2/datahub_listener.py`): + +```python +def _render_templates(task_instance: "TaskInstance") -> "TaskInstance": + # Render templates in a copy of the task instance + try: + task_instance_copy = copy.deepcopy(task_instance) + task_instance_copy.render_templates() + return task_instance_copy + except Exception as e: + logger.info(f"Error rendering templates: {e}") + return task_instance +``` + +**Airflow 3.x** (`airflow3/datahub_listener.py`): + +```python +def _render_templates(task_instance: "TaskInstance") -> "TaskInstance": + """ + Templates are already rendered in Airflow 3.x by the task execution system. + + RuntimeTaskInstance contains unpickleable thread locks, so we cannot use deepcopy. + RuntimeTaskInstance.task contains the operator with rendered templates. + """ + logger.debug( + "Skipping template rendering for Airflow 3.0+ (already rendered by task worker)" + ) + return task_instance +``` + +**Impact:** Jinja-templated SQL queries are correctly parsed in both Airflow 2.x and 3.x. + +**Files Updated:** + +- `src/datahub_airflow_plugin/airflow2/datahub_listener.py` - Template rendering for Airflow 2.x +- `src/datahub_airflow_plugin/airflow3/datahub_listener.py` - Template rendering for Airflow 3.x + +### 10. SQL Parser Integration for Airflow 3.x + +Airflow 3.x removed the extractor-based SQL parsing mechanism. SQL operators now call `SQLParser.generate_openlineage_metadata_from_sql()` directly. + +#### Architecture Change + +**Airflow 2.x:** + +``` +SQL Operator → OpenLineage Extractor → DataHub Extractor → DataHub SQL Parser → Lineage +``` + +**Airflow 3.x:** + +``` +SQL Operator → SQLParser.generate_openlineage_metadata_from_sql() → [Patched by DataHub] → DataHub SQL Parser → Lineage +``` + +#### Implementation + +The plugin patches `SQLParser.generate_openlineage_metadata_from_sql()` to use DataHub's SQL parser: + +```python +def patch_sqlparser(): + from airflow.providers.openlineage.sqlparser import SQLParser + + # Store original method for fallback + SQLParser._original_generate_openlineage_metadata_from_sql = ( + SQLParser.generate_openlineage_metadata_from_sql + ) + + # Replace with DataHub-enhanced version + SQLParser.generate_openlineage_metadata_from_sql = ( + _datahub_generate_openlineage_metadata_from_sql + ) +``` + +The patched method: + +1. Calls DataHub's SQL parser to extract column-level lineage +2. Converts DataHub URNs to OpenLineage Dataset objects +3. Stores the SQL parsing result in `run_facets` for later retrieval +4. Falls back to the original implementation on errors + +**Column-Level Lineage:** The SQL parsing result is stored in the `OperatorLineage.run_facets` dictionary: + +```python +DATAHUB_SQL_PARSING_RESULT_KEY = "datahub_sql_parsing_result" +run_facets = { + DATAHUB_SQL_PARSING_RESULT_KEY: sql_parsing_result +} + +operator_lineage = OperatorLineage( + inputs=inputs, + outputs=outputs, + job_facets={"sql": SqlJobFacet(query=sql)}, + run_facets=run_facets, +) +``` + +The listener retrieves it and processes column lineage: + +```python +if DATAHUB_SQL_PARSING_RESULT_KEY in operator_lineage.run_facets: + sql_parsing_result = operator_lineage.run_facets[DATAHUB_SQL_PARSING_RESULT_KEY] + + # Process column lineage + if sql_parsing_result.column_lineage: + fine_grained_lineages.extend( + FineGrainedLineageClass( + upstreamType=FineGrainedLineageUpstreamTypeClass.FIELD_SET, + downstreamType=FineGrainedLineageDownstreamTypeClass.FIELD, + upstreams=[...], + downstreams=[...], + ) + for column_lineage in sql_parsing_result.column_lineage + ) +``` + +**Important:** `OperatorLineage` uses `@define` (attrs library) which creates a frozen dataclass. We cannot add arbitrary attributes to it, so we use the `run_facets` dictionary instead. + +**Supported Databases:** All databases supported by DataHub's SQL parser: + +- Snowflake, BigQuery, Redshift, PostgreSQL, MySQL, Oracle, SQL Server, Athena, Presto, Trino, etc. + +**Files Updated:** + +- `src/datahub_airflow_plugin/_airflow3_sql_parser_patch.py` - SQLParser patch implementation +- `src/datahub_airflow_plugin/datahub_listener.py:433-439` - Retrieve sql_parsing_result from run_facets +- `src/datahub_airflow_plugin/_config.py` - Enable SQL parser patch for Airflow 3.x + +### 11. Emitter Initialization: SDK Connection API Instead of BaseHook + +In Airflow 3.x, the DataHub plugin uses the SDK's `Connection.get()` method to initialize the emitter instead of relying on `BaseHook.get_connection()`. + +#### Problem: SUPERVISOR_COMMS Limitation + +The `BaseHook.get_connection()` method requires `SUPERVISOR_COMMS` to be available in the execution context. However, in Airflow 3.x listener hooks (such as `on_dag_start`, `on_dag_run_running`), the `SUPERVISOR_COMMS` context is not available, causing connection retrieval to fail: + +```python +# This fails in listener context: +# ImportError: cannot import name 'SUPERVISOR_COMMS' from 'airflow.sdk.execution_time.task_runner' +hook = self.config.make_emitter_hook() +emitter = hook.make_emitter() # ❌ Fails: SUPERVISOR_COMMS not available +``` + +**Why SUPERVISOR_COMMS is unavailable:** + +- Listener hooks run in a different context than task execution +- `SUPERVISOR_COMMS` is only available during actual task execution (when tasks are running) +- Listener hooks are called by the scheduler/webserver, not by task workers +- The supervisor communication mechanism is not initialized in listener context + +#### Solution: Use SDK Connection API + +The plugin now uses the Airflow SDK's `Connection.get()` method to retrieve connection details: + +```python +def _create_single_emitter_from_connection(self, conn_id: str): + """ + Create a single emitter from a connection ID. + + Uses Connection.get() from SDK which works in all contexts. + """ + from airflow.sdk import Connection + + # Get connection using SDK API (works in all contexts) + conn = Connection.get(conn_id) + if not conn: + logger.warning( + f"Connection '{conn_id}' not found in secrets backend or environment variables" + ) + return None + + # Build emitter from connection details + host = conn.host or "" + if not host: + logger.warning(f"Connection '{conn_id}' has no host configured") + return None + + # Parse URL and add port if needed + from urllib.parse import urlparse, urlunparse + parsed = urlparse(host if "://" in host else f"http://{host}") + netloc = parsed.netloc + if conn.port and not parsed.port: + netloc = f"{parsed.hostname}:{conn.port}" + + host = urlunparse(( + parsed.scheme or "http", + netloc, + parsed.path, + parsed.params, + parsed.query, + parsed.fragment + )) + + token = conf.get("datahub", "token", fallback=None) or conn.password + + return DataHubRestEmitter( + host, token, + client_mode=ClientMode.INGESTION, + datahub_component="airflow-plugin", + **conn.extra_dejson + ) +``` + +**Why `Connection.get()` from SDK:** + +- ✅ **Official Airflow 3.x API** - `airflow.sdk.Connection` is the proper SDK method +- ✅ **Not deprecated** - Unlike `Connection.get_connection_from_secrets()` from `airflow.models` +- ✅ **Works in all contexts** - Listener hooks, task execution, DAG parsing +- ✅ **No SUPERVISOR_COMMS dependency** - Checks environment variables, secrets backends, and database through proper APIs +- ✅ **More reliable** - Doesn't depend on execution context being fully initialized +- ✅ **Consistent behavior** - Same initialization method regardless of when it's called +- ✅ **Cleaner imports** - Uses SDK module structure instead of legacy models module + +**Import Path:** + +```python +# Airflow 3.x (correct, non-deprecated) +from airflow.sdk import Connection + +# Airflow 2.x (deprecated in Airflow 3.x) +from airflow.models import Connection +``` + +**Benefits:** + +- Uses Python's `urllib.parse` for robust URL handling (handles IPv6, paths, query strings) +- Supports all DataHub connection types (REST, Kafka, File) +- Handles multiple comma-separated connection IDs via `CompositeEmitter` + +**Token Configuration:** + +The plugin supports token configuration via `airflow.cfg` (takes precedence) or connection password: + +```ini +# airflow.cfg +[datahub] +token = your_datahub_token_here +``` + +If not set in `airflow.cfg`, the connection password field is used as a fallback. + +**Files Updated:** + +- `src/datahub_airflow_plugin/airflow3/datahub_listener.py:297-398` - Emitter initialization using SDK Connection API + +### 12. Operator-Specific Patches for OpenLineage + +In Airflow 3.x, some operators require specific patches to enable proper lineage extraction because they either: + +1. Don't implement required OpenLineage methods (`get_openlineage_database_info()`) +2. Use non-standard SQL storage mechanisms (e.g., configuration dictionaries) +3. Use generic SQL dialects that don't support column-level lineage + +The plugin patches these operators to provide full lineage support. + +#### 12a. SQLite Operator Patch + +**Problem:** `SqliteHook` doesn't implement `get_openlineage_database_info()`, causing lineage extraction to fail. + +**Solution:** Patch `SqliteHook.get_openlineage_database_info()` to return proper database info: + +```python +def get_openlineage_database_info(connection: Connection) -> DatabaseInfo: + # Extract database name from SQLite file path + db_path = connection.host + db_name = os.path.splitext(os.path.basename(db_path))[0] + + return DatabaseInfo( + scheme="sqlite", + authority=None, # SQLite doesn't have host:port + database=db_name, + normalize_name_method=lambda x: x.lower(), + ) +``` + +**Files:** `src/datahub_airflow_plugin/airflow3/_sqlite_openlineage_patch.py` + +#### 12b. Athena Operator Patch + +**Problem:** `AthenaOperator` uses `SQLParser` with `dialect="generic"`, which doesn't provide column-level lineage. + +**Solution:** Wrap `AthenaOperator.get_openlineage_facets_on_complete()` to: + +1. Call the original OpenLineage implementation +2. Enhance it with DataHub's SQL parser for column-level lineage + +```python +def get_openlineage_facets_on_complete(self, task_instance): + # Get original OpenLineage result + operator_lineage = original_method(self, task_instance) + + # Enhance with DataHub SQL parsing + sql_parsing_result = create_lineage_sql_parsed_result( + query=self.query, + platform="athena", + default_db=self.database, + ) + + # Store result in run_facets for DataHub listener + operator_lineage.run_facets["datahub_sql_parsing_result"] = sql_parsing_result + + return operator_lineage +``` + +**Files:** `src/datahub_airflow_plugin/airflow3/_athena_openlineage_patch.py` + +#### 12c. BigQuery InsertJobOperator Patch + +**Problem:** `BigQueryInsertJobOperator` stores SQL in a `configuration` dictionary, not as a direct attribute. This means: + +- The standard SQLParser patch (Section 10) can't intercept it because it doesn't go through `SQLParser.generate_openlineage_metadata_from_sql()` +- The official OpenLineage implementation extracts table-level lineage from BigQuery's job metadata/API response, **not by parsing SQL** +- Therefore, the official implementation provides table-level lineage but **no column-level lineage** + +**Solution:** Wrap `get_openlineage_facets_on_complete()` to: + +1. Call the original OpenLineage implementation (gets table-level lineage from BigQuery job metadata) +2. Extract SQL from `self.configuration.get("query", {}).get("query")` +3. Run DataHub's SQL parser with BigQuery dialect to add column-level lineage +4. Handle destination table from configuration +5. Store result in `run_facets` + +```python +def get_openlineage_facets_on_complete(self, task_instance): + # Extract SQL from configuration + sql = self.configuration.get("query", {}).get("query") + + # Get original result + operator_lineage = original_method(self, task_instance) + + # Run DataHub parser + sql_parsing_result = create_lineage_sql_parsed_result( + query=sql, + platform="bigquery", + default_db=self.project_id, + ) + + # Add destination table if specified in configuration + destination_table = self.configuration.get("query", {}).get("destinationTable") + if destination_table: + # Add to output tables + ... + + operator_lineage.run_facets["datahub_sql_parsing_result"] = sql_parsing_result + return operator_lineage +``` + +**Files:** `src/datahub_airflow_plugin/airflow3/_bigquery_openlineage_patch.py` + +#### Patch Registration + +All patches are automatically applied when the plugin is loaded in Airflow 3.x: + +```python +# In _airflow_compat.py +from datahub_airflow_plugin.airflow3._sqlite_openlineage_patch import patch_sqlite_hook +from datahub_airflow_plugin.airflow3._athena_openlineage_patch import patch_athena_operator +from datahub_airflow_plugin.airflow3._bigquery_openlineage_patch import patch_bigquery_insert_job_operator + +patch_sqlite_hook() +patch_athena_operator() +patch_bigquery_insert_job_operator() +``` + +**Key Points:** + +- ✅ Patches are applied at import time, before any DAGs are loaded +- ✅ Patches are idempotent (safe to call multiple times) +- ✅ Patches gracefully handle missing providers (no error if provider not installed) +- ✅ Each patch wraps the original method, ensuring compatibility with Airflow's OpenLineage implementation +- ✅ Column-level lineage is consistently stored in `run_facets["datahub_sql_parsing_result"]` for the listener to process + +**Files Updated:** + +- `src/datahub_airflow_plugin/airflow3/_airflow_compat.py` - Patch registration +- `src/datahub_airflow_plugin/airflow3/_sqlite_openlineage_patch.py` - SQLite hook patch +- `src/datahub_airflow_plugin/airflow3/_athena_openlineage_patch.py` - Athena operator patch +- `src/datahub_airflow_plugin/airflow3/_bigquery_openlineage_patch.py` - BigQuery operator patch + +### 13. Column-Level Lineage in Airflow 3.x + +**Status:** ✅ Fully Working + +Column-level (fine-grained) lineage is now supported in Airflow 3.x through: + +1. The **SQLParser patch** (Section 10) for standard SQL operators +2. **Operator-specific patches** (Section 12) for special-case operators + +**Example:** For a SQL query like: + +```sql +INSERT INTO processed_costs (id, month, total_cost, area, cost_per_area) +SELECT id, month, total_cost, area, total_cost / area +FROM costs +``` + +The plugin generates fine-grained lineage: + +- `costs.id → processed_costs.id` +- `costs.month → processed_costs.month` +- `costs.total_cost → processed_costs.total_cost` +- `costs.area → processed_costs.area` +- `costs.area + costs.total_cost → processed_costs.cost_per_area` (derived column) + +**Verification:** Run the Snowflake operator test: + +```bash +tox -e py311-airflow302 -- -k "v2_snowflake_operator_airflow3" +``` + +Check the golden file for `fineGrainedLineages`: + +```bash +grep -A 10 "fineGrainedLineages" tests/integration/goldens/v2_snowflake_operator_airflow3.json +``` + +## Known Limitations + +### 1. SubDAG Lineage Not Supported in Airflow 3.x + +**Impact:** If you're upgrading from Airflow 2.x and using SubDAGs, lineage tracking for subdags will no longer work in Airflow 3.x. + +**Reason:** SubDAGs were completely removed from Airflow 3.x. + +**Migration Path:** Use TaskGroups instead of SubDAGs. TaskGroups provide visual grouping without creating separate DAG runs. + +```python +# Old (Airflow 2.x) - SubDAG +from airflow.operators.subdag import SubDagOperator + +def subdag(parent_dag_name, child_dag_name, args): + dag = DAG(f"{parent_dag_name}.{child_dag_name}", **args) + # Add tasks... + return dag + +with DAG("parent_dag") as dag: + subdag_task = SubDagOperator( + task_id="subdag", + subdag=subdag("parent_dag", "subdag", default_args) + ) + +# New (Airflow 3.x) - TaskGroup +from airflow.utils.task_group import TaskGroup + +with DAG("parent_dag") as dag: + with TaskGroup("task_group") as tg: + # Add tasks to the group... + task1 = BashOperator(...) + task2 = BashOperator(...) +``` + +**Lineage Note:** TaskGroup lineage is tracked at the task level, not as a separate DAG entity. + +### 2. Configuration Migration Required + +When upgrading to Airflow 3.x, update your configuration: + +```bash +# Old Airflow 2.x config +AIRFLOW__WEBSERVER__WEB_SERVER_PORT=8080 +AIRFLOW__API__AUTH_BACKEND=airflow.api.auth.backend.basic_auth + +# New Airflow 3.x config +AIRFLOW__API__PORT=8080 +# AUTH_BACKEND no longer needed - uses SimpleAuthManager by default +``` + +### 3. Test Compatibility Matrix + +The DataHub Airflow plugin tests are designed to work with: + +| Airflow Version | Test Support | Notes | +| --------------- | ------------ | ---------------------- | +| 2.3.x | ✅ Limited | Only v1 plugin tested | +| 2.4.x - 2.9.x | ✅ Full | Both v1 and v2 plugins | +| 3.0.x+ | ✅ Full | v2 plugin only | + +**Note:** Airflow 3.x requires the v2 plugin (listener-based). The v1 plugin is not compatible. + +## Testing + +### Running Tests Against Airflow 3.x + +```bash +# Test with Airflow 3.0.x +tox -e py311-airflow310 + +# Test with Airflow 3.1.x +tox -e py311-airflow31 +``` + +### Verifying Compatibility + +The following checks are performed in tests: + +1. ✅ Import compatibility - All modules import without warnings +2. ✅ DAG parsing - DAGs parse successfully without errors +3. ✅ API authentication - JWT token authentication works +4. ✅ Lineage extraction - Inlets/outlets are correctly extracted +5. ✅ Listener functionality - All listener hooks execute properly + +## Troubleshooting + +### Import Errors + +**Error:** `ImportError: cannot import name 'BaseOperator' from 'airflow.models.baseoperator'` + +**Solution:** This is expected in Airflow 3.x. The plugin's shims handle this automatically. If you see this error, ensure you're using the latest version of the plugin. + +### Authentication Failures + +**Error:** `401 Unauthorized` when calling Airflow API + +**Solution:** + +- Airflow 3.x requires JWT authentication +- Ensure the username/password are correct +- Check that the `/auth/token` endpoint is accessible + +### SubDAG Warnings + +**Warning:** Code references `is_subdag` attribute + +**Solution:** This is expected and safe. The plugin uses `getattr(dag, "is_subdag", False)` which returns `False` in Airflow 3.x without errors. + +### Schedule Parameter Issues + +**Error:** `TypeError: __init__() got an unexpected keyword argument 'schedule_interval'` + +**Solution:** Update DAG definitions to use `schedule=` instead of `schedule_interval=`. The `schedule` parameter is supported in Airflow 2.4+ and required in Airflow 3.x. + +### Default View Parameter Issues + +**Error:** `TypeError: __init__() got an unexpected keyword argument 'default_view'` + +**Solution:** Remove the `default_view` parameter from DAG definitions. This parameter was removed in Airflow 3.x. User view preferences are now persistent in the UI. + +## Migration Checklist + +When upgrading to Airflow 3.x: + +- [ ] Update all DAG definitions to use `schedule=` instead of `schedule_interval=` +- [ ] Remove `default_view=` parameter from all DAG definitions +- [ ] Replace SubDAGs with TaskGroups +- [ ] Update Airflow configuration (port, auth settings) +- [ ] Update any custom operators to use new import paths +- [ ] Test lineage extraction with sample DAGs +- [ ] Verify API authentication works +- [ ] Update CI/CD pipelines to use Airflow 3.x + +## References + +- [Airflow 3.0 Release Notes](https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html#airflow-3-0-0) +- [Airflow 3.0 Migration Guide](https://airflow.apache.org/docs/apache-airflow/stable/migration-guide-to-3.html) +- [TaskGroups Documentation](https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/dags.html#taskgroups) +- [DataHub Airflow Plugin Documentation](https://datahubproject.io/docs/metadata-ingestion/integration_docs/airflow/) + +## Support + +For issues related to Airflow 3.x compatibility: + +1. Check this migration guide +2. Review the [GitHub Issues](https://github.com/datahub-project/datahub/issues) +3. Ask in the [DataHub Slack](https://slack.datahubproject.io/) + +## Version History + +| Version | Date | Changes | +| ------- | ---------- | --------------------------------- | +| 1.0 | 2025-01-XX | Initial Airflow 3.x support added | diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/DOCKER_TEST_GUIDE.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/DOCKER_TEST_GUIDE.md new file mode 100644 index 00000000..bbf3f43c --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/DOCKER_TEST_GUIDE.md @@ -0,0 +1,483 @@ +--- +title: Docker Test Guide +slug: /metadata-ingestion-modules/airflow-plugin/docker_test_guide +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/DOCKER_TEST_GUIDE.md +--- +# Docker Test Guide + +This guide explains how to run Airflow plugin tests in Docker with automatic volume mounts for source code and golden files. + +## Overview + +The Docker test environment uses **tox** to manage all dependencies: + +- ✅ Airflow (multiple versions: 2.7, 2.8, 2.9, 2.10, 3.1) +- ✅ Airflow constraints files (for reproducible builds) +- ✅ Provider packages (for Airflow 3.x) +- ✅ All test dependencies +- ✅ **Automatic volume mounts** for source code and golden files + +**Three ways to run tests:** + +1. **Wrapper Script** (easiest) - `./run-tests.sh` +2. **Docker Compose** (recommended for CI/CD) +3. **Docker CLI** (most control) + +## Quick Start + +### Option 1: Wrapper Script (Easiest) + +The wrapper script automatically handles volume mounts and permissions: + +```bash +cd metadata-ingestion-modules/airflow-plugin + +# Run all tests with Airflow 3.1 (default) +./run-tests.sh + +# Run tests with Airflow 2.9 +./run-tests.sh py311-airflow29 + +# Run specific test +./run-tests.sh py311-airflow31 -- tests/integration/test_plugin.py::test_v2_basic_dag -v + +# Update golden files (automatically mounted!) +./run-tests.sh py311-airflow31 -- --update-golden-files + +# Rebuild Docker image before running +REBUILD=true ./run-tests.sh +``` + +**Benefits:** + +- ✅ Automatic volume mounts (source code + golden files) +- ✅ Correct user permissions (no root-owned files) +- ✅ Auto-builds image if needed +- ✅ Simple, easy-to-remember commands + +### Option 2: Docker Compose (Recommended) + +Docker Compose automatically mounts volumes and caches tox environments: + +```bash +cd metadata-ingestion-modules/airflow-plugin + +# Run all tests with Airflow 3.1 (default) +docker-compose -f docker-compose.test.yml run --rm airflow-plugin-test + +# Run tests with Airflow 2.9 +docker-compose -f docker-compose.test.yml run --rm airflow-plugin-test py311-airflow29 + +# Run specific test +docker-compose -f docker-compose.test.yml run --rm airflow-plugin-test py311-airflow31 -- tests/integration/test_plugin.py::test_v2_basic_dag -v + +# Update golden files (automatically mounted!) +docker-compose -f docker-compose.test.yml run --rm airflow-plugin-test py311-airflow31 -- --update-golden-files + +# Build the image +docker-compose -f docker-compose.test.yml build +``` + +**Benefits:** + +- ✅ Automatic volume mounts (source code + golden files) +- ✅ Persistent tox cache (faster subsequent runs) +- ✅ Easy to customize via docker-compose.test.yml +- ✅ Great for CI/CD + +### Option 3: Docker CLI (Most Control) + +If you need full control or don't want the wrapper: + +```bash +# Build from repository root +cd /path/to/datahub +docker build -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test -t airflow-plugin-test . + +# Run all tests with Airflow 3.1 (default) +docker run --rm airflow-plugin-test + +# Run tests with Airflow 2.9 +docker run --rm airflow-plugin-test py311-airflow29 + +# Run specific test +docker run --rm airflow-plugin-test py311-airflow31 -- tests/integration/test_plugin.py::test_v2_basic_dag -v + +# Update golden files (manual volume mount) +docker run --rm \ + -v $(pwd)/metadata-ingestion-modules/airflow-plugin:/app/metadata-ingestion-modules/airflow-plugin \ + airflow-plugin-test py311-airflow31 -- --update-golden-files +``` + +## Building the Image + +### Default Build (Python 3.11, Airflow 3.1) + +```bash +# Must be run from repository root +docker build -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test -t airflow-plugin-test . +``` + +### Custom Python Version + +```bash +docker build -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test \ + --build-arg PYTHON_VERSION=3.10 \ + -t airflow-plugin-test:py310 . +``` + +### Custom Default Tox Environment + +```bash +docker build -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test \ + --build-arg TOX_ENV=py311-airflow29 \ + -t airflow-plugin-test:af29 . +``` + +## Running Tests + +### Available Tox Environments + +From `tox.ini`: + +- `py39-airflow27` - Python 3.9, Airflow 2.7 +- `py310-airflow27` - Python 3.10, Airflow 2.7 +- `py310-airflow28` - Python 3.10, Airflow 2.8 +- `py311-airflow29` - Python 3.11, Airflow 2.9 +- `py311-airflow210` - Python 3.11, Airflow 2.10 +- `py311-airflow31` - Python 3.11, Airflow 3.1 (default) + +### Run All Tests + +```bash +# With default environment (py311-airflow31) +docker run airflow-plugin-test + +# With specific environment +docker run airflow-plugin-test py311-airflow29 +``` + +### Run Specific Test File + +```bash +docker run airflow-plugin-test py311-airflow31 -- tests/integration/test_plugin.py -v +``` + +### Run Specific Test Function + +```bash +docker run airflow-plugin-test py311-airflow31 -- tests/integration/test_plugin.py::test_v2_snowflake_operator_airflow3 -v +``` + +### Run Tests Matching Pattern + +```bash +docker run airflow-plugin-test py311-airflow31 -- -k "snowflake" -v +``` + +### Run with Additional pytest Options + +```bash +# Stop on first failure +docker run airflow-plugin-test py311-airflow31 -- -x + +# Show local variables in tracebacks +docker run airflow-plugin-test py311-airflow31 -- -l + +# Verbose output +docker run airflow-plugin-test py311-airflow31 -- -vv + +# Show captured output +docker run airflow-plugin-test py311-airflow31 -- -s +``` + +## Updating Golden Files + +Golden files are **automatically mounted** when using the wrapper script or docker-compose! + +### With Wrapper Script (Easiest) + +```bash +# Update all golden files +./run-tests.sh py311-airflow31 -- --update-golden-files + +# Update golden files for specific test +./run-tests.sh py311-airflow31 -- tests/integration/test_plugin.py::test_v2_snowflake_operator_airflow3 --update-golden-files +``` + +**That's it!** Golden files are automatically written to your local filesystem. + +### With Docker Compose + +```bash +# Update all golden files +docker-compose -f docker-compose.test.yml run --rm airflow-plugin-test py311-airflow31 -- --update-golden-files + +# Update golden files for specific test +docker-compose -f docker-compose.test.yml run --rm airflow-plugin-test py311-airflow31 -- tests/integration/test_plugin.py::test_v2_snowflake_operator_airflow3 --update-golden-files +``` + +### With Docker CLI (Manual Volume Mount) + +If you're using raw Docker commands, you need to mount the golden files directory: + +```bash +# From repository root +docker run --rm \ + -v $(pwd)/metadata-ingestion-modules/airflow-plugin:/app/metadata-ingestion-modules/airflow-plugin \ + airflow-plugin-test py311-airflow31 -- --update-golden-files + +# Or just mount the goldens directory +docker run --rm \ + -v $(pwd)/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens:/app/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens \ + airflow-plugin-test py311-airflow31 -- --update-golden-files +``` + +## Advanced Usage + +### Interactive Shell + +```bash +docker run -it --entrypoint /bin/bash airflow-plugin-test + +# Inside container: +cd metadata-ingestion-modules/airflow-plugin +tox -e py311-airflow31 -- -v +``` + +### Run Tox Directly (Without Entrypoint) + +```bash +docker run --entrypoint tox airflow-plugin-test -e py311-airflow29 -- tests/integration/ -v +``` + +### Volume Mount for Development + +Mount your local code for live editing: + +```bash +docker run -v $(pwd):/app airflow-plugin-test py311-airflow31 -- tests/integration/ -v +``` + +**Note:** You may need to rebuild the tox environment after code changes: + +```bash +docker run -v $(pwd):/app airflow-plugin-test py311-airflow31 --recreate +``` + +### Override Default Tox Environment + +```bash +# Set different default at runtime +docker run -e TOX_ENV=py311-airflow29 airflow-plugin-test +``` + +## Testing Different Airflow Versions + +### Airflow 2.7 + +```bash +docker run airflow-plugin-test py310-airflow27 +``` + +### Airflow 2.9 + +```bash +docker run airflow-plugin-test py311-airflow29 +``` + +### Airflow 3.1 + +```bash +docker run airflow-plugin-test py311-airflow31 +``` + +## Troubleshooting + +### Build Context Issues + +The Dockerfile must be run from the repository root because it needs access to `metadata-ingestion`: + +```bash +# ✅ Correct +cd /path/to/datahub +docker build -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test -t airflow-plugin-test . + +# ❌ Wrong (will fail - can't find ../../metadata-ingestion) +cd /path/to/datahub/metadata-ingestion-modules/airflow-plugin +docker build -f Dockerfile.test -t airflow-plugin-test . +``` + +### Tox Cache Issues + +If you encounter stale dependencies: + +```bash +# Rebuild tox environment +docker run airflow-plugin-test py311-airflow31 --recreate + +# Or rebuild Docker image with no cache +docker build --no-cache -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test -t airflow-plugin-test . +``` + +### Python Version Mismatch + +Ensure the Python version in the Docker build matches the tox environment: + +```bash +# For py310-* environments +docker build --build-arg PYTHON_VERSION=3.10 \ + -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test \ + -t airflow-plugin-test:py310 . + +docker run airflow-plugin-test:py310 py310-airflow28 +``` + +### Permission Issues with Golden Files + +When using volume mounts: + +```bash +# Run with current user's UID/GID +docker run --user $(id -u):$(id -g) \ + -v $(pwd)/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens:/app/metadata-ingestion-modules/airflow-plugin/tests/integration/goldens \ + airflow-plugin-test py311-airflow31 -- --update-golden-files +``` + +### Out of Memory + +Tox may use significant memory when building environments: + +```bash +docker run --memory=4g airflow-plugin-test +``` + +## CI/CD Integration + +### GitHub Actions Example + +```yaml +name: Test Airflow Plugin +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + tox-env: + - py310-airflow27 + - py310-airflow28 + - py311-airflow29 + - py311-airflow210 + - py311-airflow31 + + steps: + - uses: actions/checkout@v3 + + - name: Build test image + run: | + docker build -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test \ + -t airflow-plugin-test . + + - name: Run tests + run: docker run airflow-plugin-test ${{ matrix.tox-env }} +``` + +### GitLab CI Example + +```yaml +test: + image: docker:latest + services: + - docker:dind + script: + - docker build -f metadata-ingestion-modules/airflow-plugin/Dockerfile.test -t airflow-plugin-test . + - docker run airflow-plugin-test $TOX_ENV + parallel: + matrix: + - TOX_ENV: + - py310-airflow27 + - py310-airflow28 + - py311-airflow29 + - py311-airflow31 +``` + +## How It Works + +### Tox Configuration + +The `tox.ini` file defines all test environments with: + +- Airflow version constraints +- Official Airflow constraints files for reproducible builds +- Provider packages for Airflow 3.x +- All necessary dependencies + +### Entrypoint Logic + +The entrypoint script intelligently routes commands: + +```bash +# No args → run default tox environment +docker run airflow-plugin-test +# Executes: tox -e $TOX_ENV + +# Tox env specified → run that environment +docker run airflow-plugin-test py311-airflow29 +# Executes: tox -e py311-airflow29 + +# Other args → pass to pytest via default environment +docker run airflow-plugin-test -- -k snowflake -v +# Executes: tox -e $TOX_ENV -- -k snowflake -v + +# Tox env + pytest args +docker run airflow-plugin-test py311-airflow31 -- tests/integration/test_plugin.py -v +# Executes: tox -e py311-airflow31 -- tests/integration/test_plugin.py -v +``` + +## Comparison: Docker vs Local Tox + +| Aspect | Docker | Local Tox | +| ---------------------- | ------------------------------ | --------------------------- | +| **Setup** | Build once, run anywhere | Requires local Python setup | +| **Reproducibility** | Guaranteed (same OS, packages) | Varies by local environment | +| **CI/CD** | Native support | Needs Python pre-installed | +| **Speed (first run)** | Slower (Docker build) | Slower (tox setup) | +| **Speed (subsequent)** | Fast if cached | Fast if cached | +| **Disk Usage** | Higher (Docker layers) | Lower | +| **Isolation** | Complete (OS level) | Python environment only | +| **Golden Files** | Volume mounts needed | Direct access | + +**Use Docker for:** + +- CI/CD pipelines +- Consistent cross-platform testing +- Complete environment isolation +- Sharing test environments + +**Use Local Tox for:** + +- Day-to-day development +- Faster iteration +- Direct file access +- Debugging + +## Best Practices + +1. **Always build from repository root** - The Dockerfile needs access to `metadata-ingestion` +2. **Use volume mounts for golden files** - Makes it easy to extract updated files +3. **Match Python versions** - Build arg should match tox environment (py310 → PYTHON_VERSION=3.10) +4. **Cache Docker layers** - Organize changes to maximize layer reuse +5. **Clean up** - Remove unused images: `docker image prune` +6. **Use specific tox envs** - Don't rely on defaults in CI/CD + +## Support + +For issues: + +- **Tox configuration**: See `tox.ini` in the airflow-plugin directory +- **Docker logs**: `docker logs ` +- **Build output**: `docker build --progress=plain` +- **Interactive debugging**: `docker run -it --entrypoint /bin/bash airflow-plugin-test` diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.docker.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.docker.md new file mode 100644 index 00000000..5d33f195 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.docker.md @@ -0,0 +1,137 @@ +--- +title: Docker Test Environment +slug: /metadata-ingestion-modules/airflow-plugin/readme.docker +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/README.docker.md +--- +# Docker Test Environment + +This directory contains a complete Docker-based test environment for the Airflow plugin. + +## Files + +- **`Dockerfile.test`** - Docker image that runs tests via tox +- **`docker-compose.test.yml`** - Docker Compose config with automatic volume mounts +- **`run-tests.sh`** - Wrapper script for the easiest test experience +- **`DOCKER_TEST_GUIDE.md`** - Complete documentation with examples + +## Quick Start + +### Easiest: Use the Wrapper Script + +```bash +# Run all tests +./run-tests.sh + +# Run specific test +./run-tests.sh py311-airflow31 -- tests/integration/test_plugin.py::test_v2_basic_dag -v + +# Update golden files (automatically saved to your local filesystem!) +./run-tests.sh py311-airflow31 -- --update-golden-files +``` + +**Benefits:** + +- ✅ Automatic volume mounts (no manual `-v` flags) +- ✅ Correct file permissions (no root-owned files) +- ✅ Auto-builds image if needed +- ✅ Simple to use + +### Alternative: Docker Compose + +```bash +# Run all tests +docker compose -f docker-compose.test.yml run --rm airflow-plugin-test + +# Update golden files +docker compose -f docker-compose.test.yml run --rm airflow-plugin-test +``` + +## Features + +### Automatic Volume Mounts + +Both the wrapper script and Docker Compose automatically mount: + +- **Source code** (`metadata-ingestion` and `airflow-plugin`) +- **Golden files** (`tests/integration/goldens`) +- **Tox cache** (`.tox` directory - Docker Compose only) + +This means: + +- ✅ No manual volume mount commands +- ✅ Golden files are automatically updated on your local filesystem +- ✅ Source code changes are immediately available +- ✅ Tox environments are cached between runs (Docker Compose) + +### Tox-Based Dependency Management + +The Docker image uses **tox** to install all dependencies: + +- Airflow versions (2.7, 2.8, 2.9, 2.10, 3.1) +- Airflow constraints files (for reproducible builds) +- Provider packages (Airflow 3.x) +- All test dependencies + +This ensures: + +- ✅ No duplicate dependency logic +- ✅ Same dependencies as local development +- ✅ Automatic constraint file handling + +### Multiple Airflow Versions + +Test against any Airflow version in one image: + +```bash +./run-tests.sh py310-airflow27 # Airflow 2.7 +./run-tests.sh py311-airflow29 # Airflow 2.9 +./run-tests.sh py311-airflow31 # Airflow 3.1 +``` + +## Why Use Docker? + +| Use Case | Recommended Approach | +| ---------------------------- | ------------------------------- | +| **Local development** | Local tox (faster iteration) | +| **CI/CD pipelines** | Docker (complete isolation) | +| **Cross-platform testing** | Docker (consistent environment) | +| **Sharing test environment** | Docker (works everywhere) | +| **Testing on different OS** | Docker (same Linux base) | + +## Documentation + +See **`DOCKER_TEST_GUIDE.md`** for: + +- Detailed usage examples +- All three methods (wrapper, compose, CLI) +- Advanced configuration +- CI/CD integration examples +- Troubleshooting guide + +## Quick Reference + +```bash +# Wrapper Script (easiest) +./run-tests.sh # All tests +./run-tests.sh py311-airflow29 # Airflow 2.9 +./run-tests.sh py311-airflow31 -- -k snowflake -v # Specific tests +./run-tests.sh py311-airflow31 -- --update-golden-files # Update golden files +REBUILD=true ./run-tests.sh # Force rebuild + +# Docker Compose (recommended for CI/CD) +docker compose -f docker-compose.test.yml run --rm airflow-plugin-test +docker compose -f docker-compose.test.yml run --rm airflow-plugin-test py311-airflow29 +docker compose -f docker-compose.test.yml build + +# Docker CLI (most control) +docker build -f Dockerfile.test -t airflow-plugin-test ../../ +docker run --rm airflow-plugin-test +docker run --rm airflow-plugin-test py311-airflow31 -- -v +``` + +## Support + +- **Issues with tests**: See `tox.ini` and test files +- **Issues with Docker**: See `DOCKER_TEST_GUIDE.md` +- **Issues with dependencies**: See `setup.py` and tox configuration diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.md new file mode 100644 index 00000000..f8c82cd4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/README.md @@ -0,0 +1,232 @@ +--- +title: Datahub Airflow Plugin +slug: /metadata-ingestion-modules/airflow-plugin +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/README.md +--- +# Datahub Airflow Plugin + +See [the DataHub Airflow docs](/docs/lineage/airflow) for details. + +## Version Compatibility + +The plugin supports Apache Airflow versions 2.7+ and 3.1+. + +| Airflow Version | Extra to Install | Status | Notes | +| --------------- | ---------------- | ---------------------- | -------------------------------- | +| 2.7-2.10 | `[airflow2]` | ✅ Fully Supported | | +| 3.0.x | `[airflow3]` | ⚠️ Requires manual fix | Needs `pydantic>=2.11.8` upgrade | +| 3.1+ | `[airflow3]` | ✅ Fully Supported | | + +**Note on Airflow 3.0.x**: Airflow 3.0.6 pins pydantic==2.11.7, which contains a bug that prevents the DataHub plugin from importing correctly. This issue is resolved in Airflow 3.1.0+ which uses pydantic>=2.11.8. If you must use Airflow 3.0.6, you can manually upgrade pydantic to >=2.11.8, though this may conflict with Airflow's dependency constraints. We recommend upgrading to Airflow 3.1.0 or later. + +Related issue: https://github.com/pydantic/pydantic/issues/10963 + +## Installation + +The installation command varies depending on your Airflow version due to different OpenLineage dependencies. + +### For Airflow 2.x (2.7+) + +```bash +pip install 'acryl-datahub-airflow-plugin[airflow2]' +``` + +This installs the plugin with Legacy OpenLineage (`openlineage-airflow>=1.2.0`), which is required for Airflow 2.x lineage extraction. + +#### Alternative: Using Native OpenLineage Provider on Airflow 2.7+ + +If your Airflow 2.7+ environment rejects the Legacy OpenLineage package (e.g., due to dependency conflicts), you can use the native OpenLineage provider instead: + +```bash +# Install the native Airflow provider first +pip install 'apache-airflow-providers-openlineage>=1.0.0' + +# Then install the DataHub plugin without OpenLineage extras +pip install acryl-datahub-airflow-plugin +``` + +The plugin will automatically detect and use `apache-airflow-providers-openlineage` when available, providing the same functionality. + +### For Airflow 3.x (3.1+) + +```bash +pip install 'acryl-datahub-airflow-plugin[airflow3]' +``` + +This installs the plugin with `apache-airflow-providers-openlineage>=1.0.0`, which is the native OpenLineage provider for Airflow 3.x. + +**Note**: If using Airflow 3.0.x (3.0.6 specifically), you'll need to manually upgrade pydantic: + +```bash +pip install 'acryl-datahub-airflow-plugin[airflow3]' 'pydantic>=2.11.8' +``` + +We recommend using Airflow 3.1.0+ which resolves this issue. See the Version Compatibility section above for details. + +### What Gets Installed + +#### Base Installation (No Extras) + +When you install without any extras: + +```bash +pip install acryl-datahub-airflow-plugin +``` + +You get: + +- `acryl-datahub[sql-parser,datahub-rest]` - DataHub SDK with SQL parsing and REST emitter +- `pydantic>=2.4.0` - Required for data validation +- `apache-airflow>=2.5.0,<4.0.0` - Airflow itself +- **No OpenLineage package** - You'll need to provide your own or use one of the extras below + +#### With `[airflow2]` Extra + +```bash +pip install 'acryl-datahub-airflow-plugin[airflow2]' +``` + +Adds: + +- `openlineage-airflow>=1.2.0` - Standalone OpenLineage package for Airflow 2.x + +#### With `[airflow3]` Extra + +```bash +pip install 'acryl-datahub-airflow-plugin[airflow3]' +``` + +Adds: + +- `apache-airflow-providers-openlineage>=1.0.0` - Native OpenLineage provider for Airflow 3.x + +### Additional Extras + +You can combine multiple extras if needed: + +```bash +# For Airflow 3.x with Kafka emitter support +pip install 'acryl-datahub-airflow-plugin[airflow3,datahub-kafka]' + +# For Airflow 2.x with file emitter support +pip install 'acryl-datahub-airflow-plugin[airflow2,datahub-file]' +``` + +Available extras: + +- `airflow2`: OpenLineage support for Airflow 2.x (adds `openlineage-airflow>=1.2.0`) +- `airflow3`: OpenLineage support for Airflow 3.x (adds `apache-airflow-providers-openlineage>=1.0.0`) +- `datahub-kafka`: Kafka-based metadata emission (adds `acryl-datahub[datahub-kafka]`) +- `datahub-file`: File-based metadata emission (adds `acryl-datahub[sync-file-emitter]`) - useful for testing + +### Why Different Extras? + +Airflow 2.x and 3.x have different OpenLineage integrations: + +- **Airflow 2.x (2.5-2.6)** typically uses Legacy OpenLineage (`openlineage-airflow` package) +- **Airflow 2.x (2.7+)** can use either Legacy OpenLineage or native OpenLineage Provider (`apache-airflow-providers-openlineage`) +- **Airflow 3.x** uses native OpenLineage Provider (`apache-airflow-providers-openlineage`) + +The plugin automatically detects which OpenLineage variant is installed and uses it accordingly. This means: + +1. **With extras** (`[airflow2]` or `[airflow3]`): The appropriate OpenLineage dependency is installed automatically +2. **Without extras**: You provide your own OpenLineage installation, and the plugin auto-detects it + +This flexibility allows you to adapt to different Airflow environments and dependency constraints. + +## Configuration + +The plugin can be configured via `airflow.cfg` under the `[datahub]` section. Below are the key configuration options: + +### Extractor Patching (OpenLineage Enhancements) + +When `enable_extractors=True` (default), the DataHub plugin enhances OpenLineage extractors to provide better lineage. You can fine-tune these enhancements: + +```ini +[datahub] +# Enable/disable all OpenLineage extractors +enable_extractors = True # Default: True + +# Fine-grained control over DataHub's OpenLineage enhancements + +# --- SQL Parsing Configuration --- + +# Enable multi-statement SQL parsing (resolves temp tables, merges lineage) +enable_multi_statement_sql_parsing = False # Default: False + +# --- Patches (work with both Legacy OpenLineage and OpenLineage Provider) --- + +# Patch SqlExtractor to use DataHub's advanced SQL parser (enables column-level lineage) +patch_sql_parser = True # Default: True + +# Patch SnowflakeExtractor to fix default schema detection +patch_snowflake_schema = True # Default: True + +# --- Custom Extractors (only apply to Legacy OpenLineage) --- + +# Use DataHub's custom AthenaOperatorExtractor (better Athena lineage) +extract_athena_operator = True # Default: True + +# Use DataHub's custom BigQueryInsertJobOperatorExtractor (handles BQ job configuration) +extract_bigquery_insert_job_operator = True # Default: True +``` + +**Multi-Statement SQL Parsing:** + +When `enable_multi_statement_sql_parsing=True`, if a task executes multiple SQL statements (e.g., `CREATE TEMP TABLE ...; INSERT ... FROM temp_table;`), DataHub parses all statements together and resolves temporary table dependencies within that task. By default (False), only the first statement is parsed. + +**How it works:** + +**Patches** (apply to both Legacy OpenLineage and OpenLineage Provider): + +- Apply **monkey-patching** to OpenLineage extractor/operator classes at runtime +- Work on **both Airflow 2.x and Airflow 3.x** +- When `patch_sql_parser=True`: + - **Airflow 2**: Patches `SqlExtractor.extract()` method + - **Airflow 3**: Patches `SQLParser.generate_openlineage_metadata_from_sql()` method + - Provides: More accurate lineage extraction, column-level lineage (CLL), better SQL dialect support +- When `patch_snowflake_schema=True`: + - **Airflow 2**: Patches `SnowflakeExtractor.default_schema` property + - **Airflow 3**: Currently not needed (handled by Airflow's native support) + - Fixes Snowflake schema detection issues + +**Custom Extractors/Operator Patches**: + +- Register DataHub's custom implementations for specific operators +- Work on **both Airflow 2.x and Airflow 3.x** +- `extract_athena_operator`: + - **Airflow 2 (Legacy OpenLineage only)**: Registers `AthenaOperatorExtractor` + - **Airflow 3**: Patches `AthenaOperator.get_openlineage_facets_on_complete()` + - Uses DataHub's SQL parser for better Athena lineage +- `extract_bigquery_insert_job_operator`: + - **Airflow 2 (Legacy OpenLineage only)**: Registers `BigQueryInsertJobOperatorExtractor` + - **Airflow 3**: Patches `BigQueryInsertJobOperator.get_openlineage_facets_on_complete()` + - Handles BigQuery job configuration and destination tables + +**Example use cases:** + +Disable DataHub's SQL parser to use OpenLineage's native parsing: + +```ini +[datahub] +enable_extractors = True +patch_sql_parser = False # Use OpenLineage's native SQL parser +patch_snowflake_schema = True # Still fix Snowflake schema detection +``` + +Disable custom Athena extractor (only relevant for Legacy OpenLineage): + +```ini +[datahub] +enable_extractors = True +extract_athena_operator = False # Use OpenLineage's default Athena extractor +``` + +### Other Configuration Options + +For a complete list of configuration options, see the [DataHub Airflow documentation](/docs/lineage/airflow#configuration). + +## Developing + +See the [developing docs](../../metadata-ingestion/developing.md). diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/scripts/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/scripts/README.md new file mode 100644 index 00000000..f90b6c84 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/scripts/README.md @@ -0,0 +1,15 @@ +--- +title: Scripts +slug: /metadata-ingestion-modules/airflow-plugin/scripts +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/scripts/README.md +--- +# Scripts + +This directory contains utility scripts for the DataHub Airflow plugin. + +## Scripts + +### release.sh + +Release script for publishing the plugin (pre-existing). diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/README.md new file mode 100644 index 00000000..f8e83cbf --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/README.md @@ -0,0 +1,152 @@ +--- +title: DataHub Airflow Plugin Example DAGs +sidebar_label: Airflow Plugin Example DAGs +slug: >- + /metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/README.md +--- +# DataHub Airflow Plugin Example DAGs + +This directory contains example DAGs demonstrating various features of the DataHub Airflow plugin. + +## Directory Structure + +``` +example_dags/ +├── airflow2/ # Airflow 2.x examples (with compatibility layers) +│ ├── lineage_backend_demo.py +│ ├── lineage_backend_taskflow_demo.py +│ ├── snowflake_sample_dag.py +│ ├── mysql_sample_dag.py +│ ├── generic_recipe_sample_dag.py +│ ├── lineage_emission_dag.py +│ └── graph_usage_sample_dag.py +│ +├── airflow3/ # Airflow 3.0+ examples (native syntax, no compatibility) +│ ├── lineage_backend_demo.py +│ ├── lineage_backend_taskflow_demo.py +│ └── snowflake_sample_dag.py +│ +└── *.py # Legacy examples (deprecated - use airflow2/ or airflow3/) +``` + +## Choosing the Right Examples + +### For Airflow 3.0+ Users + +Use the examples in `airflow3/`. These DAGs: + +- Use native Airflow 3.0 syntax (`schedule` instead of `schedule_interval`) +- Don't include compatibility layers +- Are simpler and easier to read +- Demonstrate best practices for Airflow 3.0+ + +### For Airflow 2.x Users + +Use the examples in `airflow2/`. These DAGs: + +- Work across Airflow 2.3 through 2.9 +- Include compatibility helpers from `_airflow_version_specific.py` +- Handle parameter name changes between versions + +### For Cross-Version Support + +If you need DAGs that work on both Airflow 2.x and 3.x: + +- See `airflow2/` examples for patterns using `get_airflow_compatible_dag_kwargs()` +- Use `days_ago()` helper for start_date compatibility +- Import hooks conditionally based on Airflow version + +## Example Categories + +### 1. Lineage Collection + +Examples showing how to automatically collect lineage from your Airflow DAGs: + +- **lineage_backend_demo.py**: Basic lineage collection using `inlets` and `outlets` +- **lineage_backend_taskflow_demo.py**: Lineage with TaskFlow API (`@task` decorator) + +### 2. Metadata Ingestion + +Examples showing how to ingest metadata from data sources into DataHub: + +- **snowflake_sample_dag.py**: Ingest Snowflake metadata +- **mysql_sample_dag.py**: Ingest MySQL metadata +- **generic_recipe_sample_dag.py**: Run any DataHub ingestion recipe + +### 3. Advanced Features + +- **lineage_emission_dag.py**: Directly emit custom lineage using `DatahubEmitterOperator` +- **graph_usage_sample_dag.py**: Complex DAG graphs with multiple task dependencies + +## Key Differences: Airflow 2 vs 3 + +### DAG Parameters + +```python +# Airflow 2.x +DAG( + schedule_interval=timedelta(days=1), + default_view="tree", # Removed in Airflow 3 +) + +# Airflow 3.0+ +DAG( + schedule=timedelta(days=1), + # default_view parameter removed +) +``` + +### Hooks Import + +```python +# Airflow 2.x +from airflow.hooks.base import BaseHook + +# Airflow 3.0+ +from airflow.hooks.base_hook import BaseHook +``` + +### Start Date + +```python +# Airflow 2.x (compatibility helper) +from datahub_airflow_plugin._airflow_version_specific import days_ago +start_date=days_ago(2) + +# Airflow 3.0+ (direct) +from datetime import datetime +start_date=datetime(2023, 1, 1) +``` + +## Running the Examples + +1. Copy the appropriate example to your Airflow DAGs folder: + + ```bash + # For Airflow 3.0+ + cp airflow3/lineage_backend_demo.py $AIRFLOW_HOME/dags/ + + # For Airflow 2.x + cp airflow2/lineage_backend_demo.py $AIRFLOW_HOME/dags/ + ``` + +2. Configure the DataHub connection in Airflow: + + ```bash + airflow connections add datahub_rest_default \ + --conn-type datahub-rest \ + --conn-host http://localhost:8080 + ``` + +3. Enable the DAG in the Airflow UI or trigger it manually: + ```bash + airflow dags trigger datahub_lineage_backend_demo + ``` + +## Additional Resources + +- [DataHub Airflow Plugin Documentation](https://datahubproject.io/docs/lineage/airflow/) +- [Airflow 3.0 Migration Guide](https://airflow.apache.org/docs/apache-airflow/stable/migration-guide.html) +- [DataHub Metadata Ingestion](https://datahubproject.io/docs/metadata-ingestion/) diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/dagster-plugin/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/dagster-plugin/README.md new file mode 100644 index 00000000..9575af0d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/dagster-plugin/README.md @@ -0,0 +1,9 @@ +--- +title: Datahub Dagster Plugin +slug: /metadata-ingestion-modules/dagster-plugin +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/dagster-plugin/README.md +--- +# Datahub Dagster Plugin + +See the [DataHub Dagster docs](/docs/lineage/dagster/) for details. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/gx-plugin/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/gx-plugin/README.md new file mode 100644 index 00000000..7e3956a3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/gx-plugin/README.md @@ -0,0 +1,9 @@ +--- +title: Datahub GX Plugin +slug: /metadata-ingestion-modules/gx-plugin +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/gx-plugin/README.md +--- +# Datahub GX Plugin + +See the [DataHub GX docs](/docs/metadata-ingestion/integration_docs/great-expectations) for details. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/prefect-plugin/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/prefect-plugin/README.md new file mode 100644 index 00000000..b23bcf3a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion-modules/prefect-plugin/README.md @@ -0,0 +1,138 @@ +--- +title: prefect-datahub +slug: /metadata-ingestion-modules/prefect-plugin +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/prefect-plugin/README.md +--- +# prefect-datahub + +Emit flows & tasks metadata to DataHub REST API with `prefect-datahub` + +

+ + PyPI + + + + + + +
+ + +

+ +## Introduction + +The `prefect-datahub` collection allows you to easily integrate DataHub's metadata ingestion capabilities into your Prefect workflows. With this collection, you can emit metadata about your flows, tasks, and workspace to DataHub's metadata service. + +## Features + +- Seamless integration with Prefect workflows +- Support for ingesting metadata of flows, tasks, and workspaces to DataHub GMS REST API +- Easy configuration using Prefect blocks + +## Prerequisites + +- Python 3.10+ +- Prefect 2.0.0+ and < 3.0.0+ +- A running instance of DataHub + +## Installation + +Install `prefect-datahub` using pip: + +```bash +pip install prefect-datahub +``` + +We recommend using a Python virtual environment manager such as pipenv, conda, or virtualenv. + +## Getting Started + +### 1. Set up DataHub + +Before using `prefect-datahub`, you need to deploy an instance of DataHub. Follow the instructions on the [DataHub Quickstart page](/docs/quickstart) to set up DataHub. + +After successful deployment, the DataHub GMS service should be running on `http://localhost:8080` if deployed locally. + +### 2. Configure DataHub Emitter + +Save your DataHub configuration as a Prefect block: + +```python +from prefect_datahub.datahub_emitter import DatahubEmitter + +datahub_emitter = DatahubEmitter( + datahub_rest_url="http://localhost:8080", + env="DEV", + platform_instance="local_prefect", + token=None, # generate auth token in the datahub and provide here if gms endpoint is secure +) +datahub_emitter.save("datahub-emitter-test") +``` + +Configuration options: + +| Config | Type | Default | Description | +| ----------------- | ----- | ----------------------- | ---------------------------------------------------------------------------------------------------------- | +| datahub_rest_url | `str` | `http://localhost:8080` | DataHub GMS REST URL | +| env | `str` | `PROD` | Environment for assets (see [FabricType](/docs/graphql/enums/#fabrictype)) | +| platform_instance | `str` | `None` | Platform instance for assets (see [Platform Instances](/docs/platform-instances/)) | + +### 3. Use DataHub Emitter in Your Workflows + +Here's an example of how to use the DataHub Emitter in a Prefect workflow: + +```python +from prefect import flow, task +from prefect_datahub.datahub_emitter import DatahubEmitter +from prefect_datahub.entities import Dataset + +datahub_emitter_block = DatahubEmitter.load("datahub-emitter-test") + +@task(name="Extract", description="Extract the data") +def extract(): + return "This is data" + +@task(name="Transform", description="Transform the data") +def transform(data, datahub_emitter): + transformed_data = data.split(" ") + datahub_emitter.add_task( + inputs=[Dataset("snowflake", "mydb.schema.tableX")], + outputs=[Dataset("snowflake", "mydb.schema.tableY")], + ) + return transformed_data + +@flow(name="ETL", description="Extract transform load flow") +def etl(): + datahub_emitter = datahub_emitter_block + data = extract() + transformed_data = transform(data, datahub_emitter) + datahub_emitter.emit_flow() + +if __name__ == "__main__": + etl() +``` + +**Note**: To emit task metadata, you must call `emit_flow()` at the end of your flow. Otherwise, no metadata will be emitted. + +## Advanced Usage + +For more advanced usage and configuration options, please refer to the [prefect-datahub documentation](/docs/lineage/prefect/). + +## Contributing + +We welcome contributions to `prefect-datahub`! Please refer to our [Contributing Guidelines](/docs/contributing) for more information on how to get started. + +## Support + +If you encounter any issues or have questions, you can: + +- Open an issue in the [DataHub GitHub repository](https://github.com/datahub-project/datahub/issues) +- Join the [DataHub Slack community](https://datahubspace.slack.com) +- Seek help in the [Prefect Slack community](https://prefect.io/slack) + +## License + +`prefect-datahub` is released under the Apache 2.0 license. See the [LICENSE](https://github.com/datahub-project/datahub/blob/master/LICENSE) file for details. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/CLAUDE.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/CLAUDE.md new file mode 100644 index 00000000..a0cfc090 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/CLAUDE.md @@ -0,0 +1,141 @@ +--- +title: DataHub Metadata Ingestion Development Guide +sidebar_label: Metadata Ingestion Development Guide +slug: /metadata-ingestion/claude +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/CLAUDE.md +--- +# DataHub Metadata Ingestion Development Guide + +## Build and Test Commands + +**Using Gradle (slow but reliable):** + +```bash +# Development setup from repository root +../gradlew :metadata-ingestion:installDev # Setup Python environment +source venv/bin/activate # Activate virtual environment + +# Linting and formatting +../gradlew :metadata-ingestion:lint # Run ruff + mypy +../gradlew :metadata-ingestion:lintFix # Auto-fix linting issues + +# Testing +../gradlew :metadata-ingestion:testQuick # Fast unit tests +../gradlew :metadata-ingestion:testFull # All tests +../gradlew :metadata-ingestion:testSingle -PtestFile=tests/unit/test_file.py # Single test +``` + +**Direct Python commands (when venv is activated):** + +```bash +# Linting +ruff format src/ tests/ +ruff check src/ tests/ +mypy src/ tests/ + +# Testing +pytest -vv # Run all tests +pytest -m 'not integration' # Unit tests only +pytest -m 'integration' # Integration tests +pytest tests/path/to/file.py # Single test file +pytest tests/path/to/file.py::TestClass # Single test class +pytest tests/path/to/file.py::TestClass::test_method # Single test +``` + +## Environment Variables + +**Build configuration:** + +- `DATAHUB_VENV_USE_COPIES`: Set to `true` to use `--copies` flag when creating Python virtual environments. This copies the Python binary instead of creating a symlink. Useful for Nix environments, immutable filesystems, Windows, or container environments where symlinks don't work correctly. Increases disk usage and setup time, so only enable if needed. + + ```bash + export DATAHUB_VENV_USE_COPIES=true + ../gradlew :metadata-ingestion:installDev + ``` + +## Directory Structure + +- `src/datahub/`: Source code for the DataHub CLI and ingestion framework +- `tests/`: All tests (NOT in `src/` directory) +- `tests/unit/`: Unit tests +- `tests/integration/`: Integration tests +- `scripts/`: Build and development scripts +- `examples/`: Example ingestion configurations +- `developing.md`: Detailed development environment information + +## Code Style Guidelines + +- **Formatting**: Uses ruff, 88 character line length +- **Imports**: Sorted with ruff.lint.isort, no relative imports, Imports are always put at the top of the file, just after any module comments and docstrings, and before module globals and constants. +- **Types**: Always use type annotations, prefer Protocol for interfaces + - Avoid `Any` type - use specific types (`Dict[str, int]`, `TypedDict`, or typevars) + - Use `isinstance` checks instead of `hasattr` + - Prefer `assert isinstance(...)` over `cast` +- **Data Structures**: Use dataclasses/pydantic for internal data representation + - Return dataclasses instead of tuples from methods + - Centralize utility functions to avoid code duplication +- **Naming**: Descriptive names, match source system terminology in configs +- **Error Handling**: Validators throw only ValueError/TypeError/AssertionError + - Add robust error handling with layers of protection for known failure points +- **Code Quality**: Avoid global state, use named arguments, don't re-export in `__init__.py` +- **Documentation**: All configs need descriptions +- **Dependencies**: Avoid version pinning, use ranges with comments +- **Architecture**: Avoid tall inheritance hierarchies, prefer mixins + +## Testing Conventions + +- **Location**: Tests go in `tests/` directory alongside `src/`, NOT in `src/` +- **Structure**: Test files should mirror the source directory structure +- **Framework**: Use pytest, not unittest +- **Assertions**: Use `assert` statements, not `self.assertEqual()` or `self.assertIsNone()` + - Boolean: `assert func()` or `assert not func()`, not `assert func() is True/False` + - None: `assert result is None` (correct), not `assert result == None` + - Keep tests concise, avoid verbose repetitive patterns +- **Classes**: Use regular classes, not `unittest.TestCase` +- **Imports**: Import `pytest` in test files +- **Naming**: Test files should be named `test_*.py` +- **Categories**: + - Unit tests: `tests/unit/` - fast, no external dependencies + - Integration tests: `tests/integration/` - may use Docker/external services + +## Configuration Guidelines (Pydantic) + +- **Naming**: Match terminology of the source system (e.g., `account_id` for Snowflake, not `host_port`) +- **Descriptions**: All configs must have descriptions +- **Patterns**: Use AllowDenyPatterns for filtering, named `*_pattern` +- **Defaults**: Set reasonable defaults, avoid config-driven filtering that should be automatic +- **Validation**: Single pydantic validator per validation concern +- **Security**: Use `SecretStr` for passwords, auth tokens, etc. +- **Deprecation**: Use `pydantic_removed_field` helper for field deprecations + +## Key Files + +- `src/datahub/emitter/mcp_builder.py`: Examples of defining various aspect types +- `setup.py`, `pyproject.toml`, `setup.cfg`: Code style and dependency configuration +- `.github/workflows/metadata-ingestion.yml`: CI workflow configuration + +## Connector Documentation + +Connector documentation follows a structured format with specific file types and heading conventions. + +**Authoring guidelines**: See `docs/sources/AGENTS.md` in this directory for: + +- File structure (`_pre.md`, `_post.md`, `_recipe.yml`) +- Heading level rules (H3 baseline, H2 reserved for module headings) +- Section ordering (Overview → Concept Mapping → Prerequisites) +- Content placement guidelines +- Style conventions and formatting requirements + +**Quick reference:** + +```bash +# Regenerate documentation after changes +./gradlew :metadata-ingestion:docGen + +# Preview locally +./gradlew :docs-website:yarnStart # Opens localhost:3001 + +# Format markdown before committing +./gradlew :datahub-web-react:mdPrettierWrite +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/KAFKA_CONNECT_LINEAGE.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/KAFKA_CONNECT_LINEAGE.md new file mode 100644 index 00000000..7b7a7573 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/KAFKA_CONNECT_LINEAGE.md @@ -0,0 +1,532 @@ +--- +title: Kafka Connect Lineage Extraction - Production Architecture +slug: /metadata-ingestion/kafka_connect_lineage +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/KAFKA_CONNECT_LINEAGE.md +--- +# Kafka Connect Lineage Extraction - Production Architecture + +## Overview + +DataHub extracts lineage from Kafka Connect by mapping source tables to Kafka topics. The current implementation provides **production-ready** support for both **Confluent Cloud** and **Self-hosted Kafka Connect** environments with comprehensive type safety, robust error handling, and extensive test coverage. + +## Production Architecture + +### Key Components + +#### 1. Type-Safe Factory Pattern Implementation + +**Connector Factory** (`common.py`): + +- **✅ PRODUCTION READY**: Type-safe connector instantiation with full MyPy compliance +- **Factory Methods**: + - `extract_lineages()`: Creates connector instance and extracts lineages + - `_get_connector_class_type()`: Determines connector type from configuration + - `_get_source_connector_type()`: Routes to appropriate source connector class + - `_get_sink_connector_type()`: Routes to appropriate sink connector class + +**JDBC Configuration Parsing** (`source_connectors.py`): + +- **✅ IMPLEMENTED**: Unified parsing for Platform and Cloud configurations +- **Purpose**: Handles both Platform (`connection.url`) and Cloud (individual fields) configurations +- **Features**: Robust URL validation, quoted identifier support, comprehensive error handling + +#### 2. Connector Class Architecture + +**Source Connectors**: + +- **ConfluentJDBCSourceConnector** - JDBC connectors (Platform & Cloud) +- **DebeziumSourceConnector** - CDC connectors (MySQL, PostgreSQL, etc.) +- **MongoSourceConnector** - MongoDB source connectors + +**Sink Connectors**: + +- **BigQuerySinkConnector** - BigQuery sink with table name sanitization +- **ConfluentS3SinkConnector** - S3 sink connector +- **SnowflakeSinkConnector** - Snowflake sink connector + +#### 3. Environment-Aware Lineage Extraction + +**✅ IMPLEMENTED**: Environment detection and strategy selection + +- **Cloud Detection**: Uses `CLOUD_JDBC_SOURCE_CLASSES` for automatic detection +- **Strategy Selection**: + - Cloud: Config-based inference with prefix matching fallback + - Platform: API-based topic retrieval with transform pipeline + +```python +def _extract_lineages_with_environment_awareness(self, parser: JdbcParser) -> List[KafkaConnectLineage]: + connector_class = self.connector_manifest.config.get(CONNECTOR_CLASS, "") + is_cloud_environment = connector_class in CLOUD_JDBC_SOURCE_CLASSES + + if is_cloud_environment: + return self._extract_lineages_cloud_environment(parser) + else: + return self._extract_lineages_platform_environment(parser) +``` + +#### 4. Transform Pipeline + +**✅ IMPLEMENTED**: `TransformPipeline` class with forward transform application + +- **Supported Transforms**: + - `RegexRouter` - Pattern-based topic renaming (✅ Working) + - `EventRouter` - Outbox pattern for CDC (⚠️ Limited - warns about unpredictability) +- **Features**: + - Forward pipeline: Source tables → transforms → final topics + - Connector-specific topic naming strategies + - Java regex compatibility for exact Kafka Connect behavior + +#### 5. BigQuery Sink Enhancements + +**✅ IMPLEMENTED**: Official Kafka Connect compatible table name sanitization + +- **Follows**: Aiven and Confluent BigQuery connector implementations +- **Rules**: Invalid character replacement, digit handling, length limits +- **✅ COMPREHENSIVE TESTING**: 15 test methods covering all edge cases + +#### 6. Centralized Constants + +**✅ IMPLEMENTED**: `connector_constants.py` module + +- **Contents**: + - Connector class constants + - Transform type classifications + - Platform-specific constants (2-level container detection) + - Utility functions for transform classification + +#### 7. Advanced Type Safety Implementation + +**✅ PRODUCTION EXCELLENCE**: Full type annotation coverage with 100% MyPy compliance + +**Type Safety Features**: + +- **Function Signatures**: Every function has complete parameter and return type annotations +- **Generic Types**: Proper use of `List[str]`, `Dict[str, str]`, `Optional[T]` throughout +- **Union Types**: Explicit handling of multiple possible types with `Union[]` +- **Type Guards**: Runtime type checking with `isinstance()` and proper type narrowing +- **Protocol Usage**: Interface definitions for extensible architecture +- **Dataclass Integration**: Structured data with automatic type validation + +**Benefits for Developers**: + +- **IDE Support**: Full autocomplete, type hints, and error detection in VS Code/PyCharm +- **Runtime Safety**: Early detection of type mismatches during development +- **Documentation**: Type annotations serve as inline documentation +- **Refactoring Safety**: Confident code changes with type-aware refactoring tools +- **Team Collaboration**: Clear contracts between functions and modules + +**Example Type Safety Implementation**: + +```python +from typing import Dict, List, Optional, Union +from dataclasses import dataclass + +@dataclass +class ConnectorManifest: + name: str + type: str + config: Dict[str, str] + tasks: List[Dict[str, dict]] + topic_names: List[str] = field(default_factory=list) + + def extract_lineages( + self, + config: "KafkaConnectSourceConfig", + report: "KafkaConnectSourceReport" + ) -> List[KafkaConnectLineage]: + """Type-safe lineage extraction with full annotation coverage.""" + connector_class_type = self._get_connector_class_type() + if not connector_class_type: + return [] + + connector_instance = connector_class_type(self, config, report) + return connector_instance.extract_lineages() +``` + +**MyPy Compliance**: + +- ✅ **0 errors** across all 9 source files (5,713+ lines of code) +- ✅ **Strict mode compatible** with comprehensive type checking +- ✅ **CI/CD integrated** with automated type checking in build pipeline + +## Lineage Matching Process Flow + +### Source Connector Flow + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Database │ │ Kafka Connect │ │ Kafka Topics │ +│ │ │ Connector │ │ │ +│ ┌─────────────┐ │ │ │ │ ┌─────────────┐ │ +│ │ schema.users│ │───▶│ Extract Config │───▶│ │finance_users│ │ +│ │schema.orders│ │ │ │ │ │finance_orders│ │ +│ │schema.items │ │ │ Apply Transforms│ │ │finance_items │ │ +│ └─────────────┘ │ │ (RegexRouter) │ │ └─────────────┘ │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│Source Dataset │ │ Lineage Mapping │ │Target Dataset │ +│ │ │ │ │ │ +│mydb.schema.users│◀───┤ Source → Topic ├───▶│ kafka:finance_ │ +│mydb.schema.orders│ │ │ │ users │ +│mydb.schema.items│ │ DataHub Lineage │ │ kafka:finance_ │ +└─────────────────┘ │ Representation │ │ orders │ + └──────────────────┘ │ kafka:finance_ │ + │ items │ + └─────────────────┘ +``` + +### Sink Connector Flow (Reverse Direction) + +``` +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│ Kafka Topics │ │ Kafka Connect │ │ Target System │ +│ │ │ Connector │ │ │ +│ ┌─────────────┐ │ │ │ │ ┌─────────────┐ │ +│ │ user_events│ │───▶│ Topic Config │───▶│ │ users │ │ +│ │order_events │ │ │ │ │ │ orders │ │ +│ │product_data │ │ │ Table Mapping │ │ │ products │ │ +│ └─────────────┘ │ │ (Sanitization) │ │ └─────────────┘ │ +└─────────────────┘ └──────────────────┘ └─────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐ +│Source Dataset │ │ Lineage Mapping │ │Target Dataset │ +│ │ │ │ │ │ +│kafka:user_events│───▶┤ Topic → Table ├───▶│bq:project. │ +│kafka:order_events│ │ │ │ dataset.users │ +│kafka:product_data│ │ DataHub Lineage │ │bq:project. │ +└─────────────────┘ │ Representation │ │ dataset.orders│ + └──────────────────┘ │bq:project. │ + │ dataset.products│ + └─────────────────┘ +``` + +### Environment-Specific Matching Strategies + +#### Self-hosted Kafka Connect + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Self-hosted Environment │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌──────────────────┐ ┌──────────────┐ │ +│ │ Connector │───▶│ Connect API Call │───▶│ Actual Topics│ │ +│ │ Configuration│ │/connectors/{name}│ │ List │ │ +│ └──────────────┘ │ /topics │ └──────────────┘ │ +│ │ └──────────────────┘ │ │ +│ ▼ ▼ │ +│ ┌──────────────┐ ┌─────────────────────────────────┐ │ +│ │Parse Source │ │ Direct Topic Mapping │ │ +│ │Tables/Config │──────────▶│ (Highest Accuracy: 95-98%) │ │ +│ └──────────────┘ └─────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +#### Confluent Cloud Environment + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Confluent Cloud Environment │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌──────────────────┐ ┌──────────────┐ │ +│ │ Connector │───▶│Transform Pipeline│───▶│Predicted │ │ +│ │Configuration │ │ Prediction │ │Topics │ │ +│ └──────────────┘ └──────────────────┘ └──────────────┘ │ +│ │ │ │ │ +│ ▼ ▼ ▼ │ +│ ┌──────────────┐ ┌──────────────────┐ ┌──────────────┐ │ +│ │Parse Source │ │ Kafka REST │ │ Validate & │ │ +│ │Tables/Config │ │ API v3 Call │ │ Filter │ │ +│ └──────────────┘ │ (All Topics) │ │ Topics │ │ +│ └──────────────────┘ └──────────────┘ │ +│ │ │ │ +│ ▼ ▼ │ +│ ┌─────────────────────────────────────────┐ │ +│ │ Transform-Aware Strategy │ │ +│ │ (Accuracy: 90-95% with fallback) │ │ +│ └─────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Transform Processing Pipeline + +``` +Original Source Tables Transform Pipeline Final Topics +┌─────────────────┐ ┌─────────────────────┐ ┌─────────────────┐ +│ │ │ │ │ │ +│ schema.users │─────▶│ 1. Generate │───▶│ finance_users │ +│ schema.orders │ │ Original │ │ finance_orders │ +│ schema.products │ │ Topic Names │ │ finance_products│ +└─────────────────┘ │ │ └─────────────────┘ + │ 2. Apply Regex │ +Topic Prefix: "finance_" │ Router │ RegexRouter Applied: +Table Include List │ Transform │ "finance_(.*)" → "$1" + │ │ + │ 3. Apply Other │ ┌─────────────────┐ + │ Transforms │───▶│ users │ + │ (if supported) │ │ orders │ + └─────────────────────┘ │ products │ + └─────────────────┘ +``` + +### Handler Selection Logic + +``` +Connector Class Detection + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ Handler Selection │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ "io.confluent.connect.jdbc.JdbcSourceConnector" │ +│ │ │ +│ ▼ │ +│ ┌──────────────────┐ │ +│ │JDBCSourceTopic │ │ +│ │Handler │ │ +│ └──────────────────┘ │ +│ │ +│ "io.debezium.connector.mysql.MySqlConnector" │ +│ │ │ +│ ▼ │ +│ ┌──────────────────┐ │ +│ │DebeziumSource │ │ +│ │TopicHandler │ │ +│ └──────────────────┘ │ +│ │ +│ "PostgresCdcSource" (Cloud) │ +│ │ │ +│ ▼ │ +│ ┌──────────────────┐ │ +│ │CloudJDBCSource │ │ +│ │TopicHandler │ │ +│ └──────────────────┘ │ +│ │ +│ Unknown Connector │ +│ │ │ +│ ▼ │ +│ ┌──────────────────┐ │ +│ │GenericConnector │ │ +│ │TopicHandler │ │ +│ └──────────────────┘ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Current Lineage Extraction Strategies + +### Strategy 1: Environment-Aware Extraction (Primary) + +**✅ CURRENTLY ACTIVE**: Automatic environment detection and strategy selection + +**Self-hosted Environment**: + +1. **API-Based Resolution**: Uses `/connectors/{name}/topics` endpoint +2. **Transform Application**: Applies configured transforms to actual topics +3. **Direct Mapping**: Creates lineage from actual topics to source tables + +**Confluent Cloud Environment**: + +1. **Transform-Aware Resolution**: Applies transform pipelines to predict expected topics +2. **Topic Validation**: Validates predicted topics against actual cluster topics from Kafka REST API +3. **Config-Based Fallback**: Falls back to configuration-based inference when transforms fail +4. **1:1 Mapping Detection**: Handles explicit table-to-topic mappings + +### Strategy 2: Transform Pipeline Processing + +**✅ IMPLEMENTED**: Forward transform pipeline with predictable transforms only + +**Process**: + +1. Extract source tables from configuration +2. Generate original topic names using connector-specific naming +3. Apply RegexRouter transforms (other transforms skipped with warnings) +4. Create lineage mappings from sources to final topics + +**Transform Support**: + +- **✅ RegexRouter**: Full support with Java regex compatibility +- **⚠️ EventRouter**: Warns about unpredictability, provides safe fallback +- **❌ Custom Transforms**: Recommends explicit `generic_connectors` mapping + +### Strategy 3: Cloud Transform Pipeline (New) + +**✅ NEW FEATURE**: Transform-aware lineage extraction for Confluent Cloud connectors + +**Key Capabilities**: + +- **Full Transform Support**: Cloud connectors now support complete transform pipelines (previously missing) +- **Source Table Extraction**: Extracts tables from Cloud connector configuration (`table.include.list`, `query` modes) +- **Forward Transform Application**: Applies RegexRouter and other transforms to predict expected topics +- **Topic Validation**: Validates predicted topics against actual cluster topics from Kafka REST API +- **Graceful Fallback**: Falls back to config-based strategies when transforms can't be applied + +**Implementation Details**: + +```python +def _extract_lineages_cloud_with_transforms( + self, all_topics: List[str], parser: JdbcParser +) -> List[KafkaConnectLineage]: + """Cloud-specific transform-aware lineage extraction.""" + source_tables = self._get_source_tables_from_config() + expected_topics = self._apply_forward_transforms(source_tables, parser) + connector_topics = [topic for topic in expected_topics if topic in all_topics] + # Create lineages from source tables to validated topics + return self._create_lineages_from_tables_to_topics(source_tables, connector_topics, parser) +``` + +**Benefits**: + +- **90-95% Accuracy**: Significant improvement over previous config-only approach (80-85%) +- **Complex Transform Support**: Handles multi-step RegexRouter transforms correctly +- **Schema Preservation**: Maintains full schema information (e.g., `public.users`, `inventory.products`) +- **Production Ready**: 8 comprehensive test methods covering all scenarios + +### Strategy 4: Graceful Fallback Hierarchy + +**✅ IMPLEMENTED**: Multiple fallback levels for reliability + +1. **Primary**: Cloud transform-aware extraction (for Cloud connectors) +2. **Secondary**: Environment-aware extraction +3. **Tertiary**: Unified configuration-based approach +4. **Final**: Default lineage extraction with warnings + +## Production Features & Quality Metrics + +### ✅ **Production-Ready Implementation** + +1. **Type-Safe Architecture**: 100% type annotation coverage with MyPy compliance (0 errors) +2. **Factory Pattern Implementation**: Clean separation of concerns with connector-specific factories +3. **Comprehensive Testing**: 117 test methods across 27 test classes (3,799 lines of tests with comprehensive coverage across all connector types) +4. **Environment Detection**: Automatic Cloud vs Platform detection and strategy selection +5. **Transform Pipeline**: Fully functional forward transform pipeline with Java regex compatibility +6. **BigQuery Sink Enhancement**: Official Kafka Connect compatible table name sanitization +7. **Robust Error Handling**: 124+ try/catch blocks with graceful degradation +8. **Comprehensive Logging**: 138+ structured log statements for monitoring and debugging + +### 📊 **Quality Metrics** + +| **Metric** | **Value** | **Status** | +| -------------------- | --------------------------------- | ------------------- | +| **Lines of Code** | 5,713+ lines across 9 files | ✅ Production Scale | +| **Type Safety** | 0 MyPy errors | ✅ Full Compliance | +| **Test Coverage** | 117 test methods, 27 test classes | ✅ Comprehensive | +| **Code Quality** | All Ruff checks passing | ✅ Clean Code | +| **Error Handling** | 124 exception handlers | ✅ Robust | +| **Logging Coverage** | 138 log statements | ✅ Observable | + +### 🏗️ **Architecture Strengths** + +1. **Type Safety Excellence**: Every function, parameter, and return type annotated +2. **Modular Design**: Clear separation between source/sink connectors and transform logic +3. **Environment Awareness**: Intelligent detection and handling of Platform vs Cloud environments +4. **Configuration Robustness**: Comprehensive validation with helpful error messages +5. **Transform Support**: Java regex compatibility ensures exact Kafka Connect behavior match +6. **Testing Quality**: Real-world scenarios, edge cases, and integration testing coverage + +## Current Performance and Reliability + +### Actual Measured Performance + +- **MyPy**: 0 errors across 9 source files +- **Ruff**: All linting checks pass +- **Tests**: BigQuery sanitization - 15/15 tests passing +- **Core Tests**: 67/67 Kafka Connect core tests passing + +### Reliability Features + +- **Graceful Degradation**: Multiple fallback strategies prevent complete failure +- **Type Safety**: Runtime type safety through comprehensive annotations +- **Error Logging**: Detailed logging for troubleshooting and monitoring +- **Configuration Validation**: Input validation for JDBC URLs, topic names, etc. + +## 🏷️ **Type Safety Implementation** + +The Kafka Connect implementation serves as an **exemplary model** for type safety in DataHub ingestion sources. + +### **100% Type Annotation Coverage** + +Every function, parameter, and return value is fully annotated: + +```python +# Example from source_connectors.py +def _extract_lineages_with_environment_awareness( + self, + parser: JdbcParser +) -> List[KafkaConnectLineage]: + """Environment-aware lineage extraction with complete type safety.""" + connector_class = self.connector_manifest.config.get(CONNECTOR_CLASS, "") + is_cloud_environment = connector_class in CLOUD_JDBC_SOURCE_CLASSES + + if is_cloud_environment: + return self._extract_lineages_cloud_environment(parser) + else: + return self._extract_lineages_platform_environment(parser) +``` + +### **Advanced Type Features Used** + +- **Generic Types**: `List[KafkaConnectLineage]`, `Dict[str, str]`, `Optional[TableId]` +- **Union Types**: `Union[str, List[str]]` for flexible parameter types +- **Type Guards**: Runtime type checking with `isinstance()` +- **Dataclasses**: Structured data with automatic type validation +- **Protocol Usage**: Interface definitions for extensible architecture + +### **Benefits for Kafka Connect Developers** + +1. **IDE Autocomplete**: Full IntelliSense support in VS Code/PyCharm +2. **Error Prevention**: Type mismatches caught before runtime +3. **Self-Documenting Code**: Types serve as inline documentation +4. **Refactoring Safety**: Confident code changes with type-aware tools +5. **Team Collaboration**: Clear contracts between connector components + +### **MyPy Compliance Verification** + +```bash +# Verify type safety (should show 0 errors) +mypy src/datahub/ingestion/source/kafka_connect/ + +# Integration with build system +./gradlew :metadata-ingestion:lint # Includes type checking +``` + +**Result**: ✅ **0 MyPy errors across 5,713+ lines of Kafka Connect code** + +### **Type Safety Best Practices Demonstrated** + +The implementation showcases several type safety best practices: + +```python +# 1. Structured data with dataclasses +@dataclass +class TransformResult: + source_table: str + schema: str + final_topics: List[str] + original_topic: str + +# 2. Factory methods with proper typing +def _get_connector_class_type(self) -> Optional[Type["BaseConnector"]]: + """Factory method with type-safe returns.""" + pass + +# 3. Configuration parsing with validation +def parse_comma_separated_list(value: str) -> List[str]: + """Type-safe configuration parsing with validation.""" + if not value or not value.strip(): + return [] + return [item.strip() for item in value.split(",") if item.strip()] +``` + +This comprehensive type safety implementation makes the Kafka Connect source one of the most maintainable and developer-friendly components in the DataHub ingestion framework. + +--- + +_This document reflects the actual current implementation as of the latest code analysis and removes inaccurate claims from the previous documentation._ diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/README.md new file mode 100644 index 00000000..ffe9b225 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/README.md @@ -0,0 +1,46 @@ +--- +slug: /metadata-ingestion +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/README.md +--- +# Introduction to Metadata Ingestion + +:::tip Find Integration Source +Please see our **[Integrations page](/integrations)** to browse our ingestion sources and filter on their features. +::: + +## Integration Methods + +DataHub offers three methods for data ingestion: + +- [UI Ingestion](../docs/ui-ingestion.md) : Easily configure and execute a metadata ingestion pipeline through the UI. +- [CLI Ingestion guide](cli-ingestion.md) : Configure the ingestion pipeline using YAML and execute by it through CLI. +- SDK-based ingestion : Use [Python Emitter](./as-a-library.md) or [Java emitter](../metadata-integration/java/as-a-library.md) to programmatically control the ingestion pipelines. + +## Types of Integration + +Integration can be divided into two concepts based on the method: + +### Push-based Integration + +Push-based integrations allow you to emit metadata directly from your data systems when metadata changes. +Examples of push-based integrations include [Airflow](../docs/lineage/airflow.md), [Spark](../metadata-integration/java/acryl-spark-lineage/README.md), [Great Expectations](./integration_docs/great-expectations.md) and [Protobuf Schemas](../metadata-integration/java/datahub-protobuf/README.md). This allows you to get low-latency metadata integration from the "active" agents in your data ecosystem. + +### Pull-based Integration + +Pull-based integrations allow you to "crawl" or "ingest" metadata from the data systems by connecting to them and extracting metadata in a batch or incremental-batch manner. +Examples of pull-based integrations include BigQuery, Snowflake, Looker, Tableau and many others. + +## Core Concepts + +The following are the core concepts related to ingestion: + +- [Sources](source_overview.md): Data systems from which extract metadata. (e.g. BigQuery, MySQL) +- [Sinks](sink_overview.md): Destination for metadata (e.g. File, DataHub) +- [Recipe](recipe_overview.md): The main configuration for ingestion in the form or .yaml file + +For more advanced guides, please refer to the following: + +- [Developing on Metadata Ingestion](./developing.md) +- [Adding a Metadata Ingestion Source](./adding-source.md) +- [Using Transformers](./docs/transformer/intro.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/adding-source.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/adding-source.md new file mode 100644 index 00000000..ab8ad709 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/adding-source.md @@ -0,0 +1,271 @@ +--- +title: Adding a Metadata Ingestion Source +slug: /metadata-ingestion/adding-source +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/adding-source.md +--- +# Adding a Metadata Ingestion Source + +:::tip Build Connectors Faster with DataHub Skills + +The fastest way to build a new connector is with **DataHub Skills** — an AI-assisted framework that generates production-ready connectors from a simple configuration. Before diving into the manual steps below, check out the [DataHub Skills guide](./datahub-skills.md) and build your integration in minutes instead of weeks. + +::: + +There are two ways of adding a metadata ingestion source. + +1. You are going to contribute the custom source directly to the Datahub project. +2. You are writing the custom source for yourself and are not going to contribute back (yet). + +If you are going for case (1) just follow the steps 1 to 9 below. In case you are building it for yourself you can skip +steps 4-8 (but maybe write tests and docs for yourself as well) and follow the documentation +on [how to use custom ingestion sources](../docs/how/add-custom-ingestion-source.md) +without forking Datahub. + +:::note + +This guide assumes that you've already followed the metadata ingestion [developing guide](./developing.md) to set up +your local environment. + +::: + +### 1. Set up the configuration model + +We use [pydantic](https://pydantic-docs.helpmanual.io/) for configuration, and all models must inherit +from `ConfigModel`. The [file source](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/file.py) is a good example. + +#### Documentation for Configuration Classes + +We use [pydantic](https://pydantic-docs.helpmanual.io) conventions for documenting configuration flags. Use the `description` attribute to write rich documentation for your configuration field. + +For example, the following code: + +```python +from pydantic import Field +from datahub.api.configuration.common import ConfigModel + +class LookerAPIConfig(ConfigModel): + client_id: str = Field(description="Looker API client id.") + client_secret: str = Field(description="Looker API client secret.") + base_url: str = Field( + description="Url to your Looker instance: `https://company.looker.com:19999` or `https://looker.company.com`, or similar. Used for making API calls to Looker and constructing clickable dashboard and chart urls." + ) + transport_options: Optional[TransportOptionsConfig] = Field( + default=None, + description="Populates the [TransportOptions](https://github.com/looker-open-source/sdk-codegen/blob/94d6047a0d52912ac082eb91616c1e7c379ab262/python/looker_sdk/rtl/transport.py#L70) struct for looker client", + ) +``` + +generates the following documentation: + +

+ +

+ +:::note +Inline markdown or code snippets are not yet supported for field level documentation. +::: + +### 2. Set up the reporter + +The reporter interface enables the source to report statistics, warnings, failures, and other information about the run. +Some sources use the default `SourceReport` class, but others inherit and extend that class. + +### 3. Implement the source itself + +The core for the source is the `get_workunits_internal` method, which produces a stream of metadata events (typically MCP objects) wrapped up in a MetadataWorkUnit. +The [file source](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/file.py) is a good and simple example. + +The MetadataChangeEventClass is defined in the metadata models which are generated +under `metadata-ingestion/src/datahub/metadata/schema_classes.py`. There are also +some [convenience methods](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/emitter/mce_builder.py) for commonly used operations. + +### 4. Set up the dependencies + +Note: Steps 4-8 are only required if you intend to contribute the source back to the Datahub project. + +Declare the source's pip dependencies in the `plugins` variable of the [setup script](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/setup.py). + +### 5. Enable discoverability + +Declare the source under the `entry_points` variable of the [setup script](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/setup.py). This enables the source to be +listed when running `datahub check plugins`, and sets up the source's shortened alias for use in recipes. + +### 6. Write tests + +Tests go in the `tests` directory. We use the [pytest framework](https://pytest.org/). + +### 7. Write docs + +#### 7.1 Set up the source class for automatic documentation + +- Indicate the platform name that this source class produces metadata for using the `@platform_name` decorator. We prefer using the human-readable platform name, so e.g. BigQuery (not bigquery). +- Indicate the config class being used by the source by using the `@config_class` decorator. +- Indicate the support status of the connector by using the `@support_status` decorator. +- Indicate what capabilities the connector supports (and what important capabilities it does NOT support) by using the `@capability` decorator. +- Add rich documentation for the connector by utilizing docstrings on your Python class. Markdown is supported. + +See below a simple example of how to do this for any source. + +```python + +from datahub.ingestion.api.decorators import ( + SourceCapability, + SupportStatus, + capability, + config_class, + platform_name, + support_status, +) + +@platform_name("File") +@support_status(SupportStatus.CERTIFIED) +@config_class(FileSourceConfig) +@capability( + SourceCapability.PLATFORM_INSTANCE, + "File based ingestion does not support platform instances", + supported=False, +) +@capability(SourceCapability.DOMAINS, "Enabled by default") +@capability(SourceCapability.DATA_PROFILING, "Optionally enabled via configuration") +@capability(SourceCapability.DESCRIPTIONS, "Enabled by default") +@capability(SourceCapability.LINEAGE_COARSE, "Enabled by default") +class FileSource(Source): + """ + + The File Source can be used to produce all kinds of metadata from a generic metadata events file. + :::note + Events in this file can be in MCE form or MCP form. + ::: + + """ + + ... source code goes here + +``` + +#### 7.2 Write custom documentation + +- Create a copy of [`source-docs-template.md`](./source-docs-template.md) and edit all relevant components. +- Name the document as `<plugin.md>` and move it to `metadata-ingestion/docs/sources//.md`. For example for the Kafka platform, under the `kafka` plugin, move the document to `metadata-ingestion/docs/sources/kafka/kafka.md`. +- Add a quickstart recipe corresponding to the plugin under `metadata-ingestion/docs/sources//_recipe.yml`. For example, for the Kafka platform, under the `kafka` plugin, there is a quickstart recipe located at `metadata-ingestion/docs/sources/kafka/kafka_recipe.yml`. +- To write platform-specific documentation (that is cross-plugin), write the documentation under `metadata-ingestion/docs/sources//README.md`. For example, cross-plugin documentation for the BigQuery platform is located under `metadata-ingestion/docs/sources/bigquery/README.md`. + +#### 7.3 Viewing the Documentation + +Documentation for the source can be viewed by running the documentation generator from the `docs-website` module. + +##### Step 1: Build the Ingestion docs + +```console +# From the root of DataHub repo +./gradlew :metadata-ingestion:docGen +``` + +If this finishes successfully, you will see output messages like: + +```console +Ingestion Documentation Generation Complete +############################################ +{ + "source_platforms": { + "discovered": 40, + "generated": 40 + }, + "plugins": { + "discovered": 47, + "generated": 47, + "failed": 0 + } +} +############################################ +``` + +You can also find documentation files generated at `./docs/generated/ingestion/sources` relative to the root of the DataHub repo. You should be able to locate your specific source's markdown file here and investigate it to make sure things look as expected. + +#### Step 2: Build the Entire Documentation + +To view how this documentation looks in the browser, there is one more step. Just build the entire docusaurus page from the `docs-website` module. + +```console +# From the root of DataHub repo +./gradlew :docs-website:build +``` + +This will generate messages like: + +```console +... +> Task :docs-website:yarnGenerate +yarn run v1.22.0 +$ rm -rf genDocs/* && ts-node -O '{ "lib": ["es2020"], "target": "es6" }' generateDocsDir.ts && mv -v docs/* genDocs/ +Including untracked files in docs list: +docs/graphql -> genDocs/graphql +Done in 2.47s. + +> Task :docs-website:yarnBuild +yarn run v1.22.0 +$ docusaurus build + +╭──────────────────────────────────────────────────────────────────────────────╮│ ││ Update available 2.0.0-beta.8 → 2.0.0-beta.18 ││ ││ To upgrade Docusaurus packages with the latest version, run the ││ following command: ││ yarn upgrade @docusaurus/core@latest ││ @docusaurus/plugin-ideal-image@latest @docusaurus/preset-classic@latest ││ │╰──────────────────────────────────────────────────────────────────────────────╯ + + +[en] Creating an optimized production build... +Invalid docusaurus-plugin-ideal-image version 2.0.0-beta.7. +All official @docusaurus/* packages should have the exact same version as @docusaurus/core (2.0.0-beta.8). +Maybe you want to check, or regenerate your yarn.lock or package-lock.json file? +Browserslist: caniuse-lite is outdated. Please run: + npx browserslist@latest --update-db + Why you should do it regularly: https://github.com/browserslist/browserslist#browsers-data-updating +ℹ Compiling Client +ℹ Compiling Server +✔ Client: Compiled successfully in 1.95s +✔ Server: Compiled successfully in 7.52s +Success! Generated static files in "build". + +Use `npm run serve` command to test your build locally. + +Done in 11.59s. + +Deprecated Gradle features were used in this build, making it incompatible with Gradle 7.0. +Use '--warning-mode all' to show the individual deprecation warnings. +See https://docs.gradle.org/6.9.2/userguide/command_line_interface.html#sec:command_line_warnings + +BUILD SUCCESSFUL in 35s +36 actionable tasks: 16 executed, 20 up-to-date +``` + +After this you need to run the following script from the `docs-website` module. + +```console +cd docs-website +npm run serve +``` + +Now, browse to http://localhost:3000 or whichever port npm is running on, to browse the docs. +Your source should show up on the left sidebar under `Metadata Ingestion / Sources`. + +### 8. Add SQL Alchemy mapping (if applicable) + +Add the source in `get_platform_from_sqlalchemy_uri` function +in [sql_common.py](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py) if the source has an sqlalchemy source + +### 9. Add logo for the platform + +Add the logo image in [images folder](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/images) and add it to be ingested at [startup](https://github.com/datahub-project/datahub/blob/master/metadata-service/configuration/src/main/resources/bootstrap_mcps/data-platforms.yaml) + +### 10. Update Frontend for UI-based ingestion + +We are currently transitioning to a more dynamic approach to display available sources for UI-based Managed Ingestion. For the time being, adhere to these next steps to get your source to display in the UI Ingestion tab. + +#### 10.1 Add to sources.json + +Add new source to the list in [sources.json](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/ingest/source/builder/sources.json) including a default quickstart recipe. This will render your source in the list of options when creating a new recipe in the UI. + +#### 10.2 Add logo to the React app + +Add your source logo to the React [images folder](https://github.com/datahub-project/datahub/tree/master/datahub-web-react/src/images) so your image is available in memory. + +#### 10.3 Update constants.ts + +Create new constants in [constants.ts](https://github.com/datahub-project/datahub/blob/master/datahub-web-react/src/app/ingest/source/builder/constants.ts) for the source urn and source name. Update PLATFORM_URN_TO_LOGO to map your source urn to the newly added logo in the images folder. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/as-a-library.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/as-a-library.md new file mode 100644 index 00000000..6cd30e21 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/as-a-library.md @@ -0,0 +1,138 @@ +--- +title: Python Emitter +slug: /metadata-ingestion/as-a-library +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/as-a-library.md +--- +# Python Emitter + +In some cases, you might want to construct Metadata events directly and use programmatic ways to emit that metadata to DataHub. Use-cases are typically push-based and include emitting metadata events from CI/CD pipelines, custom orchestrators etc. + +The `acryl-datahub` Python package offers REST and Kafka emitter API-s, which can easily be imported and called from your own code. + +> **Pro Tip!** Throughout our API guides, we have examples of using Python API SDK. +> Lookout for the `| Python |` tab within our tutorials. + +## Installation + +Follow the installation guide for the main `acryl-datahub` package [here](./README.md#install-from-pypi). Read on for emitter specific installation instructions. + +## REST Emitter + +The REST emitter is a thin wrapper on top of the `requests` module and offers a blocking interface for sending metadata events over HTTP. Use this when simplicity and acknowledgement of metadata being persisted to DataHub's metadata store is more important than throughput of metadata emission. Also use this when read-after-write scenarios exist, e.g. writing metadata and then immediately reading it back. + +### Installation + +```console +pip install -U `acryl-datahub[datahub-rest]` +``` + +### Example Usage + +```python +import datahub.emitter.mce_builder as builder +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.metadata.schema_classes import DatasetPropertiesClass + +from datahub.emitter.rest_emitter import DatahubRestEmitter + +# Create an emitter to DataHub over REST +emitter = DatahubRestEmitter(gms_server="http://localhost:8080", extra_headers={}) + +# For DataHub Cloud, you will want to point to your DataHub Cloud's server's GMS endpoint +# emitter = DatahubRestEmitter(gms_server="https://.acryl.io/gms", token="", extra_headers={}) + +# Test the connection +emitter.test_connection() + +# Construct a dataset properties object +dataset_properties = DatasetPropertiesClass(description="This table stored the canonical User profile", + customProperties={ + "governance": "ENABLED" + }) + +# Construct a MetadataChangeProposalWrapper object. +metadata_event = MetadataChangeProposalWrapper( + entityUrn=builder.make_dataset_urn("bigquery", "my-project.my-dataset.user-table"), + aspect=dataset_properties, +) + +# Emit metadata! This is a blocking call +emitter.emit(metadata_event) +``` + +Other examples: + +- [lineage_emitter_mcpw_rest.py](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/lineage_emitter_mcpw_rest.py) - emits simple bigquery table-to-table (dataset-to-dataset) lineage via REST as MetadataChangeProposalWrapper. + +### Emitter Code + +If you're interested in looking at the REST emitter code, it is available [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/emitter/rest_emitter.py) + +## Kafka Emitter + +The Kafka emitter is a thin wrapper on top of the SerializingProducer class from `confluent-kafka` and offers a non-blocking interface for sending metadata events to DataHub. Use this when you want to decouple your metadata producer from the uptime of your datahub metadata server by utilizing Kafka as a highly available message bus. For example, if your DataHub metadata service is down due to planned or unplanned outages, you can still continue to collect metadata from your mission critical systems by sending it to Kafka. Also use this emitter when throughput of metadata emission is more important than acknowledgement of metadata being persisted to DataHub's backend store. + +**_Note_**: The Kafka emitter uses Avro to serialize the Metadata events to Kafka. Changing the serializer will result in unprocessable events as DataHub currently expects the metadata events over Kafka to be serialized in Avro. + +### Installation + +```console +# For emission over Kafka +pip install -U `acryl-datahub[datahub-kafka]` +``` + +### Example Usage + +```python +import datahub.emitter.mce_builder as builder +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.metadata.schema_classes import DatasetPropertiesClass + +from datahub.emitter.kafka_emitter import DatahubKafkaEmitter, KafkaEmitterConfig +# Create an emitter to Kafka +kafka_config = { + "connection": { + "bootstrap": "localhost:9092", + "schema_registry_url": "http://localhost:8081", + "schema_registry_config": {}, # schema_registry configs passed to underlying schema registry client + "producer_config": {}, # extra producer configs passed to underlying kafka producer + } +} + +emitter = DatahubKafkaEmitter( + KafkaEmitterConfig.parse_obj(kafka_config) +) + +# Construct a dataset properties object +dataset_properties = DatasetPropertiesClass(description="This table stored the canonical User profile", + customProperties={ + "governance": "ENABLED" + }) + +# Construct a MetadataChangeProposalWrapper object. +metadata_event = MetadataChangeProposalWrapper( + entityUrn=builder.make_dataset_urn("bigquery", "my-project.my-dataset.user-table"), + aspect=dataset_properties, +) + + +# Emit metadata! This is a non-blocking call +emitter.emit( + metadata_event, + callback=lambda exc, message: print(f"Message sent to topic:{message.topic()}, partition:{message.partition()}, offset:{message.offset()}") if message else print(f"Failed to send with: {exc}") +) + +#Send all pending events +emitter.flush() +``` + +### Emitter Code + +If you're interested in looking at the Kafka emitter code, it is available [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/emitter/kafka_emitter.py) + +## Other Languages + +Emitter API-s are also supported for: + +- [Java](../metadata-integration/java/as-a-library.md) diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/cli-ingestion.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/cli-ingestion.md new file mode 100644 index 00000000..f24746fd --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/cli-ingestion.md @@ -0,0 +1,134 @@ +--- +title: CLI Ingestion +slug: /metadata-ingestion/cli-ingestion +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/cli-ingestion.md +--- +# CLI Ingestion + +Batch ingestion involves extracting metadata from a source system in bulk. Typically, this happens on a predefined schedule using the [Metadata Ingestion](../docs/components.md#ingestion-framework) framework. +The metadata that is extracted includes point-in-time instances of dataset, chart, dashboard, pipeline, user, group, usage, and task metadata. + +## Installing DataHub CLI + +:::note Required Python Version +Installing DataHub CLI requires Python 3.6+. +::: + +Run the following commands in your terminal: + +``` +python3 -m pip install --upgrade pip wheel setuptools +python3 -m pip install --upgrade acryl-datahub +python3 -m datahub version +``` + +Your command line should return the proper version of DataHub upon executing these commands successfully. + +Check out the [CLI Installation Guide](../docs/cli.md#installation) for more installation options and troubleshooting tips. + +## Installing Connector Plugins + +Our CLI follows a plugin architecture. You must install connectors for different data sources individually. +For a list of all supported data sources, see [the open source docs](../docs/cli.md#sources). +Once you've found the connectors you care about, simply install them using `pip install`. +For example, to install the `mysql` connector, you can run + +```shell +pip install --upgrade 'acryl-datahub[mysql]' +``` + +Check out the [alternative installation options](../docs/cli.md#alternate-installation-options) for more reference. + +:::tip Building a new connector? + +If you need a connector that doesn't exist yet, [datahub-skills](./datahub-skills.md) is a +Claude Code plugin that accelerates connector development — from planning and scaffolding to +standards review and community testing. + +::: + +## Configuring a Recipe + +Create a [Recipe](recipe_overview.md) yaml file that defines the source and sink for metadata, as shown below. + +```yaml +# example-recipe.yml + +# MySQL source configuration +source: + type: mysql + config: + username: root + password: password + host_port: localhost:3306 + +# Recipe sink configuration. +sink: + type: "datahub-rest" + config: + server: "https://.acryl.io/gms" + token: +``` + +The **source** configuration block defines where to extract metadata from. This can be an OLTP database system, a data warehouse, or something as simple as a file. Each source has custom configuration depending on what is required to access metadata from the source. To see configurations required for each supported source, refer to the [Sources](source_overview.md) documentation. + +The **sink** configuration block defines where to push metadata into. Each sink type requires specific configurations, the details of which are detailed in the [Sinks](sink_overview.md) documentation. + +To configure your instance of DataHub as the destination for ingestion, set the "server" field of your recipe to point to your DataHub Cloud instance's domain suffixed by the path `/gms`, as shown below. +A complete example of a DataHub recipe file, which reads from MySQL and writes into a DataHub instance: + +For more information and examples on configuring recipes, please refer to [Recipes](recipe_overview.md). + +### Using Recipes with Authentication + +In DataHub Cloud deployments, only the `datahub-rest` sink is supported, which simply means that metadata will be pushed to the REST endpoints exposed by your DataHub instance. The required configurations for this sink are + +1. **server**: the location of the REST API exposed by your instance of DataHub +2. **token**: a unique API key used to authenticate requests to your instance's REST API + +The token can be retrieved by logging in as admin. You can go to Settings page and generate a Personal Access Token with your desired expiration date. + +

+ +

+ +

+ +

+ +:::info Secure Your API Key +Please keep Your API key secure & avoid sharing it. +If you are on DataHub Cloud and your key is compromised for any reason, please reach out to the DataHub team at support@acryl.io. +::: + +## Ingesting Metadata + +The final step requires invoking the DataHub CLI to ingest metadata based on your recipe configuration file. +To do so, simply run `datahub ingest` with a pointer to your YAML recipe file: + +```shell +datahub ingest -c +``` + +## Scheduling Ingestion + +Ingestion can either be run in an ad-hoc manner by a system administrator or scheduled for repeated executions. Most commonly, ingestion will be run on a daily cadence. +To schedule your ingestion job, we recommend using a job schedule like [Apache Airflow](https://airflow.apache.org/). In cases of simpler deployments, a CRON job scheduled on an always-up machine can also work. +Note that each source system will require a separate recipe file. This allows you to schedule ingestion from different sources independently or together. +Learn more about scheduling ingestion in the [Scheduling Ingestion Guide](/metadata-ingestion/schedule_docs/intro.md). + +## Reference + +Please refer the following pages for advanced guids on CLI ingestion. + +- [Reference for `datahub ingest` command](../docs/cli.md#ingest) +- [UI Ingestion Guide](../docs/ui-ingestion.md) + +:::tip Compatibility + +DataHub server uses a 3 digit versioning scheme, while the CLI uses a 4 digit scheme. For example, if you're using DataHub server version 0.10.0, you should use CLI version 0.10.0.x, where x is a patch version. +We do this because we do CLI releases at a much higher frequency than server releases, usually every few days vs twice a month. + +For ingestion sources, any breaking changes will be highlighted in the [release notes](../docs/how/updating-datahub.md). When fields are deprecated or otherwise changed, we will try to maintain backwards compatibility for two server releases, which is about 4-6 weeks. The CLI will also print warnings whenever deprecated options are used. +::: diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/datahub-skills.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/datahub-skills.md new file mode 100644 index 00000000..461775c4 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/datahub-skills.md @@ -0,0 +1,211 @@ +--- +title: Building Connectors with datahub-skills +slug: /metadata-ingestion/datahub-skills +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/datahub-skills.md +--- +# Building Connectors with datahub-skills + +datahub-skills is a [Claude Code](https://claude.ai/code) plugin that accelerates connector development with specialized skills for planning, standards loading, and code review. It is designed to be used alongside the [adding a source guide](./adding-source.md) — the skills handle research, scaffolding, and review; you handle the code. + +:::note + +This guide assumes you have already set up your local development environment per the [developing guide](./developing.md). + +::: + +## Installing datahub-skills + +**Option 1 — install script (recommended):** + +```bash +curl -fsSL https://raw.githubusercontent.com/acryldata/datahub-skills/main/install.sh | bash +``` + +**Option 2 — npx:** + +```bash +npx datahub-skills install +``` + +After installation, confirm the plugin is registered: + +```bash +cat ~/.claude/plugins/installed_plugins.json +``` + +```json +{ + "version": 2, + "plugins": { + "datahub-skills@datahub-skills": [ + { + "scope": "user", + "installPath": "/Users/you/.claude/plugins/cache/datahub-skills/datahub-skills/1.2.0", + "version": "1.2.0" + } + ] + } +} +``` + +## Available Skills + +| Skill | Command | When to use | +| ---------------- | --------------------------------------------- | -------------------------------------------------------------------------- | +| Load standards | `/datahub-skills:load-standards` | Start of every session — loads golden connector standards into context | +| Plan connector | `/datahub-skills:datahub-connector-planning` | Before writing any code — research, entity mapping, architecture decisions | +| Review connector | `/datahub-skills:datahub-connector-pr-review` | Before opening a PR — automated review against standards | + +## Workflow + +### 1. Load standards + +Run this at the start of every Claude Code session. It loads the golden connector standards into context so all subsequent skills and code generation follow established patterns. + +``` +/datahub-skills:load-standards +``` + +Expected output: + +``` +Loaded connector standards. Ready to review. + +Standards loaded: +- main.md — base classes, SDK V2, config patterns +- patterns.md — file organization, error handling, warning/failure reporting +- testing.md — test requirements, golden files, anti-patterns +- code_style.md — type safety, naming conventions, mypy compliance +- containers.md — container hierarchy design +- lineage.md — SqlParsingAggregator, view lineage, query log lineage +... +``` + +:::note + +Standards are loaded into your current Claude Code context window. If you start a new session, run `/load-standards` again before continuing. + +::: + +### 2. Plan the connector + +Run the planning skill before writing any implementation code. It researches the source system, asks you a set of scoping questions, then generates a `_PLANNING.md` document that serves as the implementation blueprint. + +``` +/datahub-skills:datahub-connector-planning build a connector for [source name] +``` + +The skill runs in four stages: + +1. **Classify** — identifies the source category (SQL database, BI tool, orchestration, etc.) and asks you to confirm +2. **Research** — gathers API docs, Python client libraries, similar existing connectors, Docker images for testing +3. **Scope** — asks you three questions about test environment, permissions, and which features to prioritize +4. **Document** — generates `_PLANNING.md` at the repo root with entity mapping, architecture decisions, and an ordered implementation plan + +After the document is generated, you will see a summary and a prompt: + +``` +## Planning Document Created + +Location: `_PLANNING.md` + +### Key Decisions: +- Base class: StatefulIngestionSourceBase — no SQLAlchemy dialect exists +- Entity mapping: Pipeline → DataFlow, Table → DataJob, outlet lineage to destination Datasets +- Lineage approach: outlet URNs from state files; inlet URNs user-configured +- Test strategy: filesystem fixtures (no live service required) + +Do you approve proceeding to implementation? +``` + +Reply `approved` to proceed, or describe changes and the skill will revise the document before continuing. + +### 3. Build + +This is your implementation step. Use the plan in `_PLANNING.md` as the guide and refer to `CLAUDE.md` for build, lint, and test commands. + +Key commands during development: + +```bash +# Lint and format Python code +./gradlew :metadata-ingestion:lintFix + +# Run unit tests +./gradlew :metadata-ingestion:testQuick + +# Run a single test file +pytest tests/unit/myconnector/test_myconnector_source.py -v +``` + +See the [developing guide](./developing.md) for the complete command reference. + +### 4. Review the connector + +Run the review skill before opening a PR. It performs a full standards compliance review — architecture, code quality, type safety, error handling, test coverage, and documentation — and produces a verdict with prioritized findings. + +``` +/datahub-skills:datahub-connector-pr-review +``` + +The skill outputs a structured report: + +``` +## Connector Review Report + +Verdict: NEEDS CHANGES — 3 blockers require fixes before merge. + +| Category | Status | +| ----------------- | -------------- | +| Architecture | ✅ PASS | +| Code Organization | ✅ PASS | +| Error Handling | 🔴 NEEDS FIXES | +| Type Safety | 🟡 NEEDS FIXES | +| Test Quality | 🔴 NEEDS FIXES | +| Performance | ✅ PASS | + +### 🔴 BLOCKERS (Must Fix) + +B1 — Silent exception swallow in state.json parsing + dlt_client.py:269 — except Exception: pass with no logging... +``` + +Severity levels: + +| Level | Meaning | Action | +| ----------------- | ----------------------------------------------------- | --------------------- | +| 🔴 **BLOCKER** | Violates standards or will cause issues in production | Must fix before merge | +| 🟡 **WARNING** | Significant issue that should be addressed | Should fix | +| ℹ️ **SUGGESTION** | Would improve quality | Optional | + +After fixing blockers, re-run the review skill. It resumes with full context from the previous session: + +``` +/datahub-skills:datahub-connector-pr-review +``` + +## The `skill_docs/` convention + +Commit the skill-generated artifacts alongside your connector code. This gives PR reviewers full context on design decisions and known issues, and creates a paper trail for follow-up work. + +Place them in `src/datahub/ingestion/source//skill_docs/`: + +``` +src/datahub/ingestion/source/myconnector/ +├── __init__.py +├── config.py +├── myconnector.py +├── myconnector_client.py +├── myconnector_report.py +└── skill_docs/ + ├── _PLANNING.md # /datahub-connector-planning output + └── _datahub-connector-pr-review-YYYY-MM-DD.md # /datahub-connector-pr-review output +``` + +## Tips + +- Always run `/load-standards` at the start of a new Claude Code session — standards are not persisted across sessions +- The planning skill will ask you scoping questions before generating `_PLANNING.md`; answer them rather than skipping, as they determine the architecture decisions in the plan +- The review skill catches issues the checklist cannot — error handling gaps, trivial tests, type safety violations — run it even if your code looks clean +- Commit `skill_docs/` alongside the connector code; PR reviewers use it to understand design decisions without re-reading the full diff +- If a review session is interrupted, re-invoke the skill and it will resume from where it left off diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/developing.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/developing.md new file mode 100644 index 00000000..b841e1f0 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/developing.md @@ -0,0 +1,359 @@ +--- +title: Developing on Metadata Ingestion +slug: /metadata-ingestion/developing +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/developing.md +--- +# Developing on Metadata Ingestion + +If you just want to use metadata ingestion, check the [user-centric](./README.md) guide. +This document is for developers who want to develop and possibly contribute to the metadata ingestion framework. + +Also take a look at the guide to [adding a source](./adding-source.md). + +## Getting Started + +### Requirements + +1. Python 3.9+ must be installed in your host environment. +2. Java 17 (gradle won't work with newer or older versions) +3. On Debian/Ubuntu: `sudo apt install python3-dev python3-venv` +4. On Fedora (if using LDAP source integration): `sudo yum install openldap-devel` + +### Set up your Python environment + +From the repository root: + +```shell +cd metadata-ingestion +../gradlew :metadata-ingestion:installDev +source venv/bin/activate +datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" +``` + +### (Optional) Set up your Python environment for developing on Airflow Plugin + +From the repository root: + +```shell +cd metadata-ingestion-modules/airflow-plugin +../../gradlew :metadata-ingestion-modules:airflow-plugin:installDev +source venv/bin/activate +datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" + +# start the airflow web server +export AIRFLOW_HOME=~/airflow +airflow webserver --port 8090 -d + +# start the airflow scheduler +airflow scheduler + +# access the airflow service and run any of the DAG +# open http://localhost:8090/ +# select any DAG and click on the `play arrow` button to start the DAG + +# add the debug lines in the codebase, i.e. in ./src/datahub_airflow_plugin/datahub_listener.py +logger.debug("this is the sample debug line") + +# run the DAG again and you can see the debug lines in the task_run log at, +#1. click on the `timestamp` in the `Last Run` column +#2. select the task +#3. click on the `log` option +``` + +> **P.S. if you are not able to see the log lines, then restart the `airflow scheduler` and rerun the DAG** + +### (Optional) Set up your Python environment for developing on Dagster Plugin + +From the repository root: + +```shell +cd metadata-ingestion-modules/dagster-plugin +../../gradlew :metadata-ingestion-modules:dagster-plugin:installDev +source venv/bin/activate +datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" +``` + +### (Optional) Set up your Python environment for developing on Prefect Plugin + +From the repository root: + +```shell +cd metadata-ingestion-modules/prefect-plugin +../../gradlew :metadata-ingestion-modules:prefect-plugin:installDev +source venv/bin/activate +datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" +``` + +### (Optional) Set up your Python environment for developing on GX Plugin + +From the repository root: + +```shell +cd metadata-ingestion-modules/gx-plugin +../../gradlew :metadata-ingestion-modules:gx-plugin:installDev +source venv/bin/activate +datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" +``` + +### (Optional) Set up your Python environment for developing on Dagster Plugin + +From the repository root: + +```shell +cd metadata-ingestion-modules/dagster-plugin +../../gradlew :metadata-ingestion-modules:dagster-plugin:installDev +source venv/bin/activate +datahub version # should print "DataHub CLI version: unavailable (installed in develop mode)" +``` + +### Common setup issues + +Common issues (click to expand): + +
+ Virtual environment creation fails with symlink errors (Nix, immutable filesystems, Windows) + +If you're using Nix, an immutable Python installation, Windows with certain filesystem configurations, or working in container environments where symlinks don't work correctly, you may encounter errors during virtual environment creation. + +You can enable the `--copies` flag for Python's venv by setting an environment variable before running the gradle commands: + +```shell +export DATAHUB_VENV_USE_COPIES=true +../gradlew :metadata-ingestion:installDev +``` + +This copies the Python binary instead of creating a symlink. Note that this increases disk usage and setup time, so only enable it if you're experiencing issues with the default symlink-based approach. + +
+ +
+ datahub command not found with PyPI install + +If you've already run the pip install, but running `datahub` in your command line doesn't work, then there is likely an issue with your PATH setup and Python. + +The easiest way to circumvent this is to install and run via Python, and use `python3 -m datahub` in place of `datahub`. + +```shell +python3 -m pip install --upgrade acryl-datahub +python3 -m datahub --help +``` + +
+ +
+ Wheel issues e.g. "Failed building wheel for avro-python3" or "error: invalid command 'bdist_wheel'" + +This means Python's `wheel` is not installed. Try running the following commands and then retry. + +```shell +pip install --upgrade pip wheel setuptools +pip cache purge +``` + +
+ +
+ Failure to install confluent_kafka: "error: command 'x86_64-linux-gnu-gcc' failed with exit status 1" + +This sometimes happens if there's a version mismatch between the Kafka's C library and the Python wrapper library. Try running `pip install confluent_kafka==1.5.0` and then retrying. + +
+ +
+ Conflict: acryl-datahub requires pydantic 1.10 + +The base `acryl-datahub` package supports both Pydantic 1.x and 2.x. However, some of our specific sources require Pydantic 1.x because of transitive dependencies. + +If you're primarily using `acryl-datahub` for the SDKs, you can install `acryl-datahub` and some extras, like `acryl-datahub[sql-parser]`, without getting conflicts related to Pydantic versioning. + +We recommend not installing full ingestion sources into your main environment (e.g. avoid having a dependency on `acryl-datahub[snowflake]` or other ingestion sources). +Instead, we recommend using UI-based ingestion or isolating the ingestion pipelines using [virtual environments](https://docs.python.org/3/library/venv.html). If you're using an orchestrator, they often have first-class support for virtual environments - here's an [example for Airflow](./schedule_docs/airflow.md). + +
+ +### Using Plugins in Development + +The syntax for installing plugins is slightly different in development. For example: + +```diff +- uv pip install 'acryl-datahub[bigquery,datahub-rest]' ++ uv pip install -e '.[bigquery,datahub-rest]' +``` + +## Architecture + +

+ +

+ +The architecture of this metadata ingestion framework is heavily inspired by [Apache Gobblin](https://gobblin.apache.org/) (also originally a LinkedIn project!). We have a standardized format - the MetadataChangeEvent - and sources and sinks which respectively produce and consume these objects. The sources pull metadata from a variety of data systems, while the sinks are primarily for moving this metadata into DataHub. + +## Code layout + +- The CLI interface is defined in [entrypoints.py](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/entrypoints.py) and in the [cli](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/cli) directory. +- The high level interfaces are defined in the [API directory](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/api). +- The actual [sources](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source) and [sinks](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/sink) have their own directories. The registry files in those directories import the implementations. +- The metadata models are created using code generation, and eventually live in the `./src/datahub/metadata` directory. However, these files are not checked in and instead are generated at build time. See the [codegen](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/scripts/codegen.sh) script for details. +- Tests live in the [`tests`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/tests) directory. They're split between smaller unit tests and larger integration tests. + +## Code style + +We use ruff, and mypy to ensure consistent code style and quality. + +```shell +# Assumes: ../gradlew :metadata-ingestion:installDev and venv is activated +ruff check src/ tests/ +mypy src/ tests/ +``` + +or you can run from root of the repository + +```shell +./gradlew :metadata-ingestion:lint + +# This will auto-fix some linting issues. +./gradlew :metadata-ingestion:lintFix +``` + +Some other notes: + +- Prefer mixin classes over tall inheritance hierarchies. +- Write type annotations wherever possible. +- Use `typing.Protocol` to make implicit interfaces explicit. +- If you ever find yourself copying and pasting large chunks of code, there's probably a better way to do it. +- Prefer a standalone helper method over a `@staticmethod`. +- You probably should not be defining a `__hash__` method yourself. Using `@dataclass(frozen=True)` is a good way to get a hashable class. +- Avoid global state. In sources, this includes instance variables that effectively function as "global" state for the source. +- Avoid defining functions within other functions. This makes it harder to read and test the code. +- When interacting with external APIs, parse the responses into a dataclass rather than operating directly on the response object. + +## Dependency Management + +The vast majority of our dependencies are not required by the "core" package but instead can be optionally installed using Python "extras". This allows us to keep the core package lightweight. We should be deliberate about adding new dependencies to the core framework. + +Where possible, we should avoid pinning version dependencies. The `acryl-datahub` package is frequently used as a library and hence installed alongside other tools. If you need to restrict the version of a dependency, use a range like `>=1.2.3,<2.0.0` or a negative constraint like `>=1.2.3, !=1.2.7` instead. Every upper bound and negative constraint should be accompanied by a comment explaining why it's necessary. + +Caveat: Some packages like Great Expectations and Airflow frequently make breaking changes. For such packages, it's ok to add a "defensive" upper bound with the current latest version, accompanied by a comment. It's critical that we revisit these upper bounds at least once a month and broaden them if possible. + +### Updating the Lock File + +After changing dependencies in `setup.py`, regenerate all generated files: + +```shell +../gradlew :metadata-ingestion:updateLockFile +``` + +This runs the full chain: `setup.py` → `pyproject.toml` → `uv.lock` → `constraints.txt`. + +To validate that all generated files are up to date without modifying them: + +```shell +../gradlew :metadata-ingestion:checkLockFile +``` + +This runs automatically in CI as part of `check`, so PRs with stale generated files will fail. + +You can also run the steps manually: + +```shell +python scripts/generate_pyproject_deps.py # setup.py → pyproject.toml +python scripts/verify_pyproject_equivalence.py # verify equivalence +uv lock # update uv.lock +uv export --format requirements-txt --no-hashes --all-extras --no-emit-project -o constraints.txt +``` + +## Guidelines for Ingestion Configs + +We use [pydantic](https://pydantic-docs.helpmanual.io/) to define the ingestion configs. +In order to ensure that the configs are consistent and easy to use, we have a few guidelines: + +#### Naming + +- Most important point: we should **match the terminology of the source system**. For example, snowflake shouldn’t have a `host_port`, it should have an `account_id`. +- We should prefer slightly more verbose names when the alternative isn’t descriptive enough. For example `client_id` or `tenant_id` over a bare `id` and `access_secret` over a bare `secret`. +- AllowDenyPatterns should be used whenever we need to filter a list. The pattern should always apply to the fully qualified name of the entity. These configs should be named `*_pattern`, for example `table_pattern`. +- Avoid `*_only` configs like `profile_table_level_only` in favor of `profile_table_level` and `profile_column_level`. `include_tables` and `include_views` are a good example. + +#### Content + +- All configs should have a description. +- When using inheritance or mixin classes, make sure that the fields and documentation is applicable in the base class. The `bigquery_temp_table_schema` field definitely shouldn’t be showing up in every single source’s profiling config! +- Set reasonable defaults! + - The configs should not contain a default that you’d reasonably expect to be built in. As a **bad** example, the Postgres source’s `schema_pattern` has a default deny pattern containing `information_schema`. This means that if the user overrides the schema_pattern, they’ll need to manually add the information_schema to their deny patterns. This is a bad, and the filtering should’ve been handled automatically by the source’s implementation, not added at runtime by its config. + +#### Coding + +- Use a single pydantic validator per thing to validate - we shouldn’t have validation methods that are 50 lines long. +- Use `SecretStr` for passwords, auth tokens, etc. +- When doing simple field renames, use the `pydantic_renamed_field` helper. +- When doing field deprecations, use the `pydantic_removed_field` helper. +- Validator methods must only throw ValueError, TypeError, or AssertionError. Do not throw ConfigurationError from validators. +- Set `hidden_from_docs` for internal-only config flags. However, needing this often indicates a larger problem with the code structure. The hidden field should probably be a class attribute or an instance variable on the corresponding source. + +## Testing + +```shell +# Follow standard install from source procedure - see above. + +# Install all dev and test requirements. +../gradlew :metadata-ingestion:installDevTest + +# Run the full testing suite +pytest -vv + +# Run unit tests. +pytest -m 'not integration' + +# Run Docker-based integration tests. +pytest -m 'integration' + +# You can also run these steps via the gradle build: +../gradlew :metadata-ingestion:lint +../gradlew :metadata-ingestion:lintFix +../gradlew :metadata-ingestion:testQuick +../gradlew :metadata-ingestion:testFull +../gradlew :metadata-ingestion:check +# Run all tests in a single file +../gradlew :metadata-ingestion:testSingle -PtestFile=tests/unit/test_bigquery_source.py +# Run all tests under tests/unit +../gradlew :metadata-ingestion:testSingle -PtestFile=tests/unit +``` + +### Updating golden test files + +If you made some changes that require generating new "golden" data files for use in testing a specific ingestion source, you can run the following to re-generate them: + +```shell +pytest tests/integration//.py --update-golden-files +``` + +For example, + +```shell +pytest tests/integration/dbt/test_dbt.py --update-golden-files +``` + +### Testing the Airflow plugin + +For the Airflow plugin, we use `tox` to test across multiple sets of dependencies. + +```sh +cd metadata-ingestion-modules/airflow-plugin + +# Run all tests. +tox + +# Run a specific environment. +# These are defined in the `tox.ini` file +tox -e py310-airflow26 + +# Run a specific test. +tox -e py310-airflow26 -- tests/integration/test_plugin.py + +# Update all golden files. +tox -- --update-golden-files + +# Update golden files for a specific environment. +tox -e py310-airflow26 -- --update-golden-files +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source.md new file mode 100644 index 00000000..aec196ab --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source.md @@ -0,0 +1,202 @@ +--- +title: Adding Stateful Ingestion to a Source +slug: /metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/dev_guides/add_stateful_ingestion_to_source.md +--- +# Adding Stateful Ingestion to a Source + +Currently, datahub supports the [Stale Metadata Removal](./stateful.md#stale-entity-removal) and +the [Redunant Run Elimination](./stateful.md#redundant-run-elimination) use-cases on top of the more generic stateful ingestion +capability available for the sources. This document describes how to add support for these two use-cases to new sources. + +## Adding Stale Metadata Removal to a Source + +Adding the stale metadata removal use-case to a new source involves modifying the source config, source report, and the source itself. + +For a full example of all changes required: [Adding stale metadata removal to the MongoDB source](https://github.com/datahub-project/datahub/pull/9118). + +The [datahub.ingestion.source.state.stale_entity_removal_handler](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py) module provides the supporting infrastructure for all the steps described +above and substantially simplifies the implementation on the source side. Below is a detailed explanation of each of these +steps along with examples. + +### 1. Modify the source config + +The source's config must inherit from `StatefulIngestionConfigBase`, and should declare a field named `stateful_ingestion` of type `Optional[StatefulStaleMetadataRemovalConfig]`. + +Example: + +```python +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StatefulStaleMetadataRemovalConfig, + StatefulIngestionConfigBase, +) + +class MySourceConfig(StatefulIngestionConfigBase): + # ...... + + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None +``` + +### 2. Modify the source report + +The report class of the source should inherit from `StaleEntityRemovalSourceReport` instead of `SourceReport`. + +```python +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, +) + +@dataclass +class MySourceReport(StatefulIngestionReport): + # + pass +``` + +### 3. Modify the source + +1. The source must inherit from `StatefulIngestionSourceBase` instead of `Source`. +2. The source should contain a custom `get_workunit_processors` method. + +```python +from datahub.ingestion.source.state.stateful_ingestion_base import StatefulIngestionSourceBase +from datahub.ingestion.source.state.stale_entity_removal_handler import StaleEntityRemovalHandler + +class MySource(StatefulIngestionSourceBase): + def __init__(self, config: MySourceConfig, ctx: PipelineContext): + super().__init__(config, ctx) + + self.config = config + self.report = MySourceReport() + + # other initialization code here + + def get_workunit_processors(self) -> List[Optional[MetadataWorkUnitProcessor]]: + return [ + *super().get_workunit_processors(), + StaleEntityRemovalHandler.create( + self, self.config, self.ctx + ).workunit_processor, + ] + + # other methods here +``` + +## Adding Redundant Run Elimination to a Source + +This use-case applies to the sources that drive ingestion by querying logs over a specified duration via the config(such +as snowflake usage, bigquery usage etc.). It typically involves expensive and long-running queries. To add redundant +run elimination to a new source to prevent the expensive reruns for the same time range(potentially due to a user +error or a scheduler malfunction), the following steps +are required. + +1. Update the `SourceConfig` +2. Update the `SourceReport` +3. Modify the `Source` to + 1. Instantiate the RedundantRunSkipHandler object. + 2. Check if the current run should be skipped. + 3. Update the state for the current run(start & end times). + +The [datahub.ingestion.source.state.redundant_run_skip_handler](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/state/redundant_run_skip_handler.py) +modules provides the supporting infrastructure required for all the steps described above. + +NOTE: The handler currently uses a simple state, +the [BaseUsageCheckpointState](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/state/usage_common_state.py), +across all sources it supports (unlike the StaleEntityRemovalHandler). + +### 1. Modifying the SourceConfig + +The `SourceConfig` must inherit from the [StatefulRedundantRunSkipConfig](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/state/redundant_run_skip_handler.py#L23) class. + +Examples: + +1. Snowflake Usage + +```python +from datahub.ingestion.source.state.redundant_run_skip_handler import ( + StatefulRedundantRunSkipConfig, +) +class SnowflakeStatefulIngestionConfig(StatefulRedundantRunSkipConfig): + pass +``` + +### 2. Modifying the SourceReport + +The `SourceReport` must inherit from the [StatefulIngestionReport](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/state/stateful_ingestion_base.py#L102) class. +Examples: + +1. Snowflake Usage + +```python +@dataclass +class SnowflakeUsageReport(BaseSnowflakeReport, StatefulIngestionReport): + # +``` + +### 3. Modifying the Source + +The source must inherit from `StatefulIngestionSourceBase`. + +#### 3.1 Instantiate RedundantRunSkipHandler in the `__init__` method of the source. + +The source should instantiate an instance of the `RedundantRunSkipHandler` in its `__init__` method. +Examples: +Snowflake Usage + +```python +from datahub.ingestion.source.state.redundant_run_skip_handler import ( + RedundantRunSkipHandler, +) +class SnowflakeUsageSource(StatefulIngestionSourceBase): + + def __init__(self, config: SnowflakeUsageConfig, ctx: PipelineContext): + super(SnowflakeUsageSource, self).__init__(config, ctx) + self.config: SnowflakeUsageConfig = config + self.report: SnowflakeUsageReport = SnowflakeUsageReport() + # Create and register the stateful ingestion use-case handlers. + self.redundant_run_skip_handler = RedundantRunSkipHandler( + source=self, + config=self.config, + pipeline_name=self.ctx.pipeline_name, + run_id=self.ctx.run_id, + ) +``` + +#### 3.2 Checking if the current run should be skipped. + +The sources can query if the current run should be skipped using `should_skip_this_run` method of `RedundantRunSkipHandler`. This should done from the `get_workunits` method, before doing any other work. + +Example code: + +```python +def get_workunits(self) -> Iterable[MetadataWorkUnit]: + # Skip a redundant run + if self.redundant_run_skip_handler.should_skip_this_run( + cur_start_time_millis=datetime_to_ts_millis(self.config.start_time) + ): + return + # Generate the workunits. +``` + +#### 3.3 Updating the state for the current run. + +The source should use the `update_state` method of `RedundantRunSkipHandler` to update the current run's state if the run has not been skipped. This step can be performed in the `get_workunits` if the run has not been skipped. + +Example code: + +```python + def get_workunits(self) -> Iterable[MetadataWorkUnit]: + # Skip a redundant run + if self.redundant_run_skip_handler.should_skip_this_run( + cur_start_time_millis=self.config.start_time + ): + return + + # Generate the workunits. + # + # Update checkpoint state for this run. + self.redundant_run_skip_handler.update_state( + start_time_millis=self.config.start_time, + end_time_millis=self.config.end_time, + ) +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/classification.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/classification.md new file mode 100644 index 00000000..1f18ec69 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/classification.md @@ -0,0 +1,460 @@ +--- +title: Classification (Deprecated) +slug: /metadata-ingestion/docs/dev_guides/classification +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/dev_guides/classification.md +--- +# Classification (Deprecated) + +The classification feature enables sources to be configured to automatically predict info types for columns and use them as glossary terms. This is an explicit opt-in feature and is not enabled by default. + +## Config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Type | Description | Default | +| ------------------------- | -------- | --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | +| enabled | | boolean | Whether classification should be used to auto-detect glossary terms | False | +| sample_size | | int | Number of sample values used for classification. | 100 | +| max_workers | | int | Number of worker processes to use for classification. Set to 1 to disable. | Number of CPU cores | +| info_type_to_term | | Dict[str,string] | Optional mapping to provide glossary term identifier for info type. | By default, info type is used as glossary term identifier. | +| classifiers | | Array of object | Classifiers to use to auto-detect glossary terms. If more than one classifier, infotype predictions from the classifier defined later in sequence take precedance. | [{'type': 'datahub', 'config': None}] | +| table_pattern | | AllowDenyPattern (see below for fields) | Regex patterns to filter tables for classification. This is used in combination with other patterns in parent config. Specify regex to match the entire table name in `database.schema.table` format. e.g. to match all tables starting with customer in Customer database and public schema, use the regex 'Customer.public.customer.\*' | {'allow': ['.*'], 'deny': [], 'ignoreCase': True} | +| table_pattern.allow | | Array of string | List of regex patterns to include in ingestion | ['.*'] | +| table_pattern.deny | | Array of string | List of regex patterns to exclude from ingestion. | [] | +| table_pattern.ignoreCase | | boolean | Whether to ignore case sensitivity during pattern matching. | True | +| column_pattern | | AllowDenyPattern (see below for fields) | Regex patterns to filter columns for classification. This is used in combination with other patterns in parent config. Specify regex to match the column name in `database.schema.table.column` format. | {'allow': ['.*'], 'deny': [], 'ignoreCase': True} | +| column_pattern.allow | | Array of string | List of regex patterns to include in ingestion | ['.*'] | +| column_pattern.deny | | Array of string | List of regex patterns to exclude from ingestion. | [] | +| column_pattern.ignoreCase | | boolean | Whether to ignore case sensitivity during pattern matching. | True | + +## DataHub Classifier + +DataHub Classifier is the default classifier implementation, which uses [acryl-datahub-classify](https://pypi.org/project/acryl-datahub-classify/) library to predict info types. + +### Config Details + +| Field | Required | Type | Description | Default | +| ------------------------------------------------------ | ------------------------------------------------------ | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| confidence_level_threshold | | number | | 0.68 | +| strip_exclusion_formatting | | bool | A flag that determines whether the exclusion list uses exact matching or format stripping (case-insensitivity, punctuation removal, and special character removal). | True | +| info_types | | list[string] | List of infotypes to be predicted. By default, all supported infotypes are considered, along with any custom infotypes configured in `info_types_config`. | None | +| info_types_config | Configuration details for infotypes | Dict[str, InfoTypeConfig] | | See [reference_input.py](https://github.com/acryldata/datahub-classify/blob/main/datahub-classify/src/datahub_classify/reference_input.py) for default configuration. | +| info_types_config.`key`.prediction_factors_and_weights | ❓ (required if info_types_config.`key` is set) | Dict[str,number] | Factors and their weights to consider when predicting info types | | +| info_types_config.`key`.exclude_name | | list[string] | Optional list of names to exclude from classification. | None | +| info_types_config.`key`.name | | NameFactorConfig (see below for fields) | | | +| info_types_config.`key`.name.regex | | Array of string | List of regex patterns the column name follows for the info type | ['.*'] | +| info_types_config.`key`.description | | DescriptionFactorConfig (see below for fields) | | | +| info_types_config.`key`.description.regex | | Array of string | List of regex patterns the column description follows for the info type | ['.*'] | +| info_types_config.`key`.datatype | | DataTypeFactorConfig (see below for fields) | | | +| info_types_config.`key`.datatype.type | | Array of string | List of data types for the info type | ['.*'] | +| info_types_config.`key`.values | | ValuesFactorConfig (see below for fields) | | | +| info_types_config.`key`.values.prediction_type | ❓ (required if info_types_config.`key`.values is set) | string | | None | +| info_types_config.`key`.values.regex | | Array of string | List of regex patterns the column value follows for the info type | None | +| info_types_config.`key`.values.library | | Array of string | Library used for prediction | None | +| minimum_values_threshold | | number | Minimum number of non-null column values required to process `values` prediction factor. | 50 | +| | + +### Supported infotypes + +- `Email_Address` +- `Gender` +- `Credit_Debit_Card_Number` +- `Phone_Number` +- `Street_Address` +- `Full_Name` +- `Age` +- `IBAN` +- `US_Social_Security_Number` +- `Vehicle_Identification_Number` +- `IP_Address_v4` +- `IP_Address_v6` +- `US_Driving_License_Number` +- `Swift_Code` +- Regex based Custom InfoTypes + +## Supported sources + +- All SQL sources + +## Future Work + +- Classification for nested columns (struct, array type) + +## Examples + +### Basic + +```yml +source: + type: snowflake + config: + env: PROD + # Coordinates + account_id: account_name + warehouse: "COMPUTE_WH" + + # Credentials + username: user + password: pass + role: "sysadmin" + + # Options + top_n_queries: 10 + email_domain: mycompany.com + + classification: + enabled: True + classifiers: + - type: datahub +``` + +### Advanced Configuration: Customizing configuration for supported info types + +```yml +source: + type: snowflake + config: + env: PROD + # Coordinates + account_id: account_name + warehouse: "COMPUTE_WH" + + # Credentials + username: user + password: pass + role: "sysadmin" + + # Options + top_n_queries: 10 + email_domain: mycompany.com + + classification: + enabled: True + info_type_to_term: + Email_Address: "Email" + classifiers: + - type: datahub + config: + confidence_level_threshold: 0.7 + info_types_config: + Email_Address: + prediction_factors_and_weights: + name: 0.4 + description: 0 + datatype: 0 + values: 0.6 + name: + regex: + - "^.*mail.*id.*$" + - "^.*id.*mail.*$" + - "^.*mail.*add.*$" + - "^.*add.*mail.*$" + - email + - mail + description: + regex: + - "^.*mail.*id.*$" + - "^.*mail.*add.*$" + - email + - mail + datatype: + type: + - str + values: + prediction_type: regex + regex: + - "[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Z|a-z]{2,}" + library: [] + Gender: + prediction_factors_and_weights: + name: 0.4 + description: 0 + datatype: 0 + values: 0.6 + name: + regex: + - "^.*gender.*$" + - "^.*sex.*$" + - gender + - sex + description: + regex: + - "^.*gender.*$" + - "^.*sex.*$" + - gender + - sex + datatype: + type: + - int + - str + values: + prediction_type: regex + regex: + - male + - female + - man + - woman + - m + - f + - w + - men + - women + library: [] + Credit_Debit_Card_Number: + prediction_factors_and_weights: + name: 0.4 + description: 0 + datatype: 0 + values: 0.6 + name: + regex: + - "^.*card.*number.*$" + - "^.*number.*card.*$" + - "^.*credit.*card.*$" + - "^.*debit.*card.*$" + description: + regex: + - "^.*card.*number.*$" + - "^.*number.*card.*$" + - "^.*credit.*card.*$" + - "^.*debit.*card.*$" + datatype: + type: + - str + - int + values: + prediction_type: regex + regex: + - "^4[0-9]{12}(?:[0-9]{3})?$" + - "^(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12}$" + - "^3[47][0-9]{13}$" + - "^3(?:0[0-5]|[68][0-9])[0-9]{11}$" + - "^6(?:011|5[0-9]{2})[0-9]{12}$" + - "^(?:2131|1800|35\\d{3})\\d{11}$" + - "^(6541|6556)[0-9]{12}$" + - "^389[0-9]{11}$" + - "^63[7-9][0-9]{13}$" + - "^9[0-9]{15}$" + - "^(6304|6706|6709|6771)[0-9]{12,15}$" + - "^(5018|5020|5038|6304|6759|6761|6763)[0-9]{8,15}$" + - "^(62[0-9]{14,17})$" + - "^(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14})$" + - "^(4903|4905|4911|4936|6333|6759)[0-9]{12}|(4903|4905|4911|4936|6333|6759)[0-9]{14}|(4903|4905|4911|4936|6333|6759)[0-9]{15}|564182[0-9]{10}|564182[0-9]{12}|564182[0-9]{13}|633110[0-9]{10}|633110[0-9]{12}|633110[0-9]{13}$" + - "^(6334|6767)[0-9]{12}|(6334|6767)[0-9]{14}|(6334|6767)[0-9]{15}$" + library: [] + Phone_Number: + prediction_factors_and_weights: + name: 0.4 + description: 0 + datatype: 0 + values: 0.6 + name: + regex: + - ".*phone.*(num|no).*" + - ".*(num|no).*phone.*" + - ".*[^a-z]+ph[^a-z]+.*(num|no).*" + - ".*(num|no).*[^a-z]+ph[^a-z]+.*" + - ".*mobile.*(num|no).*" + - ".*(num|no).*mobile.*" + - ".*telephone.*(num|no).*" + - ".*(num|no).*telephone.*" + - ".*cell.*(num|no).*" + - ".*(num|no).*cell.*" + - ".*contact.*(num|no).*" + - ".*(num|no).*contact.*" + - ".*landline.*(num|no).*" + - ".*(num|no).*landline.*" + - ".*fax.*(num|no).*" + - ".*(num|no).*fax.*" + - phone + - telephone + - landline + - mobile + - tel + - fax + - cell + - contact + description: + regex: + - ".*phone.*(num|no).*" + - ".*(num|no).*phone.*" + - ".*[^a-z]+ph[^a-z]+.*(num|no).*" + - ".*(num|no).*[^a-z]+ph[^a-z]+.*" + - ".*mobile.*(num|no).*" + - ".*(num|no).*mobile.*" + - ".*telephone.*(num|no).*" + - ".*(num|no).*telephone.*" + - ".*cell.*(num|no).*" + - ".*(num|no).*cell.*" + - ".*contact.*(num|no).*" + - ".*(num|no).*contact.*" + - ".*landline.*(num|no).*" + - ".*(num|no).*landline.*" + - ".*fax.*(num|no).*" + - ".*(num|no).*fax.*" + - phone + - telephone + - landline + - mobile + - tel + - fax + - cell + - contact + datatype: + type: + - int + - str + values: + prediction_type: library + regex: [] + library: + - phonenumbers + Street_Address: + prediction_factors_and_weights: + name: 0.5 + description: 0 + datatype: 0 + values: 0.5 + name: + regex: + - ".*street.*add.*" + - ".*add.*street.*" + - ".*full.*add.*" + - ".*add.*full.*" + - ".*mail.*add.*" + - ".*add.*mail.*" + - add[^a-z]+ + - address + - street + description: + regex: + - ".*street.*add.*" + - ".*add.*street.*" + - ".*full.*add.*" + - ".*add.*full.*" + - ".*mail.*add.*" + - ".*add.*mail.*" + - add[^a-z]+ + - address + - street + datatype: + type: + - str + values: + prediction_type: library + regex: [] + library: + - spacy + Full_Name: + prediction_factors_and_weights: + name: 0.3 + description: 0 + datatype: 0 + values: 0.7 + name: + regex: + - ".*person.*name.*" + - ".*name.*person.*" + - ".*user.*name.*" + - ".*name.*user.*" + - ".*full.*name.*" + - ".*name.*full.*" + - fullname + - name + - person + - user + description: + regex: + - ".*person.*name.*" + - ".*name.*person.*" + - ".*user.*name.*" + - ".*name.*user.*" + - ".*full.*name.*" + - ".*name.*full.*" + - fullname + - name + - person + - user + datatype: + type: + - str + values: + prediction_type: library + regex: [] + library: + - spacy + Age: + prediction_factors_and_weights: + name: 0.65 + description: 0 + datatype: 0 + values: 0.35 + name: + regex: + - age[^a-z]+.* + - ".*[^a-z]+age" + - ".*[^a-z]+age[^a-z]+.*" + - age + description: + regex: + - age[^a-z]+.* + - ".*[^a-z]+age" + - ".*[^a-z]+age[^a-z]+.*" + - age + datatype: + type: + - int + values: + prediction_type: library + regex: [] + library: + - rule_based_logic +``` + +### Advanced Configuration: Specifying Custom InfoType + +```yml +source: + type: snowflake + config: + env: PROD + # Coordinates + account_id: account_name + warehouse: "COMPUTE_WH" + + # Credentials + username: user + password: pass + role: "sysadmin" + + # Options + top_n_queries: 10 + email_domain: mycompany.com + + classification: + enabled: True + classifiers: + - type: datahub + config: + confidence_level_threshold: 0.7 + minimum_values_threshold: 10 + info_types_config: + CloudRegion: + prediction_factors_and_weights: + name: 0 + description: 0 + datatype: 0 + values: 1 + values: + prediction_type: regex + regex: + - "(af|ap|ca|eu|me|sa|us)-(central|north|(north(?:east|west))|south|south(?:east|west)|east|west)-\\d+" + library: [] +``` + +## Additional Resources + +### DataHub Blog + +- [PII Classification just got easier with DataHub](https://medium.com/datahub-project/pii-classification-just-got-easier-with-datahub-6bab2b63abcb) diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/profiling_ingestions.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/profiling_ingestions.md new file mode 100644 index 00000000..1bfb830b --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/profiling_ingestions.md @@ -0,0 +1,92 @@ +--- +title: Profiling ingestions +slug: /metadata-ingestion/docs/dev_guides/profiling_ingestions +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/dev_guides/profiling_ingestions.md +--- +import FeatureAvailability from '@site/src/components/FeatureAvailability'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Profiling ingestions + + + +**🤝 Version compatibility** + +> DataHub Core (Open Source): **0.11.1** | DataHub Cloud: **0.2.12** + +This page documents how to perform memory profiles of ingestion runs. +It is useful when trying to size the amount of resources necessary to ingest some source or when developing new features or sources. + +## How to use + + + + +Create an ingestion as specified in the [Ingestion guide](../../../docs/ui-ingestion.md). + +Add a flag to your ingestion recipe to generate a memray memory dump of your ingestion: + +```yaml +source: ... + +sink: ... + +flags: + generate_memory_profiles: "" +``` + +In the final panel, under the advanced section, add the `debug` datahub package under the **Extra DataHub Plugins** section. +As seen below: + +

+ +

+ +Finally, save and run the ingestion process. + + + +Install the `debug` plugin for DataHub's CLI wherever the ingestion runs: + +```bash +pip install 'acryl-datahub[debug]' +``` + +This will install [memray](https://github.com/bloomberg/memray) in your python environment. + +Add a flag to your ingestion recipe to generate a memray memory dump of your ingestion: + +```yaml +source: ... + +sink: ... + +flags: + generate_memory_profiles: "" +``` + +Finally run the ingestion recipe + +```bash +$ datahub ingest -c recipe.yaml +``` + + + + +Once the ingestion run starts a binary file will be created and appended to during the execution of the ingestion. + +These files follow the pattern `file-.bin` for a unique identification. +Once the ingestion has finished you can use `memray` to analyze the memory dump in a flamegraph view using: + +`$ memray flamegraph file-None-file-2023_09_18-21_38_43.bin` + +This will generate an interactive HTML file for analysis: + +

+ +

+ +`memray` has an extensive set of features for memory investigation. Take a look at their [documentation](https://bloomberg.github.io/memray/overview.html) to see the full feature set. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/reporting_telemetry.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/reporting_telemetry.md new file mode 100644 index 00000000..b99db822 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/reporting_telemetry.md @@ -0,0 +1,112 @@ +--- +title: Datahub's Reporting Framework for Ingestion Job Telemetry +slug: /metadata-ingestion/docs/dev_guides/reporting_telemetry +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/dev_guides/reporting_telemetry.md +--- +# Datahub's Reporting Framework for Ingestion Job Telemetry + +The Datahub's reporting framework allows for configuring reporting providers with the ingestion pipelines to send +telemetry about the ingestion job runs to external systems for monitoring purposes. It is powered by the Datahub's +stateful ingestion framework. The `datahub` reporting provider comes with the standard client installation, +and allows for reporting ingestion job telemetry to the datahub backend as the destination. + +**_NOTE_**: This feature requires the server to be `statefulIngestion` capable. +This is a feature of metadata service with version >= `0.8.20`. + +To check if you are running a stateful ingestion capable server: + +```console +curl http:///config + +{ +models: { }, +statefulIngestionCapable: true, # <-- this should be present and true +retention: "true", +noCode: "true" +} +``` + +## Config details + +The ingestion reporting providers are a list of reporting provider configurations under the `reporting` config +param of the pipeline, each reporting provider configuration begin a type and config pair object. The telemetry data will +be sent to all the reporting providers in this list. + +Note that a `.` is used to denote nested fields, and `[idx]` is used to denote an element of an array of objects in the YAML recipe. + +| Field | Required | Default | Description | +| ----------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `reporting[idx].type` | ✅ | `datahub` | The type of the ingestion reporting provider registered with datahub. | +| `reporting[idx].config` | | The `datahub_api` config if set at pipeline level. Otherwise, the default `DatahubClientConfig`. See the [defaults](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/graph/client.py#L19) here. | The configuration required for initializing the datahub reporting provider. | +| `pipeline_name` | ✅ | | The name of the ingestion pipeline. This is used as a part of the identifying key for the telemetry data reported by each job in the ingestion pipeline. | + +#### Supported sources + +- All sql based sources. +- snowflake_usage. + +#### Sample configuration + +```yaml +source: + type: "snowflake" + config: + username: + password: + role: + host_port: + warehouse: + # Rest of the source specific params ... +# This is mandatory. Changing it will cause old telemetry correlation to be lost. +pipeline_name: "my_snowflake_pipeline_1" + +# Pipeline-level datahub_api configuration. +datahub_api: # Optional. But if provided, this config will be used by the "datahub" ingestion state provider. + server: "http://localhost:8080" + +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" + +reporting: + - type: "datahub" # Required + config: # Optional. + datahub_api: # default value + server: "http://localhost:8080" +``` + +## Reporting Ingestion State Provider (Developer Guide) + +An ingestion reporting state provider is responsible for saving and retrieving the ingestion telemetry +associated with the ingestion runs of various jobs inside the source connector of the ingestion pipeline. +The data model used for capturing the telemetry is [DatahubIngestionRunSummary](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/datajob/datahub/DatahubIngestionRunSummary.pdl). +A reporting ingestion state provider needs to implement the IngestionReportingProviderBase. +interface and register itself with datahub by adding an entry under `datahub.ingestion.reporting_provider.plugins` +key of the entry_points section in [setup.py](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/setup.py) +with its type and implementation class as shown below. + +```python +entry_points = { + # " + "datahub.ingestion.reporting_provider.plugins": [ + "datahub = datahub.ingestion.reporting.datahub_ingestion_run_summary_provider:DatahubIngestionRunSummaryProvider", + "file = datahub.ingestion.reporting.file_reporter:FileReporter", + ], +} +``` + +### Datahub Reporting Ingestion State Provider + +This is the reporting state provider implementation that is available out of the box in datahub. Its type is `datahub` and it is implemented on top +of the `datahub_api` client and the timeseries aspect capabilities of the datahub-backend. + +#### Config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +| -------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| `type` | ✅ | `datahub` | The type of the ingestion reporting provider registered with datahub. | +| `config` | | The `datahub_api` config if set at pipeline level. Otherwise, the default `DatahubClientConfig`. See the [defaults](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/graph/client.py#L19) here. | The configuration required for initializing the datahub reporting provider. | diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/sql_profiles.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/sql_profiles.md new file mode 100644 index 00000000..e9d00125 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/sql_profiles.md @@ -0,0 +1,62 @@ +--- +title: SQL Profiling +slug: /metadata-ingestion/docs/dev_guides/sql_profiles +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/dev_guides/sql_profiles.md +--- +# SQL Profiling + +SQL Profiling collects table level and column level statistics. +The SQL-based profiler does not run alone, but rather can be enabled for other SQL-based sources. +Enabling profiling will slow down ingestion runs. + +:::caution + +Running profiling against many tables or over many rows can run up significant costs. +While we've done our best to limit the expensiveness of the queries the profiler runs, you +should be prudent about the set of tables profiling is enabled on or the frequency +of the profiling runs. + +::: + +## Capabilities + +Extracts: + +- Row and column counts for each table +- For each column, if applicable: + - null counts and proportions + - distinct counts and proportions + - minimum, maximum, mean, median, standard deviation, some quantile values + - histograms or frequencies of unique values + +## Supported Sources + +SQL profiling is supported for all SQL sources. Check the individual source page to verify if it supports profiling. + +## Profiler Implementation + +DataHub is transitioning from Great Expectations (GE) based profiling to a custom SQLAlchemy profiler. + +### Current State + +Two profiler implementations are available: + +1. **GE Profiler** (default): Uses Great Expectations library +2. **SQLAlchemy Profiler** (opt-in): Custom implementation with no external GE dependency + +The SQLAlchemy profiler can be enabled via `profile.method = "sqlalchemy"`: + +```yaml +source: + config: + profiling: + enabled: true + method: sqlalchemy +``` + +### Rollout Plan + +- **Phase 1 (Current):** SQLAlchemy profiler available as opt-in. Users can test and validate. +- **Phase 2 (Future):** SQLAlchemy profiler becomes the default. GE profiler still available for compatibility. +- **Phase 3 (Future):** GE profiler removed from codebase to reduce maintenance and dependencies. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/stateful.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/stateful.md new file mode 100644 index 00000000..c515b201 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/dev_guides/stateful.md @@ -0,0 +1,187 @@ +--- +title: Stateful Ingestion +slug: /metadata-ingestion/docs/dev_guides/stateful +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/dev_guides/stateful.md +--- +# Stateful Ingestion + +The stateful ingestion feature enables sources to be configured to save custom checkpoint states from their +runs, and query these states back from subsequent runs to make decisions about the current run based on the state saved +from the previous run(s) using a supported ingestion state provider. This is an explicit opt-in feature and is not enabled +by default. + +**_NOTE_**: This feature requires the server to be `statefulIngestion` capable. This is a feature of metadata service with version >= `0.8.20`. + +To check if you are running a stateful ingestion capable server: + +```console +curl http:///config + +{ +models: { }, +statefulIngestionCapable: true, # <-- this should be present and true +retention: "true", +noCode: "true" +} +``` + +## Config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +| ------------------------------------------------------------ | -------- | ----------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `source.config.stateful_ingestion.enabled` | | False | The type of the ingestion state provider registered with datahub. | +| `source.config.stateful_ingestion.ignore_old_state` | | False | If set to True, ignores the previous checkpoint state. | +| `source.config.stateful_ingestion.ignore_new_state` | | False | If set to True, ignores the current checkpoint state. | +| `source.config.stateful_ingestion.max_checkpoint_state_size` | | 2^24 (16MB) | The maximum size of the checkpoint state in bytes. | +| `source.config.stateful_ingestion.state_provider` | | The default datahub ingestion state provider configuration. | The ingestion state provider configuration. | +| `pipeline_name` | ✅ | | The name of the ingestion pipeline the checkpoint states of various source connector job runs are saved/retrieved against via the ingestion state provider. | + +NOTE: If either `dry-run` or `preview` mode are set, stateful ingestion will be turned off regardless of the rest of the configuration. + +## Use-cases powered by stateful ingestion. + +Following is the list of current use-cases powered by stateful ingestion in datahub. + +### Stale Entity Removal + +Stateful ingestion can be used to automatically soft-delete the tables and views that are seen in a previous run +but absent in the current run (they are either deleted or no longer desired). + +

+ +

+ +#### Supported sources + +- All sql based sources. + +#### Additional config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +| ------------------------------------------ | -------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `stateful_ingestion.remove_stale_metadata` | | True | Soft-deletes the tables and views that were found in the last successful run but missing in the current run with stateful_ingestion enabled. | + +#### Sample configuration + +```yaml +source: + type: "snowflake" + config: + username: + password: + host_port: + warehouse: + role: + include_tables: True + include_views: True + # Rest of the source specific params ... + ## Stateful Ingestion config ## + stateful_ingestion: + enabled: True # False by default + remove_stale_metadata: True # default value + ## Default state_provider configuration ## + # state_provider: + # type: "datahub" # default value + # This section is needed if the pipeline-level `datahub_api` is not configured. + # config: # default value + # datahub_api: + # server: "http://localhost:8080" + +# The pipeline_name is mandatory for stateful ingestion and the state is tied to this. +# If this is changed after using with stateful ingestion, the previous state will not be available to the next run. +pipeline_name: "my_snowflake_pipeline_1" + +# Pipeline-level datahub_api configuration. +datahub_api: # Optional. But if provided, this config will be used by the "datahub" ingestion state provider. + server: "http://localhost:8080" + +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +### Redundant Run Elimination + +Typically, the usage runs are configured to fetch the usage data for the previous day(or hour) for each run. Once a usage +run has finished, subsequent runs until the following day would be fetching the same usage data. With stateful ingestion, +the redundant fetches can be avoided even if the ingestion job is scheduled to run more frequently than the granularity of +usage ingestion. + +#### Supported sources + +- Snowflake Usage source. + +#### Additional config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +| -------------------------------- | -------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `stateful_ingestion.force_rerun` | | False | Custom-alias for `stateful_ingestion.ignore_old_state`. Prevents a rerun for the same time window if there was a previous successful run. | + +#### Sample Configuration + +```yaml +source: + type: "snowflake-usage-legacy" + config: + username: + password: + role: + host_port: + warehouse: + # Rest of the source specific params ... + ## Stateful Ingestion config ## + stateful_ingestion: + enabled: True # default is false + force_rerun: False # Specific to this source(alias for ignore_old_state), used to override default behavior if True. + +# The pipeline_name is mandatory for stateful ingestion and the state is tied to this. +# If this is changed after using with stateful ingestion, the previous state will not be available to the next run. +pipeline_name: "my_snowflake_usage_ingestion_pipeline_1" +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +## Adding Stateful Ingestion Capability to New Sources (Developer Guide) + +See [this documentation](./add_stateful_ingestion_to_source.md) for more details on how to add stateful ingestion +capability to new sources for the use-cases supported by datahub. + +## The Checkpointing Ingestion State Provider (Developer Guide) + +The ingestion checkpointing state provider is responsible for saving and retrieving the ingestion checkpoint state associated with the ingestion runs +of various jobs inside the source connector of the ingestion pipeline. The checkpointing data model is [DatahubIngestionCheckpoint](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/datajob/datahub/DatahubIngestionCheckpoint.pdl) and it supports any custom state to be stored using the [IngestionCheckpointState](https://github.com/datahub-project/datahub/blob/master/metadata-models/src/main/pegasus/com/linkedin/datajob/datahub/IngestionCheckpointState.pdl#L9). A checkpointing ingestion state provider needs to implement the +[IngestionCheckpointingProviderBase](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/api/ingestion_job_checkpointing_provider_base.py) interface and +register itself with datahub by adding an entry under `datahub.ingestion.checkpointing_provider.plugins` key of the entry_points section in [setup.py](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/setup.py) with its type and implementation class as shown below. + +```python +entry_points = { + # " + "datahub.ingestion.checkpointing_provider.plugins": [ + "datahub = datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider:DatahubIngestionCheckpointingProvider", + ], +} +``` + +### Datahub Checkpointing Ingestion State Provider + +This is the state provider implementation that is available out of the box. Its type is `datahub` and it is implemented on top +of the `datahub_api` client and the timeseries aspect capabilities of the datahub-backend. + +#### Config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +| ----------------------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | +| `state_provider.type` | | `datahub` | The type of the ingestion state provider registered with datahub | +| `state_provider.config` | | The `datahub_api` config if set at pipeline level. Otherwise, the default `DatahubClientConfig`. See the [defaults](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/graph/client.py#L19) here. | The configuration required for initializing the state provider. | diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/dataset_transformer.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/dataset_transformer.md new file mode 100644 index 00000000..bf8bd767 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/dataset_transformer.md @@ -0,0 +1,1871 @@ +--- +title: Dataset +sidebar_label: Dataset +slug: /metadata-ingestion/docs/transformer/dataset_transformer +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/transformer/dataset_transformer.md +--- + +# Dataset Transformers + +The below table shows transformer which can transform aspects of entity [Dataset](../../../docs/generated/metamodel/entities/dataset.md). + +| Dataset Aspect | Transformer | +| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `status` | - [Mark Dataset status](#mark-dataset-status) | +| `ownership` | - [Simple Add Dataset ownership](#simple-add-dataset-ownership)
- [Pattern Add Dataset ownership](#pattern-add-dataset-ownership)
- [Simple Remove Dataset Ownership](#simple-remove-dataset-ownership)
- [Extract Ownership from Tags](#extract-ownership-from-tags)
- [Clean suffix prefix from Ownership](#clean-suffix-prefix-from-ownership) | +| `globalTags` | - [Simple Add Dataset globalTags ](#simple-add-dataset-globaltags)
- [Pattern Add Dataset globalTags](#pattern-add-dataset-globaltags)
- [Add Dataset globalTags](#add-dataset-globaltags) | +| `browsePaths` | - [Set Dataset browsePath](#set-dataset-browsepath) | +| `glossaryTerms` | - [Simple Add Dataset glossaryTerms ](#simple-add-dataset-glossaryterms)
- [Pattern Add Dataset glossaryTerms](#pattern-add-dataset-glossaryterms)
- [Tags to Term Mapping](#tags-to-term-mapping) | +| `schemaMetadata` | - [Pattern Add Dataset Schema Field glossaryTerms](#pattern-add-dataset-schema-field-glossaryterms)
- [Pattern Add Dataset Schema Field globalTags](#pattern-add-dataset-schema-field-globaltags) | +| `datasetProperties` | - [Simple Add Dataset datasetProperties](#simple-add-dataset-datasetproperties)
- [Add Dataset datasetProperties](#add-dataset-datasetproperties) | +| `domains` | - [Simple Add Dataset domains](#simple-add-dataset-domains)
- [Pattern Add Dataset domains](#pattern-add-dataset-domains)
- [Domain Mapping Based on Tags](#domain-mapping-based-on-tags) | +| `dataProduct` | - [Simple Add Dataset dataProduct ](#simple-add-dataset-dataproduct)
- [Pattern Add Dataset dataProduct](#pattern-add-dataset-dataproduct)
- [Add Dataset dataProduct](#add-dataset-dataproduct) | + +## Extract Ownership from Tags + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------------------------- | -------- | -------------- | ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tag_pattern` | | str | | Regex to use for tags to match against. Supports Regex to match a pattern which is used to remove content. Rest of string is considered owner ID for creating owner URN. | +| `is_user` | | bool | `true` | Whether should be consider a user or not. If `false` then considered a group. | +| `tag_character_mapping` | | dict[str, str] | | A mapping of tag character to datahub owner character. If provided, `tag_pattern` config should be matched against converted tag as per mapping | +| `email_domain` | | str | | If set then this is appended to create owner URN. | +| `extract_owner_type_from_tag_pattern` | | str | `false` | Whether to extract an owner type from provided tag pattern first group. If `true`, no need to provide owner_type and owner_type_urn config. For example: if provided tag pattern is `(.*)_owner_email:` and actual tag is `developer_owner_email`, then extracted owner type will be `developer`. | +| `owner_type` | | str | `TECHNICAL_OWNER` | Ownership type. | +| `owner_type_urn` | | str | `None` | Set to a custom ownership type's URN if using custom ownership. | + +Let’s suppose we’d like to add a dataset ownerships based on part of dataset tags. To do so, we can use the `extract_ownership_from_tags` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "owner_email:" +``` + +So if we have input dataset tag like + +- `urn:li:tag:owner_email:abc@email.com` +- `urn:li:tag:owner_email:xyz@email.com` + +The portion of the tag after the matched tag pattern will be converted into an owner. Hence users `abc@email.com` and `xyz@email.com` will be added as owners. + +### Examples + +- Add owners, however owner should be considered as group and also email domain not provided in tag string. For example: from tag urn `urn:li:tag:owner:abc` extracted owner urn should be `urn:li:corpGroup:abc@email.com` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "owner:" + is_user: false + email_domain: "email.com" + ``` +- Add owners, however owner type and owner type urn wanted to provide externally. For example: from tag urn `urn:li:tag:owner_email:abc@email.com` owner type should be `CUSTOM` and owner type urn as `"urn:li:ownershipType:data_product"` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "owner_email:" + owner_type: "CUSTOM" + owner_type_urn: "urn:li:ownershipType:data_product" + ``` +- Add owners, however some tag characters needs to replace with some other characters before extracting owner. For example: from tag urn `urn:li:tag:owner__email:abc--xyz-email_com` extracted owner urn should be `urn:li:corpGroup:abc.xyz@email.com` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "owner_email:" + tag_character_mapping: + "_": "." + "-": "@" + "--": "-" + "__": "_" + ``` +- Add owners, however owner type also need to extracted from tag pattern. For example: from tag urn `urn:li:tag:data_producer_owner_email:abc@email.com` extracted owner type should be `data_producer` then config would look like this: + ```yaml + transformers: + - type: "extract_ownership_from_tags" + config: + tag_pattern: "(.*)_owner_email:" + extract_owner_type_from_tag_pattern: true + ``` + +## Clean suffix prefix from Ownership + +### Config Details + +| Field | Required | Type | Default | Description | +| --------------------- | -------- | ------------ | ------- | ----------------------------------------------------- | +| `pattern_for_cleanup` | ✅ | list[string] | | List of suffix/prefix to remove from the Owner URN(s) | + +Matches against a Onwer URN and remove the matching part from the Owner URN + +```yaml +transformers: + - type: "pattern_cleanup_ownership" + config: + pattern_for_cleanup: + - "ABCDEF" + - (?<=_)(\w+) +``` + +## Mark Dataset Status + +### Config Details + +| Field | Required | Type | Default | Description | +| --------- | -------- | ------- | ------- | ------------------------------------------- | +| `removed` | ✅ | boolean | | Flag to control visbility of dataset on UI. | + +If you would like to stop a dataset from appearing in the UI, then you need to mark the status of the dataset as removed. + +You can use this transformer in your source recipe to mark status as removed. + +```yaml +transformers: + - type: "mark_dataset_status" + config: + removed: true +``` + +## Simple Add Dataset ownership + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | ------------ | ----------- | ---------------------------------------------------------------------------------------------------------- | +| `owner_urns` | ✅ | list[string] | | List of owner urns. | +| `ownership_type` | | string | "DATAOWNER" | ownership type of the owners (either as enum or ownership type urn) | +| `replace_existing` | | boolean | `false` | Whether to remove ownership from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | +| `on_conflict` | | enum | `DO_UPDATE` | Whether to make changes if domains already exist. If set to DO_NOTHING, `semantics` setting is irrelevant. | + +For transformer behaviour on `replace_existing` and `semantics`, please refer section [Relationship Between replace_existing And semantics](#relationship-between-replace_existing-and-semantics). + +
+Let’s suppose we’d like to append a series of users who we know to own a dataset but aren't detected during normal ingestion. To do so, we can use the `simple_add_dataset_ownership` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +Below configuration will add listed owner_urns in ownership aspect + +```yaml +transformers: + - type: "simple_add_dataset_ownership" + config: + owner_urns: + - "urn:li:corpuser:username1" + - "urn:li:corpuser:username2" + - "urn:li:corpGroup:groupname" + ownership_type: "PRODUCER" +``` + +`simple_add_dataset_ownership` can be configured in below different way + +- Add owners, however replace existing owners sent by ingestion source + ```yaml + transformers: + - type: "simple_add_dataset_ownership" + config: + replace_existing: true # false is default behaviour + owner_urns: + - "urn:li:corpuser:username1" + - "urn:li:corpuser:username2" + - "urn:li:corpGroup:groupname" + ownership_type: "urn:li:ownershipType:__system__producer" + ``` +- Add owners, however overwrite the owners available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_ownership" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + owner_urns: + - "urn:li:corpuser:username1" + - "urn:li:corpuser:username2" + - "urn:li:corpGroup:groupname" + ownership_type: "urn:li:ownershipType:__system__producer" + ``` +- Add owners, however keep the owners available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_ownership" + config: + semantics: PATCH + owner_urns: + - "urn:li:corpuser:username1" + - "urn:li:corpuser:username2" + - "urn:li:corpGroup:groupname" + ownership_type: "PRODUCER" + ``` + +## Pattern Add Dataset ownership + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | -------------------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `owner_pattern` | ✅ | map[regx, list[urn]] | | entity urn with regular expression and list of owners urn apply to matching entity urn. | +| `ownership_type` | | string | "DATAOWNER" | ownership type of the owners (either as enum or ownership type urn) | +| `replace_existing` | | boolean | `false` | Whether to remove owners from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | +| `is_container` | | bool | `false` | Whether to also consider a container or not. If true, then ownership will be attached to both the dataset and its container. | +| `on_conflict` | | enum | `DO_UPDATE` | Whether to make changes if domains already exist. If set to DO_NOTHING, `semantics` setting is irrelevant. | + +let’s suppose we’d like to append a series of users who we know to own a different dataset from a data source but aren't detected during normal ingestion. To do so, we can use the `pattern_add_dataset_ownership` module that’s included in the ingestion framework. This will match the pattern to `urn` of the dataset and assign the respective owners. + +If the is_container field is set to true, the module will not only attach the ownerships to the matching datasets but will also find and attach containers associated with those datasets. This means that both the datasets and their containers will be associated with the specified owners. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "pattern_add_dataset_ownership" + config: + owner_pattern: + rules: + ".*example1.*": ["urn:li:corpuser:username1"] + ".*example2.*": ["urn:li:corpuser:username2"] + ownership_type: "DEVELOPER" +``` + +`pattern_add_dataset_ownership` can be configured in below different way + +- Add owner, however replace existing owner sent by ingestion source + ```yaml + transformers: + - type: "pattern_add_dataset_ownership" + config: + replace_existing: true # false is default behaviour + owner_pattern: + rules: + ".*example1.*": ["urn:li:corpuser:username1"] + ".*example2.*": ["urn:li:corpuser:username2"] + ownership_type: "urn:li:ownershipType:__system__producer" + ``` +- Add owner, however overwrite the owners available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_ownership" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + owner_pattern: + rules: + ".*example1.*": ["urn:li:corpuser:username1"] + ".*example2.*": ["urn:li:corpuser:username2"] + ownership_type: "urn:li:ownershipType:__system__producer" + ``` +- Add owner, however keep the owners available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_ownership" + config: + semantics: PATCH + owner_pattern: + rules: + ".*example1.*": ["urn:li:corpuser:username1"] + ".*example2.*": ["urn:li:corpuser:username2"] + ownership_type: "PRODUCER" + ``` +- Add owner to dataset and its containers + ```yaml + transformers: + - type: "pattern_add_dataset_ownership" + config: + is_container: true + replace_existing: true # false is default behaviour + semantics: PATCH / OVERWRITE # Based on user + owner_pattern: + rules: + ".*example1.*": ["urn:li:corpuser:username1"] + ".*example2.*": ["urn:li:corpuser:username2"] + ownership_type: "PRODUCER" + ``` + ⚠️ Warning: + When working with two datasets in the same container but with different owners, all owners will be added for that dataset containers. + +For example: + +```yaml +transformers: + - type: "pattern_add_dataset_ownership" + config: + is_container: true + owner_pattern: + rules: + ".*example1.*": ["urn:li:corpuser:username1"] + ".*example2.*": ["urn:li:corpuser:username2"] +``` + +If example1 and example2 are in the same container, then both urns `urn:li:corpuser:username1` and `urn:li:corpuser:username2` will be added for respective dataset containers. + +## Simple Remove Dataset ownership + +If we wanted to clear existing owners sent by ingestion source we can use the `simple_remove_dataset_ownership` transformer which removes all owners sent by the ingestion source. + +```yaml +transformers: + - type: "simple_remove_dataset_ownership" + config: {} +``` + +The main use case of `simple_remove_dataset_ownership` is to remove incorrect owners present in the source. You can use it along with the [Simple Add Dataset ownership](#simple-add-dataset-ownership) to remove wrong owners and add the correct ones. + +Note that whatever owners you send via `simple_remove_dataset_ownership` will overwrite the owners present in the UI. + +## Extract Dataset globalTags + +### Config Details + +| Field | Required | Type | Default | Description | +| -------------------- | -------- | ------- | ----------- | ------------------------------------------------------------------- | +| `extract_tags_from` | ✅ | string | `urn` | Which field to extract tag from. Currently only `urn` is supported. | +| `extract_tags_regex` | ✅ | string | `.*` | Regex to use to extract tag. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +Let’s suppose we’d like to add a dataset tags based on part of urn. To do so, we can use the `extract_dataset_tags` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "extract_dataset_tags" + config: + extract_tags_from: "urn" + extract_tags_regex: ".([^._]*)_" +``` + +So if we have input URNs like + +- `urn:li:dataset:(urn:li:dataPlatform:kafka,clusterid.USA-ops-team_table1,PROD)` +- `urn:li:dataset:(urn:li:dataPlatform:kafka,clusterid.Canada-marketing_table1,PROD)` + +a tag called `USA-ops-team` and `Canada-marketing` will be added to them respectively. This is helpful in case you are using prefixes in your datasets to segregate different things. Now you can turn that segregation into a tag on your dataset in DataHub for further use. + +## Simple Add Dataset globalTags + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | ------------ | ----------- | ------------------------------------------------------------------ | +| `tag_urns` | ✅ | list[string] | | List of globalTags urn. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +Let’s suppose we’d like to add a set of dataset tags. To do so, we can use the `simple_add_dataset_tags` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "simple_add_dataset_tags" + config: + tag_urns: + - "urn:li:tag:NeedsDocumentation" + - "urn:li:tag:Legacy" +``` + +`simple_add_dataset_tags` can be configured in below different way + +- Add tags, however replace existing tags sent by ingestion source + ```yaml + transformers: + - type: "simple_add_dataset_tags" + config: + replace_existing: true # false is default behaviour + tag_urns: + - "urn:li:tag:NeedsDocumentation" + - "urn:li:tag:Legacy" + ``` +- Add tags, however overwrite the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_tags" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + tag_urns: + - "urn:li:tag:NeedsDocumentation" + - "urn:li:tag:Legacy" + ``` +- Add tags, however keep the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_tags" + config: + semantics: PATCH + tag_urns: + - "urn:li:tag:NeedsDocumentation" + - "urn:li:tag:Legacy" + ``` + +## Pattern Add Dataset globalTags + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | -------------------- | ----------- | ------------------------------------------------------------------------------------- | +| `tag_pattern` | ✅ | map[regx, list[urn]] | | Entity urn with regular expression and list of tags urn apply to matching entity urn. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +Let’s suppose we’d like to append a series of tags to specific datasets. To do so, we can use the `pattern_add_dataset_tags` module that’s included in the ingestion framework. This will match the regex pattern to `urn` of the dataset and assign the respective tags urns given in the array. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "pattern_add_dataset_tags" + config: + tag_pattern: + rules: + ".*example1.*": ["urn:li:tag:NeedsDocumentation", "urn:li:tag:Legacy"] + ".*example2.*": ["urn:li:tag:NeedsDocumentation"] +``` + +`pattern_add_dataset_tags` can be configured in below different way + +- Add tags, however replace existing tags sent by ingestion source + ```yaml + transformers: + - type: "pattern_add_dataset_tags" + config: + replace_existing: true # false is default behaviour + tag_pattern: + rules: + ".*example1.*": + ["urn:li:tag:NeedsDocumentation", "urn:li:tag:Legacy"] + ".*example2.*": ["urn:li:tag:NeedsDocumentation"] + ``` +- Add tags, however overwrite the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_tags" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + tag_pattern: + rules: + ".*example1.*": + ["urn:li:tag:NeedsDocumentation", "urn:li:tag:Legacy"] + ".*example2.*": ["urn:li:tag:NeedsDocumentation"] + ``` +- Add tags, however keep the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_tags" + config: + semantics: PATCH + tag_pattern: + rules: + ".*example1.*": + ["urn:li:tag:NeedsDocumentation", "urn:li:tag:Legacy"] + ".*example2.*": ["urn:li:tag:NeedsDocumentation"] + ``` + +## Add Dataset globalTags + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | ------------------------------------------ | ----------- | -------------------------------------------------------------------------- | +| `get_tags_to_add` | ✅ | callable[[str], list[TagAssociationClass]] | | A function which takes entity urn as input and return TagAssociationClass. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +If you'd like to add more complex logic for assigning tags, you can use the more generic add_dataset_tags transformer, which calls a user-provided function to determine the tags for each dataset. + +```yaml +transformers: + - type: "add_dataset_tags" + config: + get_tags_to_add: "." +``` + +Then define your function to return a list of TagAssociationClass tags, for example: + +```python +import logging + +import datahub.emitter.mce_builder as builder +from datahub.metadata.schema_classes import ( + TagAssociationClass +) + +def custom_tags(entity_urn: str) -> List[TagAssociationClass]: + """Compute the tags to associate to a given dataset.""" + + tag_strings = [] + + ### Add custom logic here + tag_strings.append('custom1') + tag_strings.append('custom2') + + tag_strings = [builder.make_tag_urn(tag=n) for n in tag_strings] + tags = [TagAssociationClass(tag=tag) for tag in tag_strings] + + logging.info(f"Tagging dataset {entity_urn} with {tag_strings}.") + return tags +``` + +Finally, you can install and use your custom transformer as [shown here](#installing-the-package). + +`add_dataset_tags` can be configured in below different way + +- Add tags, however replace existing tags sent by ingestion source + ```yaml + transformers: + - type: "add_dataset_tags" + config: + replace_existing: true # false is default behaviour + get_tags_to_add: "." + ``` +- Add tags, however overwrite the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "add_dataset_tags" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + get_tags_to_add: "." + ``` +- Add tags, however keep the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "add_dataset_tags" + config: + semantics: PATCH + get_tags_to_add: "." + ``` + +## Set Dataset browsePath + +> **⚠️ DEPRECATED:** This transformer will be removed in v1.3.0. Use [set browse_path](./universal_transformers.md#set-browsepaths) instead. + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | ------------ | ----------- | ------------------------------------------------------------------ | +| `path_templates` | ✅ | list[string] | | List of path templates. | +| `replace_existing` | | boolean | `false` | Whether to remove browsePath from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +If you would like to add to browse paths of dataset can use this transformer. There are 3 optional variables that you can use to get information from the dataset `urn`: + +- ENV: env passed (default: prod) +- PLATFORM: `mysql`, `postgres` or different platform supported by datahub +- DATASET_PARTS: slash separated parts of dataset name. e.g. `database_name/schema_name/[table_name]` for postgres + +e.g. this can be used to create browse paths like `/prod/postgres/superset/public/logs` for table `superset.public.logs` in a `postgres` database + +```yaml +transformers: + - type: "set_dataset_browse_path" + config: + path_templates: + - /ENV/PLATFORM/DATASET_PARTS +``` + +If you don't want the environment but wanted to add something static in the browse path like the database instance name you can use this. + +```yaml +transformers: + - type: "set_dataset_browse_path" + config: + path_templates: + - /PLATFORM/marketing_db/DATASET_PARTS +``` + +It will create browse path like `/mysql/marketing_db/sales/orders` for a table `sales.orders` in `mysql` database instance. + +You can use this to add multiple browse paths. Different people might know the same data assets by different names. + +```yaml +transformers: + - type: "set_dataset_browse_path" + config: + path_templates: + - /PLATFORM/marketing_db/DATASET_PARTS + - /data_warehouse/DATASET_PARTS +``` + +This will add 2 browse paths like `/mysql/marketing_db/sales/orders` and `/data_warehouse/sales/orders` for a table `sales.orders` in `mysql` database instance. + +Default behaviour of the transform is to add new browse paths, you can optionally set `replace_existing: True` so +the transform becomes a _set_ operation instead of an _append_. + +```yaml +transformers: + - type: "set_dataset_browse_path" + config: + replace_existing: True + path_templates: + - /ENV/PLATFORM/DATASET_PARTS +``` + +In this case, the resulting dataset will have only 1 browse path, the one from the transform. + +`set_dataset_browse_path` can be configured in below different way + +- Add browsePath, however replace existing browsePath sent by ingestion source + ```yaml + transformers: + - type: "set_dataset_browse_path" + config: + replace_existing: true # false is default behaviour + path_templates: + - /PLATFORM/marketing_db/DATASET_PARTS + ``` +- Add browsePath, however overwrite the browsePath available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "set_dataset_browse_path" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + path_templates: + - /PLATFORM/marketing_db/DATASET_PARTS + ``` +- Add browsePath, however keep the browsePath available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "set_dataset_browse_path" + config: + semantics: PATCH + path_templates: + - /PLATFORM/marketing_db/DATASET_PARTS + ``` + +## Simple Add Dataset glossaryTerms + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | ------------ | ----------- | --------------------------------------------------------------------- | +| `term_urns` | ✅ | list[string] | | List of glossaryTerms urn. | +| `replace_existing` | | boolean | `false` | Whether to remove glossaryTerms from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +We can use a similar convention to associate [Glossary Terms](../../../docs/generated/ingestion/sources/business-glossary.md) to datasets. +We can use the `simple_add_dataset_terms` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "simple_add_dataset_terms" + config: + term_urns: + - "urn:li:glossaryTerm:Email" + - "urn:li:glossaryTerm:Address" +``` + +`simple_add_dataset_terms` can be configured in below different way + +- Add terms, however replace existing terms sent by ingestion source + ```yaml + transformers: + - type: "simple_add_dataset_terms" + config: + replace_existing: true # false is default behaviour + term_urns: + - "urn:li:glossaryTerm:Email" + - "urn:li:glossaryTerm:Address" + ``` +- Add terms, however overwrite the terms available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_terms" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + term_urns: + - "urn:li:glossaryTerm:Email" + - "urn:li:glossaryTerm:Address" + ``` +- Add terms, however keep the terms available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_terms" + config: + semantics: PATCH + term_urns: + - "urn:li:glossaryTerm:Email" + - "urn:li:glossaryTerm:Address" + ``` + +## Pattern Add Dataset glossaryTerms + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | -------------------- | ----------- | ---------------------------------------------------------------------------------------------- | +| `term_pattern` | ✅ | map[regx, list[urn]] | | entity urn with regular expression and list of glossaryTerms urn apply to matching entity urn. | +| `replace_existing` | | boolean | `false` | Whether to remove glossaryTerms from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +We can add glossary terms to datasets based on a regex filter. + +```yaml +transformers: + - type: "pattern_add_dataset_terms" + config: + term_pattern: + rules: + ".*example1.*": + ["urn:li:glossaryTerm:Email", "urn:li:glossaryTerm:Address"] + ".*example2.*": ["urn:li:glossaryTerm:PostalCode"] +``` + +`pattern_add_dataset_terms` can be configured in below different way + +- Add terms, however replace existing terms sent by ingestion source + + ```yaml + transformers: + - type: "pattern_add_dataset_terms" + config: + replace_existing: true # false is default behaviour + term_pattern: + rules: + ".*example1.*": + ["urn:li:glossaryTerm:Email", "urn:li:glossaryTerm:Address"] + ".*example2.*": ["urn:li:glossaryTerm:PostalCode"] + ``` + +- Add terms, however overwrite the terms available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_terms" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + term_pattern: + rules: + ".*example1.*": + ["urn:li:glossaryTerm:Email", "urn:li:glossaryTerm:Address"] + ".*example2.*": ["urn:li:glossaryTerm:PostalCode"] + ``` +- Add terms, however keep the terms available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_terms" + config: + semantics: PATCH + term_pattern: + rules: + ".*example1.*": + ["urn:li:glossaryTerm:Email", "urn:li:glossaryTerm:Address"] + ".*example2.*": ["urn:li:glossaryTerm:PostalCode"] + ``` + +## Tags to Term Mapping + +### Config Details + +| Field | Required | Type | Default | Description | +| ----------- | -------- | --------- | ----------- | ---------------------------------------------------------------------------------------------- | +| `tags` | ✅ | List[str] | | List of tag names based on which terms will be created and associated with the dataset. | +| `semantics` | | enum | "OVERWRITE" | Determines whether to OVERWRITE or PATCH the terms associated with the dataset on DataHub GMS. | + +
+ +The `tags_to_term` transformer is designed to map specific tags to glossary terms within DataHub. It takes a configuration of tags that should be translated into corresponding glossary terms. This transformer can apply these mappings to any tags found either at the column level of a dataset or at the dataset top level. + +When specifying tags in the configuration, use the tag's simple name rather than the full tag URN. + +For example, instead of using the tag URN `urn:li:tag:snowflakedb.snowflakeschema.tag_name:tag_value`, you should specify just the tag name `tag_name` in the mapping configuration. + +```yaml +transformers: + - type: "tags_to_term" + config: + semantics: OVERWRITE # OVERWRITE is the default behavior + tags: + - "tag_name" +``` + +The `tags_to_term` transformer can be configured in the following ways: + +- Add terms based on tags, however overwrite the terms available for the dataset on DataHub GMS + +```yaml +transformers: + - type: "tags_to_term" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + tags: + - "example1" + - "example2" + - "example3" +``` + +- Add terms based on tags, however keep the terms available for the dataset on DataHub GMS + +```yaml +transformers: + - type: "tags_to_term" + config: + semantics: PATCH + tags: + - "example1" + - "example2" + - "example3" +``` + +## Tags to Structured Properties + +### Description + +The `tags_to_structured_properties` transformer converts DataHub Tags into Structured Properties, supporting both key-value formatted tags (e.g., `dept:Finance`) and simple tag names mapped via configuration. Structured Properties must be created in DataHub before running the transformation - they are not created dynamically. + +Some error scenarios that will cause the transformation (and pipeline) to fail include: + +- **Invalid property value**: Tag value doesn't match the allowed values defined in the Structured Property +- **Property doesn't exist**: Transformed property URN references a non-existent Structured Property + +These failures trigger backend validation errors and are expected behavior - they indicate configuration issues that require either creating the missing Structured Properties in DataHub or adjusting the allowed values for existing properties. + +### Config Details + +| Field | Required | Type | Default | Description | +| ----------------------------- | -------- | ------------------------------------------- | ------- | ------------------------------------------------------------------------- | +| `process_key_value_tags` | | boolean | `false` | Enable parsing of key:value formatted tags | +| `key_value_separator` | | string | `:` | Separator character for key-value tags | +| `key_value_property_prefix` | | string | `""` | Prefix to add to property IDs derived from key-value tags | +| `tag_structured_property_map` | | map[property_id, StructuredPropertyMapping] | `{}` | Map of structured property IDs to tag keyword configurations | +| `remove_original_tags` | | boolean | `false` | Whether to remove original tags after converting to structured properties | +| `semantics` | | enum | `PATCH` | Whether to OVERWRITE or PATCH the structured properties on DataHub GMS | + +The `tags_to_structured_properties` transformer converts DataHub tags into structured properties. It supports two modes: + +1. **Key-Value Tags**: Tags formatted as `key:value` (e.g., `dept:Finance`) where the key becomes the property name +2. **Keyword Tags**: Simple tag names mapped to properties via configuration + +This transformer works with any source that emits tags (Tableau, dbt, Snowflake, etc.) and can be used to migrate from tags to structured properties. + +### Key-Value Tag Example + +Convert tags like `department:Finance` and `sensitivity:PII` into structured properties: + +```yaml +transformers: + - type: "tags_to_structured_properties" + config: + process_key_value_tags: true + key_value_separator: ":" + key_value_property_prefix: "io.company." + remove_original_tags: false +``` + +With this config: + +- Tag `department:Finance` → Property `io.company.department` with value `Finance` +- Tag `sensitivity:PII` → Property `io.company.sensitivity` with value `PII` + +### Keyword Tag Mapping Example + +Map specific tag keywords to predefined structured properties: + +```yaml +transformers: + - type: "tags_to_structured_properties" + config: + tag_structured_property_map: + "io.company.department": + values: ["Finance", "Sales", "Marketing", "Engineering"] + "io.company.dataClassification": + values: ["PII", "Confidential", "Public"] + "io.company.qualityLevel": + values: ["Certified", "Draft", "Deprecated"] + remove_original_tags: false +``` + +With this config: + +- Tag `Finance` → Property `io.company.department` with value `Finance` +- Tag `PII` → Property `io.company.dataClassification` with value `PII` +- Tag `Certified` → Property `io.company.qualityLevel` with value `Certified` + +### Combined Example + +Use both key-value parsing and keyword mapping together: + +```yaml +transformers: + - type: "tags_to_structured_properties" + config: + # Enable key-value processing + process_key_value_tags: true + key_value_separator: ":" + key_value_property_prefix: "io.company." + + # Keyword mappings for tags without key-value format + tag_structured_property_map: + "io.company.department": + values: ["Finance", "Sales", "Marketing"] + "io.company.dataClassification": + values: ["PII", "Confidential"] + + # Keep original tags as well + remove_original_tags: false + + # Merge with existing structured properties + semantics: PATCH +``` + +### Notes + +- If a tag matches both key-value format and keyword mapping, key-value takes precedence +- The first matching property in `tag_structured_property_map` receives the tag value +- Unmatched tags generate a warning but do not fail the transformation +- Multiple tags can map to the same property (values are accumulated as an array) +- Structured properties must be created in DataHub before running the transformation + +## Pattern Add Dataset Schema Field glossaryTerms + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | -------------------- | ----------- | ---------------------------------------------------------------------------------------------- | +| `term_pattern` | ✅ | map[regx, list[urn]] | | entity urn with regular expression and list of glossaryTerms urn apply to matching entity urn. | +| `replace_existing` | | boolean | `false` | Whether to remove glossaryTerms from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +We can add glossary terms to schema fields based on a regex filter. + +Note that only terms from the first matching pattern will be applied. + +```yaml +transformers: + - type: "pattern_add_dataset_schema_terms" + config: + term_pattern: + rules: + ".*email.*": ["urn:li:glossaryTerm:Email"] + ".*name.*": ["urn:li:glossaryTerm:Name"] +``` + +`pattern_add_dataset_schema_terms` can be configured in below different way + +- Add terms, however replace existing terms sent by ingestion source + ```yaml + transformers: + - type: "pattern_add_dataset_schema_terms" + config: + replace_existing: true # false is default behaviour + term_pattern: + rules: + ".*email.*": ["urn:li:glossaryTerm:Email"] + ".*name.*": ["urn:li:glossaryTerm:Name"] + ``` +- Add terms, however overwrite the terms available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_schema_terms" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + term_pattern: + rules: + ".*email.*": ["urn:li:glossaryTerm:Email"] + ".*name.*": ["urn:li:glossaryTerm:Name"] + ``` +- Add terms, however keep the terms available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_schema_terms" + config: + semantics: PATCH + term_pattern: + rules: + ".*email.*": ["urn:li:glossaryTerm:Email"] + ".*name.*": ["urn:li:glossaryTerm:Name"] + ``` + +## Pattern Add Dataset Schema Field globalTags + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | -------------------- | ----------- | ------------------------------------------------------------------------------------- | +| `tag_pattern` | ✅ | map[regx, list[urn]] | | entity urn with regular expression and list of tags urn apply to matching entity urn. | +| `replace_existing` | | boolean | `false` | Whether to remove globalTags from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +We can also append a series of tags to specific schema fields. To do so, we can use the `pattern_add_dataset_schema_tags` transformer. This will match the regex pattern to each schema field path and assign the respective tags urns given in the array. + +Note that the tags from the first matching pattern will be applied, not all matching patterns. + +The config would look like this: + +```yaml +transformers: + - type: "pattern_add_dataset_schema_tags" + config: + tag_pattern: + rules: + ".*email.*": ["urn:li:tag:Email"] + ".*name.*": ["urn:li:tag:Name"] +``` + +`pattern_add_dataset_schema_tags` can be configured in below different way + +- Add tags, however replace existing tag sent by ingestion source + ```yaml + transformers: + - type: "pattern_add_dataset_schema_tags" + config: + replace_existing: true # false is default behaviour + tag_pattern: + rules: + ".*example1.*": + ["urn:li:tag:NeedsDocumentation", "urn:li:tag:Legacy"] + ".*example2.*": ["urn:li:tag:NeedsDocumentation"] + ``` +- Add tags, however overwrite the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_schema_tags" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + tag_pattern: + rules: + ".*example1.*": + ["urn:li:tag:NeedsDocumentation", "urn:li:tag:Legacy"] + ".*example2.*": ["urn:li:tag:NeedsDocumentation"] + ``` +- Add tags, however keep the tags available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_schema_tags" + config: + semantics: PATCH + tag_pattern: + rules: + ".*example1.*": + ["urn:li:tag:NeedsDocumentation", "urn:li:tag:Legacy"] + ".*example2.*": ["urn:li:tag:NeedsDocumentation"] + ``` + +## Simple Add Dataset datasetProperties + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | -------------- | ----------- | ------------------------------------------------------------------------- | +| `properties` | ✅ | dict[str, str] | | Map of key value pair. | +| `replace_existing` | | boolean | `false` | Whether to remove datasetProperties from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +`simple_add_dataset_properties` transformer assigns the properties to dataset entity from the configuration. +`properties` field is a dictionary of string values. Note in case of any key collision, the value in the config will +overwrite the previous value. + +```yaml +transformers: + - type: "simple_add_dataset_properties" + config: + properties: + prop1: value1 + prop2: value2 +``` + +`simple_add_dataset_properties` can be configured in below different way + +- Add dataset-properties, however replace existing dataset-properties sent by ingestion source + ```yaml + transformers: + - type: "simple_add_dataset_properties" + config: + replace_existing: true # false is default behaviour + properties: + prop1: value1 + prop2: value2 + ``` +- Add dataset-properties, however overwrite the dataset-properties available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_properties" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + properties: + prop1: value1 + prop2: value2 + ``` +- Add dataset-properties, however keep the dataset-properties available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_properties" + config: + semantics: PATCH + properties: + prop1: value1 + prop2: value2 + ``` + +## Add Dataset datasetProperties + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------------------- | -------- | -------------------------------------- | ----------- | ------------------------------------------------------------------------- | +| `add_properties_resolver_class` | ✅ | Type[AddDatasetPropertiesResolverBase] | | A class extends from `AddDatasetPropertiesResolverBase` | +| `replace_existing` | | boolean | `false` | Whether to remove datasetProperties from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +If you'd like to add more complex logic for assigning properties, you can use the `add_dataset_properties` transformer, which calls a user-provided class (that extends from `AddDatasetPropertiesResolverBase` class) to determine the properties for each dataset. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "add_dataset_properties" + config: + add_properties_resolver_class: "." +``` + +Then define your class to return a list of custom properties, for example: + +```python +import logging +from typing import Dict +from datahub.ingestion.transformer.add_dataset_properties import AddDatasetPropertiesResolverBase + +class MyPropertiesResolver(AddDatasetPropertiesResolverBase): + def get_properties_to_add(self, entity_urn: str) -> Dict[str, str]: + ### Add custom logic here + properties= {'my_custom_property': 'property value'} + logging.info(f"Adding properties: {properties} to dataset: {entity_urn}.") + return properties +``` + +`add_dataset_properties` can be configured in below different way + +- Add dataset-properties, however replace existing dataset-properties sent by ingestion source + + ```yaml + transformers: + - type: "add_dataset_properties" + config: + replace_existing: true # false is default behaviour + add_properties_resolver_class: "." + ``` + +- Add dataset-properties, however overwrite the dataset-properties available for the dataset on DataHub GMS + + ```yaml + transformers: + - type: "add_dataset_properties" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + add_properties_resolver_class: "." + ``` + +- Add dataset-properties, however keep the dataset-properties available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "add_dataset_properties" + config: + semantics: PATCH + add_properties_resolver_class: "." + ``` + +## Replace ExternalUrl Dataset + +### Config Details + +| Field | Required | Type | Default | Description | +| --------------- | -------- | ------ | ------- | ---------------------------- | +| `input_pattern` | ✅ | string | | String or pattern to replace | +| `replacement` | ✅ | string | | Replacement string | + +Matches the full/partial string in the externalUrl of the dataset properties and replace that with the replacement string + +```yaml +transformers: + - type: "replace_external_url" + config: + input_pattern: '\b\w*hub\b' + replacement: "sub" +``` + +## Replace ExternalUrl Container + +### Config Details + +| Field | Required | Type | Default | Description | +| --------------- | -------- | ------ | ------- | ---------------------------- | +| `input_pattern` | ✅ | string | | String or pattern to replace | +| `replacement` | ✅ | string | | Replacement string | + +Matches the full/partial string in the externalUrl of the container properties and replace that with the replacement string + +```yaml +transformers: + - type: "replace_external_url_container" + config: + input_pattern: '\b\w*hub\b' + replacement: "sub" +``` + +## Clean User URN in DatasetUsageStatistics Aspect + +### Config Details + +| Field | Required | Type | Default | Description | +| --------------------- | -------- | ------------ | ------- | ----------------------------------------------------- | +| `pattern_for_cleanup` | ✅ | list[string] | | List of suffix/prefix to remove from the Owner URN(s) | + +Matches against a User URN in DatasetUsageStatistics aspect and remove the matching part from it + +```yaml +transformers: + - type: "pattern_cleanup_dataset_usage_user" + config: + pattern_for_cleanup: + - "ABCDEF" + - (?<=_)(\w+) +``` + +## Simple Add Dataset domains + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | --------------------- | ----------- | ---------------------------------------------------------------- | +| `domains` | ✅ | list[union[urn, str]] | | List of simple domain name or domain urns. | +| `replace_existing` | | boolean | `false` | Whether to remove domains from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +For transformer behaviour on `replace_existing` and `semantics`, please refer section [Relationship Between replace_existing And semantics](#relationship-between-replace_existing-and-semantics). + +
+ +let’s suppose we’d like to add a series of domain to dataset, in this case you can use `simple_add_dataset_domain` transformer. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +Here we can set domains to either urn (i.e. `urn:li:domain:engineering)` or simple domain name (i.e. engineering) in both of the cases domain should be provisioned on DataHub GMS + +```yaml +transformers: + - type: "simple_add_dataset_domain" + config: + semantics: OVERWRITE + domains: + - urn:li:domain:engineering +``` + +`simple_add_dataset_domain` can be configured in below different way + +- Add domains, however replace existing domains sent by ingestion source + ```yaml + transformers: + - type: "simple_add_dataset_domain" + config: + replace_existing: true # false is default behaviour + domains: + - "urn:li:domain:engineering" + - "urn:li:domain:hr" + ``` +- Add domains, however overwrite the domains available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_domain" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + domains: + - "urn:li:domain:engineering" + - "urn:li:domain:hr" + ``` +- Add domains, however keep the domains available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "simple_add_dataset_domain" + config: + semantics: PATCH + domains: + - "urn:li:domain:engineering" + - "urn:li:domain:hr" + ``` + +## Pattern Add Dataset domains + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | ------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------- | +| `domain_pattern` | ✅ | map[regx, list[union[urn, str]] | | dataset urn with regular expression and list of simple domain name or domain urn need to be apply on matching dataset urn. | +| `replace_existing` | | boolean | `false` | Whether to remove domains from entity sent by ingestion source. | +| `semantics` | | enum | `OVERWRITE` | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | +| `is_container` | | bool | `false` | Whether to also consider a container or not. If true, then domains will be attached to both the dataset and its container. | + +Let’s suppose we’d like to append a series of domain to specific datasets. To do so, we can use the pattern_add_dataset_domain transformer that’s included in the ingestion framework. +This will match the regex pattern to urn of the dataset and assign the respective domain urns given in the array. + +If the is_container field is set to true, the module will not only attach the domains to the matching datasets but will also find and attach containers associated with those datasets. This means that both the datasets and their containers will be associated with the specified owners. + +The config, which we’d append to our ingestion recipe YAML, would look like this: +Here we can set domain list to either urn (i.e. `urn:li:domain:hr)` or simple domain name (i.e. hr) +in both of the cases domain should be provisioned on DataHub GMS + +```yaml +transformers: + - type: "pattern_add_dataset_domain" + config: + semantics: OVERWRITE + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': + ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': + ["urn:li:domain:finance"] +``` + +`pattern_add_dataset_domain` can be configured in below different way + +- Add domains, however replace existing domains sent by ingestion source + ```yaml + transformers: + - type: "pattern_add_dataset_domain" + config: + replace_existing: true # false is default behaviour + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': + ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': + ["urn:li:domain:finance"] + ``` +- Add domains, however overwrite the domains available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "pattern_add_dataset_domain" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': + ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': + ["urn:li:domain:finance"] + ``` +- Add domains, however keep the domains available for the dataset on DataHub GMS + + ```yaml + transformers: + - type: "pattern_add_dataset_domain" + config: + semantics: PATCH + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': + ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': + ["urn:li:domain:finance"] + ``` + +- Add domains to dataset and its containers + ```yaml + transformers: + - type: "pattern_add_dataset_domain" + config: + is_container: true + semantics: PATCH / OVERWRITE # Based on user + domain_pattern: + rules: + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.n.*': + ["hr"] + 'urn:li:dataset:\(urn:li:dataPlatform:postgres,postgres\.public\.t.*': + ["urn:li:domain:finance"] + ``` + ⚠️ Warning: + When working with two datasets in the same container but with different domains, all domains will be added for that dataset containers. + +For example: + +```yaml +transformers: + - type: "pattern_add_dataset_domain" + config: + is_container: true + domain_pattern: + rules: + ".*example1.*": ["hr"] + ".*example2.*": ["urn:li:domain:finance"] +``` + +If example1 and example2 are in the same container, then both domains hr and finance will be added for respective dataset containers. + +## Domain Mapping Based on Tags + +### Config Details + +| Field | Required | Type | Default | Description | +| ---------------- | -------- | -------------- | ----------- | --------------------------------------------------------------------------------------- | +| `domain_mapping` | ✅ | Dict[str, str] | | Dataset Entity tag as key and domain urn or name as value to map with dataset as asset. | +| `semantics` | | enum | "OVERWRITE" | Whether to OVERWRITE or PATCH the entity present on DataHub GMS. | + +
+ +let’s suppose we’d like to add domain to dataset based on tag, in this case you can use `domain_mapping_based_on_tags` transformer. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +Here we can set domains to either urn (i.e. `urn:li:domain:engineering)` or simple domain name (i.e. engineering) in both of the cases domain should be provisioned on DataHub GMS + +When specifying tags within the domain mapping, use the tag's simple name rather than the full tag URN. + +For example, instead of using the tag URN `urn:li:tag:NeedsDocumentation,` you should specify just the simple tag name NeedsDocumentation in the domain mapping configuration + +```yaml +transformers: + - type: "domain_mapping_based_on_tags" + config: + domain_mapping: + "NeedsDocumentation": "urn:li:domain:documentation" +``` + +`domain_mapping_based_on_tags` can be configured in below different way + +- Add domains based on tags, however overwrite the domains available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "domain_mapping_based_on_tags" + config: + semantics: OVERWRITE # OVERWRITE is default behaviour + domain_mapping: + "example1": "urn:li:domain:engineering" + "example2": "urn:li:domain:hr" + ``` +- Add domains based on tags, however keep the domains available for the dataset on DataHub GMS + ```yaml + transformers: + - type: "domain_mapping_based_on_tags" + config: + semantics: PATCH + domain_mapping: + "example1": "urn:li:domain:engineering" + "example2": "urn:li:domain:hr" + ``` + +## Simple Add Dataset dataProduct + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------------------ | -------- | -------------- | ------- | --------------------------------------------------------------------------------------- | +| `dataset_to_data_product_urns` | ✅ | Dict[str, str] | | Dataset Entity urn as key and dataproduct urn as value to create with dataset as asset. | + +Let’s suppose we’d like to add a set of dataproduct with specific datasets as its assets. To do so, we can use the `simple_add_dataset_dataproduct` transformer that’s included in the ingestion framework. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +```yaml +transformers: + - type: "simple_add_dataset_dataproduct" + config: + dataset_to_data_product_urns: + "urn:li:dataset:(urn:li:dataPlatform:bigquery,example1,PROD)": "urn:li:dataProduct:first" + "urn:li:dataset:(urn:li:dataPlatform:bigquery,example2,PROD)": "urn:li:dataProduct:second" +``` + +## Pattern Add Dataset dataProduct + +### Config Details + +| Field | Required | Type | Default | Description | +| -------------------------------------- | -------- | -------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------ | +| `dataset_to_data_product_urns_pattern` | ✅ | map[regx, urn] | | Dataset Entity urn with regular expression and dataproduct urn apply to matching entity urn. | +| `is_container` | | bool | `false` | Whether to also consider a container or not. If true, the data product will be attached to both the dataset and its container. | + +Let’s suppose we’d like to append a series of data products with specific datasets or their containers as assets. To do so, we can use the pattern_add_dataset_dataproduct module that’s included in the ingestion framework. This module matches a regex pattern to the urn of the dataset and creates a data product entity with the given urn, associating the matched datasets as its assets. + +If the is_container field is set to true, the module will not only attach the data product to the matching datasets but will also find and attach the containers associated with those datasets. This means that both the datasets and their containers will be associated with the specified data product. + +The config, which we’d append to our ingestion recipe YAML, would look like this: + +- Add Product to dataset + ```yaml + transformers: + - type: "pattern_add_dataset_dataproduct" + config: + dataset_to_data_product_urns_pattern: + rules: + ".*example1.*": "urn:li:dataProduct:first" + ".*example2.*": "urn:li:dataProduct:second" + ``` +- Add Product to dataset container + ```yaml + transformers: + - type: "pattern_add_dataset_dataproduct" + config: + is_container: true + dataset_to_data_product_urns_pattern: + rules: + ".*example1.*": "urn:li:dataProduct:first" + ".*example2.*": "urn:li:dataProduct:second" + ``` + ⚠️ Warning: + When working with two datasets in the same container but with different data products, only one data product can be attached to the container. + +For example: + +```yaml +transformers: + - type: "pattern_add_dataset_dataproduct" + config: + is_container: true + dataset_to_data_product_urns_pattern: + rules: + ".*example1.*": "urn:li:dataProduct:first" + ".*example2.*": "urn:li:dataProduct:second" +``` + +If example1 and example2 are in the same container, only `urn:li:dataProduct:first` will be added. However, if they are in separate containers, the system works as expected and assigns the correct data product URNs. + +## Add Dataset dataProduct + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------------- | -------- | ------------------------------ | ------- | ---------------------------------------------------------------------------------------- | +| `get_data_product_to_add` | ✅ | callable[[str], Optional[str]] | | A function which takes dataset entity urn as input and return dataproduct urn to create. | + +If you'd like to add more complex logic for creating dataproducts, you can use the more generic add_dataset_dataproduct transformer, which calls a user-provided function to determine the dataproduct to create with specified datasets as its asset. + +```yaml +transformers: + - type: "add_dataset_dataproduct" + config: + get_data_product_to_add: "." +``` + +Then define your function to return a dataproduct entity urn, for example: + +```python +import datahub.emitter.mce_builder as builder + +def custom_dataproducts(entity_urn: str) -> Optional[str]: + """Compute the dataproduct urn to a given dataset urn.""" + + dataset_to_data_product_map = { + builder.make_dataset_urn("bigquery", "example1"): "urn:li:dataProduct:first" + } + return dataset_to_data_product_map.get(dataset_urn) +``` + +Finally, you can install and use your custom transformer as [shown here](#installing-the-package). + +## Relationship Between replace_existing and semantics + +The transformer behaviour mentioned here is in context of `simple_add_dataset_ownership`, however it is applicable for all dataset transformers which are supporting `replace_existing` +and `semantics` configuration attributes, for example `simple_add_dataset_tags` will add or remove tags as per behaviour mentioned in this section. + +`replace_existing` controls whether to remove owners from currently executing ingestion pipeline. + +`semantics` controls whether to overwrite or patch owners present on DataHub GMS server. These owners might be added from DataHub Portal. + +if `replace_existing` is set to `true` and `semantics` is set to `OVERWRITE` then transformer takes below steps + +1. As `replace_existing` is set to `true`, remove the owners from input entity (i.e. dataset) +2. Add owners mentioned in ingestion recipe to input entity +3. As `semantics` is set to `OVERWRITE` no need to fetch owners present on DataHub GMS server for the input entity +4. Return input entity + +if `replace_existing` is set to `true` and `semantics` is set to `PATCH` then transformer takes below steps + +1. `replace_existing` is set to `true`, first remove the owners from input entity (i.e. dataset) +2. Add owners mentioned in ingestion recipe to input entity +3. As `semantics` is set to `PATCH` fetch owners for the input entity from DataHub GMS Server +4. Add owners fetched from DataHub GMS Server to input entity +5. Return input entity + +if `replace_existing` is set to `false` and `semantics` is set to `OVERWRITE` then transformer takes below steps + +1. As `replace_existing` is set to `false`, keep the owners present in input entity as is +2. Add owners mentioned in ingestion recipe to input entity +3. As `semantics` is set to `OVERWRITE` no need to fetch owners from DataHub GMS Server for the input entity +4. Return input entity + +if `replace_existing` is set to `false` and `semantics` is set to `PATCH` then transformer takes below steps + +1. `replace_existing` is set to `false`, keep the owners present in input entity as is +2. Add owners mentioned in ingestion recipe to input entity +3. As `semantics` is set to `PATCH` fetch owners for the input entity from DataHub GMS Server +4. Add owners fetched from DataHub GMS Server to input entity +5. Return input entity + +## Writing a custom transformer from scratch + +In the above couple of examples, we use classes that have already been implemented in the ingestion framework. However, it’s common for more advanced cases to pop up where custom code is required, for instance if you'd like to utilize conditional logic or rewrite properties. In such cases, we can add our own modules and define the arguments it takes as a custom transformer. + +As an example, suppose we want to append a set of ownership fields to our metadata that are dependent upon an external source – for instance, an API endpoint or file – rather than a preset list like above. In this case, we can set a JSON file as an argument to our custom config, and our transformer will read this file and append the included ownership elements to all metadata events. + +Our JSON file might look like the following: + +```json +[ + "urn:li:corpuser:athos", + "urn:li:corpuser:porthos", + "urn:li:corpuser:aramis", + "urn:li:corpGroup:the_three_musketeers" +] +``` + +### Defining a config + +To get started, we’ll initiate an `AddCustomOwnershipConfig` class that inherits from [`datahub.configuration.common.ConfigModel`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/configuration/common.py). The sole parameter will be an `owners_json` which expects a path to a JSON file containing a list of owner URNs. This will go in a file called `custom_transform_example.py`. + +```python +from datahub.configuration.common import ConfigModel + +class AddCustomOwnershipConfig(ConfigModel): + owners_json: str +``` + +### Defining the transformer + +Next, we’ll define the transformer itself, which must inherit from [`datahub.ingestion.api.transform.Transformer`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/api/transform.py). The framework provides a helper class called [`datahub.ingestion.transformer.base_transformer.BaseTransformer`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/transformer/base_transformer.py) that makes it super-simple to write transformers. +First, let's get all our imports in: + +```python +# append these to the start of custom_transform_example.py +import json +from typing import List, Optional + +from datahub.configuration.common import ConfigModel +from datahub.ingestion.api.common import PipelineContext +from datahub.ingestion.transformer.add_dataset_ownership import Semantics +from datahub.ingestion.transformer.base_transformer import ( + BaseTransformer, + SingleAspectTransformer, +) +from datahub.metadata.schema_classes import ( + OwnerClass, + OwnershipClass, + OwnershipTypeClass, +) + +``` + +Next, let's define the base scaffolding for the class: + +```python +# append this to the end of custom_transform_example.py + +class AddCustomOwnership(BaseTransformer, SingleAspectTransformer): + """Transformer that adds owners to datasets according to a callback function.""" + + # context param to generate run metadata such as a run ID + ctx: PipelineContext + # as defined in the previous block + config: AddCustomOwnershipConfig + + def __init__(self, config: AddCustomOwnershipConfig, ctx: PipelineContext): + super().__init__() + self.ctx = ctx + self.config = config + + with open(self.config.owners_json, "r") as f: + raw_owner_urns = json.load(f) + + self.owners = [ + OwnerClass(owner=owner, type=OwnershipTypeClass.DATAOWNER) + for owner in raw_owner_urns + ] +``` + +A transformer must have two functions: a `create()` function for initialization and a `transform()` function for executing the transformation. Transformers that extend `BaseTransformer` and `SingleAspectTransformer` can avoid having to implement the more complex `transform` function and just implement the `transform_aspect` function. + +Let's begin by adding a `create()` method for parsing our configuration dictionary: + +```python +# add this as a function of AddCustomOwnership + +@classmethod +def create(cls, config_dict: dict, ctx: PipelineContext) -> "AddCustomOwnership": + config = AddCustomOwnershipConfig.parse_obj(config_dict) + return cls(config, ctx) +``` + +Next we need to tell the helper classes which entity types and aspect we are interested in transforming. In this case, we want to only process `dataset` entities and transform the `ownership` aspect. + +```python +def entity_types(self) -> List[str]: + return ["dataset"] + +def aspect_name(self) -> str: + return "ownership" +``` + +Finally we need to implement the `transform_aspect()` method that does the work of adding our custom ownership classes. This method will be called be the framework with an optional aspect value filled out if the upstream source produced a value for this aspect. The framework takes care of pre-processing both MCE-s and MCP-s so that the `transform_aspect()` function is only called one per entity. Our job is merely to inspect the incoming aspect (or absence) and produce a transformed value for this aspect. Returning `None` from this method will effectively suppress this aspect from being emitted. + +```python +# add this as a function of AddCustomOwnership + +def transform_aspect( # type: ignore + self, entity_urn: str, aspect_name: str, aspect: Optional[OwnershipClass] +) -> Optional[OwnershipClass]: + + owners_to_add = self.owners + assert aspect is None or isinstance(aspect, OwnershipClass) + + if owners_to_add: + ownership = ( + aspect + if aspect + else OwnershipClass( + owners=[], + ) + ) + ownership.owners.extend(owners_to_add) + + return ownership +``` + +### More Sophistication: Making calls to DataHub during Transformation + +In some advanced cases, you might want to check with DataHub before performing a transformation. A good example for this might be retrieving the current set of owners of a dataset before providing the new set of owners during an ingestion process. To allow transformers to always be able to query the graph, the framework provides them access to the graph through the context object `ctx`. Connectivity to the graph is automatically instantiated anytime the pipeline uses a REST sink. In case you are using the Kafka sink, you can additionally provide access to the graph by configuring it in your pipeline. + +Here is an example of a recipe that uses Kafka as the sink, but provides access to the graph by explicitly configuring the `datahub_api`. + +```yaml +source: + type: mysql + config: + # ..source configs + +sink: + type: datahub-kafka + config: + connection: + bootstrap: localhost:9092 + schema_registry_url: "http://localhost:8081" + +datahub_api: + server: http://localhost:8080 + # standard configs accepted by datahub rest client ... +``` + +#### Advanced Use-Case: Patching Owners + +With the above capability, we can now build more powerful transformers that can check with the server-side state before issuing changes in metadata. +e.g. Here is how the AddDatasetOwnership transformer can now support PATCH semantics by ensuring that it never deletes any owners that are stored on the server. + +```python +def transform_one(self, mce: MetadataChangeEventClass) -> MetadataChangeEventClass: + if not isinstance(mce.proposedSnapshot, DatasetSnapshotClass): + return mce + owners_to_add = self.config.get_owners_to_add(mce.proposedSnapshot) + if owners_to_add: + ownership = builder.get_or_add_aspect( + mce, + OwnershipClass( + owners=[], + ), + ) + ownership.owners.extend(owners_to_add) + + if self.config.semantics == Semantics.PATCH: + assert self.ctx.graph + patch_ownership = AddDatasetOwnership.get_ownership_to_set( + self.ctx.graph, mce.proposedSnapshot.urn, ownership + ) + builder.set_aspect( + mce, aspect=patch_ownership, aspect_type=OwnershipClass + ) + return mce +``` + +### Installing the package + +Now that we've defined the transformer, we need to make it visible to DataHub. The easiest way to do this is to just place it in the same directory as your recipe, in which case the module name is the same as the file – in this case, `custom_transform_example`. + +
+ Advanced: Installing as a package and enable discoverability +Alternatively, create a `setup.py` in the same directory as our transform script to make it visible globally. After installing this package (e.g. with `python setup.py` or `pip install -e .`), our module will be installed and importable as `custom_transform_example`. + +```python +from setuptools import find_packages, setup + +setup( + name="custom_transform_example", + version="1.0", + packages=find_packages(), + # if you don't already have DataHub installed, add it under install_requires + # install_requires=["acryl-datahub"], + entry_points={ + "datahub.ingestion.transformer.plugins": [ + "custom_transform_example_alias = custom_transform_example:AddCustomOwnership", + ], + }, +) +``` + +Additionally, declare the transformer under the `entry_points` variable of the setup script. This enables the transformer to be +listed when running `datahub check plugins`, and sets up the transformer's shortened alias for use in recipes. + +
+ +### Running the transform + +```yaml +transformers: + - type: "custom_transform_example_alias" + config: + owners_json: "" # the JSON file mentioned at the start +``` + +After running `datahub ingest -c `, our MCEs will now have the following owners appended: + +```json +"owners": [ + { + "owner": "urn:li:corpuser:athos", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:porthos", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpuser:aramis", + "type": "DATAOWNER", + "source": null + }, + { + "owner": "urn:li:corpGroup:the_three_musketeers", + "type": "DATAOWNER", + "source": null + }, + // ...and any additional owners +], +``` + +### Using this in the remote executor (DataHub Cloud only) + +Build the image with your transformer + +``` +docker build -t acryldata:customtransform1 -f metadata-ingestion/examples/transforms/example.Dockerfile metadata-ingestion/examples/transforms +``` + +Test it works + +``` +docker run -it --rm acryldata:customtransform1 bash +``` + +Inside the docker container + +``` +source venv/bin/activate +datahub ingest -c ./custom_transformer/recipe.dhub.yaml +``` + +If you use this image for remote executor then you can set `file:///datahub-executor/custom_transformer` as an extra pip dependency to install the transformer in your ingestion. + +All the files for this tutorial may be found [here](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/transforms/). diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/intro.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/intro.md new file mode 100644 index 00000000..90ab8f28 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/intro.md @@ -0,0 +1,54 @@ +--- +title: Introduction +sidebar_label: Introduction +slug: /metadata-ingestion/docs/transformer/intro +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/transformer/intro.md +--- + +# Transformers + +## What’s a transformer? + +Oftentimes we want to modify metadata before it reaches the ingestion sink – for instance, we might want to add custom tags, ownership, properties, or patch some fields. A transformer allows us to do exactly these things. + +Moreover, a transformer allows one to have fine-grained control over the metadata that’s ingested without having to modify the ingestion framework's code yourself. Instead, you can write your own module that can transform metadata events however you like. To include a transformer into a recipe, all that's needed is the name of the transformer as well as any configuration that the transformer needs. + +:::note + +Providing urns for metadata that does not already exist will result in unexpected behavior. Ensure any tags, terms, domains, etc. urns that you want to apply in your transformer already exist in your DataHub instance. + +For example, adding a domain urn in your transformer to apply to datasets will not create the domain entity if it doesn't exist. Therefore, you can't add documentation to it and it won't show up in Advanced Search. This goes for any metadata you are applying in transformers. + +::: + +## Provided transformers + +Aside from the option of writing your own transformer (see below), we provide some simple transformers for the use cases of adding: tags, glossary terms, properties and ownership information. + +DataHub provided transformers for dataset are: + +- [Simple Add Dataset ownership](./dataset_transformer.md#simple-add-dataset-ownership) +- [Pattern Add Dataset ownership](./dataset_transformer.md#pattern-add-dataset-ownership) +- [Simple Remove Dataset ownership](./dataset_transformer.md#simple-remove-dataset-ownership) +- [Extract Ownership from Tags](./dataset_transformer.md#extract-ownership-from-tags) +- [Clean suffix prefix from Ownership](./dataset_transformer.md#clean-suffix-prefix-from-ownership) +- [Mark Dataset Status](./dataset_transformer.md#mark-dataset-status) +- [Simple Add Dataset globalTags](./dataset_transformer.md#simple-add-dataset-globaltags) +- [Pattern Add Dataset globalTags](./dataset_transformer.md#pattern-add-dataset-globaltags) +- [Add Dataset globalTags](./dataset_transformer.md#add-dataset-globaltags) +- [Set Dataset browsePath](./dataset_transformer.md#set-dataset-browsepath) +- [Simple Add Dataset glossaryTerms](./dataset_transformer.md#simple-add-dataset-glossaryterms) +- [Pattern Add Dataset glossaryTerms](./dataset_transformer.md#pattern-add-dataset-glossaryterms) +- [Add Dataset globalTags](./dataset_transformer.md#add-dataset-globaltags) +- [Pattern Add Dataset Schema Field glossaryTerms](./dataset_transformer.md#pattern-add-dataset-schema-field-glossaryterms) +- [Pattern Add Dataset Schema Field globalTags](./dataset_transformer.md#pattern-add-dataset-schema-field-globaltags) +- [Simple Add Dataset datasetProperties](./dataset_transformer.md#simple-add-dataset-datasetproperties) +- [Add Dataset datasetProperties](./dataset_transformer.md#add-dataset-datasetproperties) +- [Simple Add Dataset domains](./dataset_transformer.md#simple-add-dataset-domains) +- [Pattern Add Dataset domains](./dataset_transformer.md#pattern-add-dataset-domains) +- [Domain Mapping Based on Tags](./dataset_transformer.md#domain-mapping-based-on-tags) +- [Simple Add Dataset dataProduct ](./dataset_transformer.md#simple-add-dataset-dataproduct) +- [Pattern Add Dataset dataProduct](./dataset_transformer.md#pattern-add-dataset-dataproduct) +- [Add Dataset dataProduct](./dataset_transformer.md#add-dataset-dataproduct) +- [Set browsePaths](./universal_transformers.md#set-browsepaths) diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/universal_transformers.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/universal_transformers.md new file mode 100644 index 00000000..96773fb2 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/docs/transformer/universal_transformers.md @@ -0,0 +1,114 @@ +--- +title: Universal transformers +sidebar_label: Universal transformers +slug: /metadata-ingestion/docs/transformer/universal_transformers +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/docs/transformer/universal_transformers.md +--- + +# Transformers + +The below table shows transformer which can transform aspects of any entity having them. + +| Aspect | Transformer | +| --------------- | ------------------------------------- | +| `browsePathsV2` | - [Set browsePaths](#set-browsepaths) | + +## Set browsePaths + +This transformer operates on `browsePathsV2` aspect. If it is not emitted by the ingestion source, it will be created +by the transformer. By default it will prepend configured path to the original path (so it will add it as a prefix). + +### Config Details + +| Field | Required | Type | Default | Description | +| ------------------ | -------- | ------------ | ------- | --------------------------------------------------------------------------------------------------- | +| `path` | ✅ | list[string] | | List of nodes in the new path. | +| `replace_existing` | | boolean | `false` | Whether to overwrite existing browse path, if set to `false`, the configured path will be prepended | + +In the most basic case `path` contains list of static strings, for example, below config: + +```yaml +transformers: + - type: "set_browse_path" + config: + path: + - abc + - def +``` + +will be reflected as every entity having path prefixed by `abc` and `def` nodes (`def` will be contained by `abc`). + +### Variable substitution + +The transformer has a mechanism of variables substitution in the path, where list of variables are build based on +existing `browsePathsV2` aspect of the entity. Every _node_ in the existing path, as long as it contains reference to +another entity (e.g. a `container` or a `dataPlatformInstance`) is stored in the list of variables to use. Since +we can have multiple references to entities of the same type (e.g. `containers`) in the browse path, they are stored +in a list-like object, with original order being respected. Let's consider an example, real-world situation, of a table +ingested from Snowflake source, and having `platform_instance` set to some value. Such table will have `browsePathsV2` +aspect set to contain below references: + +```yaml +- urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,my_platform_instance) +- urn:li:container:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +- urn:li:container:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +``` + +where `urn:li:container:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa` identifies a `container` reflecting a Snowflake's _database_ and +`urn:li:container:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb` identifies a `container` reflecting a Snowflake's _schema_. +Such, existing, path will be mapped into variables as shown below: + +```python +dataPlatformInstance[0] = "urn:li:dataPlatformInstance:(urn:li:dataPlatform:snowflake,my_platform_instance)" +container[0] = "urn:li:container:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +container[1] = "urn:li:container:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +``` + +Those variables can be refered to, from the config, by using `$` character, like below: + +```yaml +transformers: + - type: "set_browse_path" + config: + path: + - $dataPlatformInstance[0] + - $container[0] + - $container[1] +``` + +Additionally, 2 more rules apply to the variables resolution: + +- If a variable does not exist (or if the index reached outside of list's length) - it will be ignored and not used in the path, all the other nodes will be used and path will be modified +- `$variable[*]` will expand entire list of variables to multiple _nodes_ in the path (think about it as a "flat map"), for example, the equivalent of above config, would be: + ```yaml + transformers: + - type: "set_browse_path" + config: + path: + - $dataPlatformInstance[0] + - $container[*] + ``` + +### Examples + +Add (prefix) a top-level node "datahub" to paths emitted by the source: + +```yaml +transformers: + - type: "set_browse_path" + config: + path: + - datahub +``` + +Remove data platform instance from the path (if it was set), while retaining containers structure: + +```yaml +transformers: + - type: "set_browse_path" + config: + replace_existing: true + path: + - $container[*] +``` diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/library/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/library/README.md new file mode 100644 index 00000000..a020881a --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/library/README.md @@ -0,0 +1,220 @@ +--- +title: DataHub Library Examples +sidebar_label: Library Examples +slug: /metadata-ingestion/examples/library +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/library/README.md +--- +# DataHub Library Examples + +This directory contains examples demonstrating how to use the DataHub Python SDK and metadata emission APIs. + +## Structure + +Each example is a standalone Python script that demonstrates a specific use case: + +- **Create examples**: Show how to create new metadata entities +- **Update examples**: Show how to modify existing metadata +- **Query examples**: Show how to read and query metadata +- **Delete examples**: Show how to remove metadata + +## Writing Testable Examples + +To ensure examples are maintainable and correct, follow this pattern when writing new examples: + +### Pattern Overview + +Examples should have two main components: + +1. **Testable functions**: Pure functions that take dependencies as parameters and return values/metadata +2. **Main function**: Entry point that creates dependencies and calls the testable functions + +### Example Structure + +```python +from typing import Optional +from datahub.emitter.mcp import MetadataChangeProposalWrapper +from datahub.emitter.rest_emitter import DatahubRestEmitter + + +def create_entity_metadata(...) -> MetadataChangeProposalWrapper: + """ + Create metadata for an entity. + + This function is pure and testable - it doesn't have side effects. + + Args: + ... (all required parameters) + + Returns: + MetadataChangeProposalWrapper containing the metadata + """ + # Build and return the MCP + return MetadataChangeProposalWrapper(...) + + +def main(emitter: Optional[DatahubRestEmitter] = None) -> None: + """ + Main function demonstrating the example use case. + + Args: + emitter: Optional emitter for testing. If not provided, creates a new one. + """ + emitter = emitter or DatahubRestEmitter(gms_server="http://localhost:8080") + + # Use the testable function + mcp = create_entity_metadata(...) + + # Emit the metadata + emitter.emit(mcp) + print(f"Successfully created entity") + + +if __name__ == "__main__": + main() +``` + +### For SDK-based Examples + +When using the DataHub SDK (`DataHubClient`): + +```python +from typing import Optional +from datahub.sdk import DataHubClient + + +def perform_operation(client: DataHubClient, ...) -> ...: + """ + Perform an operation using the DataHub client. + + Args: + client: DataHub client to use + ...: Other parameters + + Returns: + Result of the operation + """ + # Perform the operation + return result + + +def main(client: Optional[DataHubClient] = None) -> None: + """ + Main function demonstrating the example use case. + + Args: + client: Optional client for testing. If not provided, creates one from env. + """ + client = client or DataHubClient.from_env() + + result = perform_operation(client, ...) + print(f"Operation result: {result}") + + +if __name__ == "__main__": + main() +``` + +### Benefits of This Pattern + +1. **Testability**: Core logic can be unit tested without needing a running DataHub instance +2. **Reusability**: The testable functions can be imported and used in other code +3. **Clarity**: Separates business logic from infrastructure setup +4. **Flexibility**: Examples can still be run standalone while being testable + +### Running Examples + +**As standalone scripts:** + +```bash +python examples/library/notebook_create.py +``` + +**In tests:** + +```python +from examples.library.create_notebook import create_notebook_metadata + +# Unit test +mcp = create_notebook_metadata(...) +assert mcp.entityUrn == "..." + +# Integration test +from examples.library.create_notebook import main +main(emitter=test_emitter) # Inject test emitter +``` + +## Testing + +Examples are tested at two levels: + +### Unit Tests + +Located in `tests/unit/test_library_examples.py`: + +- Test that examples compile and imports resolve +- Test that core functions produce valid metadata structures +- Use mocking to avoid needing a real DataHub instance +- Fast and run on every commit + +### Integration Tests + +Located in `tests/integration/library_examples/`: + +- Test examples against a real DataHub instance +- Verify end-to-end functionality including reads after writes +- Test that metadata is correctly persisted and retrievable +- Slower, may run less frequently + +### Running Tests + +```bash +# Run all example tests (unit only) +pytest tests/unit/test_library_examples.py + +# Run specific unit tests +pytest tests/unit/test_library_examples.py::test_create_notebook_metadata + +# Run integration tests (requires running DataHub) +pytest tests/integration/library_examples/ -m integration + +# Run all tests +pytest tests/unit/test_library_examples.py tests/integration/library_examples/ +``` + +## Guidelines + +1. **Keep examples simple**: Focus on demonstrating one concept clearly +2. **Use realistic data**: URNs, names, and values should look like real-world usage +3. **Add comments**: Explain non-obvious choices or important details +4. **Follow the pattern**: Use the testable function + main() pattern +5. **Document parameters**: Use clear docstrings with type hints +6. **Handle errors gracefully**: Show proper error handling where relevant +7. **Test your examples**: Add unit tests for new examples + +## Example Categories + +### Entity Creation + +- `notebook_create.py` - Create a notebook entity +- `data_platform_create.py` - Create a custom data platform +- `glossary_term_create.py` - Create glossary terms + +### Metadata Updates + +- `dataset_add_term.py` - Add glossary terms to datasets +- `dataset_add_owner.py` - Add ownership information +- `notebook_add_tags.py` - Add tags to notebooks + +### Querying Metadata + +- `dataset_query_deprecation.py` - Check if a dataset is deprecated +- `search_with_query.py` - Search for entities +- `lineage_column_get.py` - Query column-level lineage + +## Getting Help + +- [DataHub Documentation](https://datahubproject.io/docs/) +- [Python SDK Reference](https://datahubproject.io/docs/python-sdk/) +- [Metadata Model](https://datahubproject.io/docs/metadata-model/) +- [GitHub Issues](https://github.com/datahub-project/datahub/issues) diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/structured_properties/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/structured_properties/README.md new file mode 100644 index 00000000..759556c3 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/structured_properties/README.md @@ -0,0 +1,57 @@ +--- +title: Extended Properties +slug: /metadata-ingestion/examples/structured_properties +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/structured_properties/README.md +--- +# Extended Properties + +## Expected Capabilities + +### structured_properties command + +```yaml +- id: io.acryl.privacy.retentionTime + # urn: urn:li:structuredProperty:<> + # fullyQualifiedName: io.acryl.privacy.retentionTime + type: STRING + cardinality: MULTIPLE + entityTypes: + - dataset # or urn:li:logicalEntity:metamodel.datahub.dataset + - dataflow + description: "Retention Time is used to figure out how long to retain records in a dataset" + allowedValues: + - value: 30 days + description: 30 days, usually reserved for datasets that are ephemeral and contain pii + - value: 3 months + description: Use this for datasets that drive monthly reporting but contain pii + - value: 2 yrs + description: Use this for non-sensitive data that can be retained for longer +- id: io.acryl.dataManagement.replicationSLA + type: NUMBER + description: "SLA for how long data can be delayed before replicating to the destination cluster" + entityTypes: + - dataset +- id: io.acryl.dataManagement.deprecationDate + type: DATE + entityTypes: + - dataset + - dataFlow + - dataJob +``` + +``` +datahub properties create -f structured_properties.yaml +``` + +``` +datahub properties create --name io.acryl.privacy.retentionTime --type STRING --cardinality MULTIPLE --entity_type DATASET --entity_type DATAFLOW +``` + +### dataset command + +``` +datahub dataset create -f dataset.yaml +``` + +See example in `dataproduct`. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/transforms/README.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/transforms/README.md new file mode 100644 index 00000000..39af4414 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/examples/transforms/README.md @@ -0,0 +1,11 @@ +--- +title: Custom transformer script +slug: /metadata-ingestion/examples/transforms +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/transforms/README.md +--- +# Custom transformer script + +This script sets up a transformer that reads in a list of owner URNs from a JSON file specified via `owners_json` and appends these owners to every MCE. + +See the transformers tutorial (https://docs.datahub.com/docs/metadata-ingestion/docs/transformer/intro) for how this module is built and run. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/integration_docs/great-expectations.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/integration_docs/great-expectations.md new file mode 100644 index 00000000..67455efb --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/integration_docs/great-expectations.md @@ -0,0 +1,72 @@ +--- +title: Great Expectations +slug: /metadata-ingestion/integration_docs/great-expectations +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/integration_docs/great-expectations.md +--- +# Great Expectations + +This guide helps to setup and configure `DataHubValidationAction` in Great Expectations to send assertions(expectations) and their results to DataHub using DataHub's Python Rest emitter. + +## Capabilities + +`DataHubValidationAction` pushes assertions metadata to DataHub. This includes + +- **Assertion Details**: Details of assertions (i.e. expectation) set on a Dataset (Table). +- **Assertion Results**: Evaluation results for an assertion tracked over time. + +This integration supports v3 api datasources using SqlAlchemyExecutionEngine and SparkDFExecutionEngine. + +For SparkDFExecutionEngine, DataHubValidationAction would map the **Data Asset** of GX to dataSet's entity name when constructing datasets URN. + +## Limitations + +This integration does not support + +- v2 Datasources such as SqlAlchemyDataset +- v3 Datasources using execution engine other than SqlAlchemyExecutionEngine,SparkDFExecutionEngine (Pandas) +- Cross-dataset expectations (those involving > 1 table) + +## Compatibility + +- DataHubValidationAction with SparkDFExecutionEngine has only been tested with **Great Expectation >= 0.18.0, <1.0.0**. Other versions may not be compatible + +## Setting up + +1. Install the required dependency in your Great Expectations environment. + + ```shell + pip install 'acryl-datahub-gx-plugin' + ``` + +2. To add `DataHubValidationAction` in Great Expectations Checkpoint, add following configuration in action_list for your Great Expectations `Checkpoint`. For more details on setting action_list, see [Checkpoints and Actions](https://docs.greatexpectations.io/docs/reference/checkpoints_and_actions/) + ```yml + action_list: + - name: datahub_action + action: + module_name: datahub_gx_plugin.action + class_name: DataHubValidationAction + server_url: http://localhost:8080 #datahub server url + ``` + **Configuration options:** + - `server_url` (required): URL of DataHub GMS endpoint + - `env` (optional, defaults to "PROD"): Environment to use in namespace when constructing dataset URNs. + - `exclude_dbname` (optional): Exclude dbname / catalog when constructing dataset URNs. (Highly applicable to Trino / Presto where we want to omit catalog e.g. `hive`) + - `platform_alias` (optional): Platform alias when constructing dataset URNs. e.g. main data platform is `presto-on-hive` but using `trino` to run the test + - `platform_instance_map` (optional): Platform instance mapping to use when constructing dataset URNs. Maps the GX 'data source' name to a platform instance on DataHub. e.g. `platform_instance_map: { "datasource_name": "warehouse" }` + - `graceful_exceptions` (defaults to true): If set to true, most runtime errors in the lineage backend will be suppressed and will not cause the overall checkpoint to fail. Note that configuration issues will still throw exceptions. + - `token` (optional): Bearer token used for authentication. + - `timeout_sec` (optional): Per-HTTP request timeout. + - `retry_status_codes` (optional): Retry HTTP request also on these status codes. + - `retry_max_times` (optional): Maximum times to retry if HTTP request fails. The delay between retries is increased exponentially. + - `extra_headers` (optional): Extra headers which will be added to the datahub request. + - `parse_table_names_from_sql` (defaults to false): The integration can use an SQL parser to try to parse the datasets being asserted. This parsing is disabled by default, but can be enabled by setting `parse_table_names_from_sql: True`. The parser is based on the [`sqllineage`](https://pypi.org/project/sqllineage/) package. + - `convert_urns_to_lowercase` (optional): Whether to convert dataset urns to lowercase. + +## Debugging + +Set environment variable `DATAHUB_DEBUG` (default `false`) to `true` to enable debug logging for `DataHubValidationAction`. + +## Learn more + +To see the Great Expectations in action, check out [this demo](https://www.loom.com/share/d781c9f0b270477fb5d6b0c26ef7f22d) from the Feb 2022 townhall. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/recipe_overview.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/recipe_overview.md new file mode 100644 index 00000000..5e0b32c6 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/recipe_overview.md @@ -0,0 +1,159 @@ +--- +title: Recipes +slug: /metadata-ingestion/recipe_overview +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/recipe_overview.md +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Recipes + +A recipe is the main configuration file for metadata ingestion. It tells our ingestion scripts where to pull data from (source) and where to put it (sink). + +

+ +

+ +## Configuring Recipe + +The basic form of the recipe file consists of: + +- `source`, which contains the configuration of the data source. (See [Sources](source_overview.md)) +- `sink`, which defines the destination of the metadata (See [Sinks](sink_overview.md)) + +Here's a simple recipe that pulls metadata from MSSQL (source) and puts it into the default sink (datahub rest). + +```yaml +# The simplest recipe that pulls metadata from MSSQL and puts it into DataHub +# using the Rest API. +source: + type: mssql + config: + username: sa + password: ${MSSQL_PASSWORD} + database: DemoData +# sink section omitted as we want to use the default datahub-rest sink +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +A number of recipes are included in the [examples/recipes](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/examples/recipes) directory. For full info and context on each source and sink, see the pages described in the [table of plugins](../docs/cli.md#installing-plugins). + +:::note One Source/Sink for One Recipe! +Note that one recipe file can only have 1 source and 1 sink. If you want multiple sources then you will need multiple recipe files. +::: + +## Running a Recipe + +DataHub supports running recipes via the CLI or UI. + + + + +Install CLI and the plugin for the ingestion. + +```shell +python3 -m pip install --upgrade acryl-datahub +pip install 'acryl-datahub[datahub-rest]' +``` + +Running this recipe is as simple as: + +```shell +datahub ingest -c recipe.dhub.yaml +``` + +For a detailed guide on running recipes via CLI, please refer to [CLI Ingestion Guide](cli-ingestion.md). + + + + + +You can configure and run the recipe in **Ingestion** tab in DataHub. + +

+ +

+ +- Make sure you have the **Manage Metadata Ingestion & Manage Secret** privileges. +- Navigate to **Ingestion** tab in DataHub. +- Create an ingestion source & configure the recipe via UI. +- Hit **Execute**. + +For a detailed guide on running recipes via UI, please refer to [UI Ingestion Guide](../docs/ui-ingestion.md). + + + + +## Advanced Configuration + +### Handling Sensitive Information in Recipes + +We automatically expand environment variables in the config (e.g. `${MSSQL_PASSWORD}`), +similar to variable substitution in GNU bash or in docker-compose files. +For details, see [variable-substitution](https://docs.docker.com/compose/compose-file/compose-file-v2/#variable-substitution). +This environment variable substitution should be used to mask sensitive information in recipe files. As long as you can get env variables securely to the ingestion process there would not be any need to store sensitive information in recipes. + +### Loading Sensitive Data as Files in Recipes + +Some sources (e.g. kafka, bigquery, mysql) require paths to files on a local file system. This doesn't work for UI ingestion, where the recipe needs to be totally self-sufficient. To add files to ingestion processes as part of the necessary configuration, DataHub offers a directive `__DATAHUB_TO_FILE_` which allows recipes to set the contents of files. + +The syntax for this directive is: `__DATAHUB_TO_FILE_: ` which will get turned into `: `. Note that value can be specified inline or using an env var/secret. + +I.e: + +```yaml +source: + type: mysql + config: + # Coordinates + host_port: localhost:3306 + database: dbname + + # Credentials + username: root + password: example + # If you need to use SSL with MySQL: + options: + connect_args: + __DATAHUB_TO_FILE_ssl_key: '${secret}' # use this for secrets that you need to mount to a file + # this will get converted into + # ssl_key: /tmp/path/to/file # where file contains the contents of ${secret} + ... +``` + +### Transformations + +If you'd like to modify data before it reaches the ingestion sinks – for instance, adding additional owners or tags – you can use a transformer to write your own module and integrate it with DataHub. Transformers require extending the recipe with a new section to describe the transformers that you want to run. + +For example, a pipeline that ingests metadata from MSSQL and applies a default "important" tag to all datasets is described below: + +```yaml +# A recipe to ingest metadata from MSSQL and apply default tags to all tables +source: + type: mssql + config: + username: sa + password: ${MSSQL_PASSWORD} + database: DemoData + +transformers: # an array of transformers applied sequentially + - type: simple_add_dataset_tags + config: + tag_urns: + - "urn:li:tag:Important" +# default sink, no config needed +``` + +Check out the [transformers guide](./docs/transformer/intro.md) to learn more about how you can create really flexible pipelines for processing metadata using Transformers! + +### Autocomplete and Syntax Validation + +Name your recipe with **.dhub.yaml** extension like `myrecipe.dhub.yaml_` to use vscode or intellij as a recipe editor with autocomplete +and syntax validation. Make sure yaml plugin is installed for your editor: + +- For vscode install [Redhat's yaml plugin](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml) +- For intellij install [official yaml plugin](https://plugins.jetbrains.com/plugin/13126-yaml) diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/airflow.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/airflow.md new file mode 100644 index 00000000..e3d393cd --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/airflow.md @@ -0,0 +1,48 @@ +--- +title: Using Airflow +slug: /metadata-ingestion/schedule_docs/airflow +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/schedule_docs/airflow.md +--- +# Using Airflow + +If you are using Apache Airflow for your scheduling then you might want to also use it for scheduling your ingestion recipes. For any Airflow specific questions you can go through [Airflow docs](https://airflow.apache.org/docs/apache-airflow/stable/) for more details. + +We've provided a few examples of how to configure your DAG: + +- [`mysql_sample_dag`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/mysql_sample_dag.py) embeds the full MySQL ingestion configuration inside the DAG. + +- [`snowflake_sample_dag`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/snowflake_sample_dag.py) avoids embedding credentials inside the recipe, and instead fetches them from Airflow's [Connections](https://airflow.apache.org/docs/apache-airflow/stable/howto/connection/index.html) feature. You must configure your connections in Airflow to use this approach. + +:::tip + +These example DAGs use the `PythonVirtualenvOperator` to run the ingestion. This is the recommended approach, since it guarantees that there will not be any conflicts between DataHub and the rest of your Airflow environment. + +When configuring the task, it's important to specify the requirements with your source and set the `system_site_packages` option to false. + +```py +ingestion_task = PythonVirtualenvOperator( + task_id="ingestion_task", + requirements=[ + "acryl-datahub[]", + ], + system_site_packages=False, + python_callable=your_callable, +) +``` + +::: + +
+Advanced: loading a recipe file + +In more advanced cases, you might want to store your ingestion recipe in a file and load it from your task. + +- Ensure the recipe file is in a folder accessible to your airflow workers. You can either specify absolute path on the machines where Airflow is installed or a path relative to `AIRFLOW_HOME`. +- Ensure [DataHub CLI](../../docs/cli.md) is installed in your airflow environment. +- Create a DAG task to read your DataHub ingestion recipe file and run it. See the example below for reference. +- Deploy the DAG file into airflow for scheduling. Typically this involves checking in the DAG file into your dags folder which is accessible to your Airflow instance. + +Example: [`generic_recipe_sample_dag`](https://github.com/datahub-project/datahub/blob/master/metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/example_dags/generic_recipe_sample_dag.py) + +
diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/cron.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/cron.md new file mode 100644 index 00000000..2c0d49b1 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/cron.md @@ -0,0 +1,35 @@ +--- +title: Using Cron +slug: /metadata-ingestion/schedule_docs/cron +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/schedule_docs/cron.md +--- +# Using Cron + +Assume you have a recipe file `/home/ubuntu/datahub_ingest/mysql_to_datahub.yml` on your machine + +``` +source: + type: mysql + config: + # Coordinates + host_port: localhost:3306 + database: dbname + + # Credentials + username: root + password: example + +sink: + type: datahub-rest + config: + server: http://localhost:8080 +``` + +We can use crontab to schedule ingestion to run five minutes after midnight, every day using [DataHub CLI](../../docs/cli.md). + +``` +5 0 * * * datahub ingest -c /home/ubuntu/datahub_ingest/mysql_to_datahub.yml +``` + +Read through [crontab docs](https://man7.org/linux/man-pages/man5/crontab.5.html) for more options related to scheduling. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/datahub.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/datahub.md new file mode 100644 index 00000000..0125f39f --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/datahub.md @@ -0,0 +1,9 @@ +--- +title: Using DataHub +slug: /metadata-ingestion/schedule_docs/datahub +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/schedule_docs/datahub.md +--- +# Using DataHub + +[UI Ingestion](../../docs/ui-ingestion.md) can be used to schedule metadata ingestion through DataHub. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/intro.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/intro.md new file mode 100644 index 00000000..50968a3d --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/intro.md @@ -0,0 +1,37 @@ +--- +title: Introduction to Scheduling Metadata Ingestion +slug: /metadata-ingestion/schedule_docs/intro +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/schedule_docs/intro.md +--- +# Introduction to Scheduling Metadata Ingestion + +Given a recipe file `/home/ubuntu/datahub_ingest/mysql_to_datahub.yml`. + +``` +source: + type: mysql + config: + # Coordinates + host_port: localhost:3306 + database: dbname + + # Credentials + username: root + password: example + +sink: + type: datahub-rest + config: + server: http://localhost:8080 +``` + +We can do ingestion of our metadata using [DataHub CLI](../../docs/cli.md) as follows + +``` +datahub ingest -c /home/ubuntu/datahub_ingest/mysql_to_datahub.yml +``` + +This will ingest metadata from the `mysql` source which is configured in the recipe file. This does ingestion once. As the source system changes we would like to have the changes reflected in DataHub. To do this someone will need to re-run the ingestion command using a recipe file. + +An alternate to running the command manually we can schedule the ingestion to run on a regular basis. In this section we give some examples of how scheduling ingestion of metadata into DataHub can be done. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/kubernetes.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/kubernetes.md new file mode 100644 index 00000000..0ad81429 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/schedule_docs/kubernetes.md @@ -0,0 +1,54 @@ +--- +title: Using Kubernetes +slug: /metadata-ingestion/schedule_docs/kubernetes +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/schedule_docs/kubernetes.md +--- +# Using Kubernetes + +If you have deployed DataHub using our official [helm charts](https://github.com/acryldata/datahub-helm) you can use the +datahub ingestion cron subchart to schedule ingestions. + +Here is an example of what that configuration would look like in your **values.yaml**: + +```yaml +datahub-ingestion-cron: + enabled: true + crons: + mysql: + schedule: "0 * * * *" # Every hour + recipe: + configmapName: recipe-config + fileName: mysql_recipe.yml +``` + +This assumes the pre-existence of a Kubernetes ConfigMap which holds all recipes being scheduled in the same namespace as +where the cron jobs will be running. + +An example could be: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: recipe-config +data: + mysql_recipe.yml: |- + source: + type: mysql + config: + # Coordinates + host_port: :3306 + database: dbname + + # Credentials + username: root + password: example + + sink: + type: datahub-rest + config: + server: http://:8080 +``` + +For more information, please see the [documentation](https://github.com/acryldata/datahub-helm/tree/master/charts/datahub/subcharts/datahub-ingestion-cron) of this sub-chart. diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/console.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/console.md new file mode 100644 index 00000000..9f5ec337 --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/console.md @@ -0,0 +1,35 @@ +--- +title: Console +slug: /metadata-ingestion/sink_docs/console +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/sink_docs/console.md +--- +# Console + +For context on getting started with ingestion, check out our [metadata ingestion guide](../README.md). + +## Setup + +Works with `acryl-datahub` out of the box. + +## Capabilities + +Simply prints each metadata event to stdout. Useful for experimentation and debugging purposes. + +## Quickstart recipe + +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + +For general pointers on writing and running a recipe, see our [main recipe guide](../README.md#recipes). + +```yml +source: + # source configs + +sink: + type: "console" +``` + +## Config details + +None! diff --git a/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/datahub.md b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/datahub.md new file mode 100644 index 00000000..908b35ec --- /dev/null +++ b/docs-archive/versioned_docs/version-1.5.0/metadata-ingestion/sink_docs/datahub.md @@ -0,0 +1,229 @@ +--- +title: DataHub +slug: /metadata-ingestion/sink_docs/datahub +custom_edit_url: >- + https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/sink_docs/datahub.md +--- +# DataHub + +## DataHub Rest + +For context on getting started with ingestion, check out our [metadata ingestion guide](../README.md). + +### Setup + +To install this plugin, run `pip install 'acryl-datahub[datahub-rest]'`. + +### Capabilities + +Pushes metadata to DataHub using the GMS REST API. The advantage of the REST-based interface +is that any errors can immediately be reported. + +### Quickstart recipe + +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + +For general pointers on writing and running a recipe, see our [main recipe guide](../README.md#recipes). This should point to the GMS server. + +```yml +source: + # source configs +sink: + type: "datahub-rest" + config: + server: "http://localhost:8080" +``` + +If you are connecting to a hosted DataHub Cloud instance, your sink will look like + +```yml +source: + # source configs +sink: + type: "datahub-rest" + config: + server: "https://.acryl.io/gms" + token: +``` + +If you are running the ingestion in a container in docker and your [GMS is also running in docker](../../docker/README.md) then you should use the internal docker hostname of the GMS pod. Usually it would look something like + +```yml +source: + # source configs +sink: + type: "datahub-rest" + config: + server: "http://datahub-gms:8080" +``` + +If GMS is running in a kubernetes pod [deployed through the helm charts](../../docs/deploy/kubernetes.md) and you are trying to connect to it from within the kubernetes cluster then you should use the Kubernetes service name of GMS. Usually it would look something like + +```yml +source: + # source configs +sink: + type: "datahub-rest" + config: + server: "http://datahub-datahub-gms.datahub.svc.cluster.local:8080" +``` + +If you are using [UI based ingestion](../../docs/ui-ingestion.md) then where GMS is deployed decides what hostname you should use. + +### Config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +| -------------------------- | -------- | -------------------- | -------------------------------------------------------------------------------------------------- | +| `server` | ✅ | | URL of DataHub GMS endpoint. | +| `token` | | | Bearer token used for authentication. | +| `timeout_sec` | | 30 | Per-HTTP request timeout. | +| `retry_max_times` | | 1 | Maximum times to retry if HTTP request fails. The delay between retries is increased exponentially | +| `retry_status_codes` | | [429, 502, 503, 504] | Retry HTTP request also on these status codes | +| `extra_headers` | | | Extra headers which will be added to the request. | +| `max_threads` | | `15` | Max parallelism for REST API calls | +| `mode` | | `ASYNC_BATCH` | [Advanced] Mode of operation - `SYNC`, `ASYNC`, or `ASYNC_BATCH` | +| `ca_certificate_path` | | | Path to server's CA certificate for verification of HTTPS communications | +| `client_certificate_path` | | | Path to client's CA certificate for HTTPS communications | +| `disable_ssl_verification` | | false | Disable ssl certificate validation | + +## DataHub Kafka + +For context on getting started with ingestion, check out our [metadata ingestion guide](../README.md). + +#### Note: Not Supported in DataHub Cloud + +The `datahub-kafka` sink requires direct network access to the Kafka cluster, which is only available in self-hosted DataHub environments (like OSS). For DataHub Cloud, use the `datahub-rest` sink instead. + +### Setup + +To install this plugin, run `pip install 'acryl-datahub[datahub-kafka]'`. + +### Capabilities + +Pushes metadata to DataHub by publishing messages to Kafka. The advantage of the Kafka-based +interface is that it's asynchronous and can handle higher throughput. + +### Quickstart recipe + +Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options. + +For general pointers on writing and running a recipe, see our [main recipe guide](../README.md#recipes). + +```yml +source: + # source configs + +sink: + type: "datahub-kafka" + config: + connection: + bootstrap: "localhost:9092" + schema_registry_url: "http://localhost:8081" +``` + +#### Kafka with OAuth + +E.g. for AWS MSK clusters using IAM authentication: + +```yml +source: + # source configs +sink: + type: "datahub-kafka" + config: + topic_routes: + MetadataChangeEvent: "custom_mce_topic_name" # Optional override + MetadataChangeProposal: "custom_mcp_topic_name" # Optional override + connection: + bootstrap: "b-1.your-msk-cluster.region.amazonaws.com:9098" + schema_registry_url: "http://datahub-gms:8080/schema-registry/api/" + producer_config: + security.protocol: "SASL_SSL" + sasl.mechanism: "OAUTHBEARER" + sasl.oauthbearer.method: "default" + oauth_cb: "datahub_actions.utils.kafka_msk_iam:oauth_cb" # OAuth callback +``` + +**Note:** MSK IAM authentication requires `pip install 'acryl-datahub-actions>=1.3.1.2'` for the OAuth callback. + +### Config details + +Note that a `.` is used to denote nested fields in the YAML recipe. + +| Field | Required | Default | Description | +| -------------------------------------------- | -------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `connection.bootstrap` | ✅ | | Kafka bootstrap URL. | +| `connection.producer_config.