diff --git a/.github/workflows/ci-infra.yml b/.github/workflows/ci-infra.yml index 2c8d1cc..fc524f9 100644 --- a/.github/workflows/ci-infra.yml +++ b/.github/workflows/ci-infra.yml @@ -48,5 +48,6 @@ jobs: TF_VAR_environment: ${{ env.ENV }} TF_VAR_project_id: ${{ secrets.GCP_PROJECT_ID }} TF_VAR_region: ${{ env.REGION }} + TF_VAR_github_repo: ${{ env.GITHUB_REPO }} TF_VAR_alert_email_map: ${{ secrets.ALERT_EMAIL_MAP }} run: terraform apply -auto-approve \ No newline at end of file diff --git a/data_extract/shared/extract_logic.py b/data_extract/shared/extract_logic.py index b54e409..6fed4bc 100644 --- a/data_extract/shared/extract_logic.py +++ b/data_extract/shared/extract_logic.py @@ -77,7 +77,7 @@ def get_valid_files( - Returns an empty list if the folder is empty or contains only reserved files. """ - # Validate instruction inside the folder + # Validate instruction.txt inside the folder if not check_handshake(drive_api, folder_id): print(f"[ERROR]: '{target_folder}' missing instruction.txt or upload not safe.") return None diff --git a/data_pipeline/semantic/semantic_executor.py b/data_pipeline/semantic/semantic_executor.py index 2eafd68..8da155c 100644 --- a/data_pipeline/semantic/semantic_executor.py +++ b/data_pipeline/semantic/semantic_executor.py @@ -90,7 +90,10 @@ def orchestrate_module( module_report = report["modules"] table_trackers = { - table_name: {"build_stage": False, "validate_stage": False} + table_name: { + "build_stage": False, + "validate_stage": False, + } for table_name in module_config["tables"] } module_report[module_name] = {**table_trackers, "export": False} @@ -129,6 +132,7 @@ def orchestrate_module( try: if table_name not in module_config["tables"]: report["status"] = "failed" + return False table_config = module_config["tables"][table_name]