Skip to content
Closed
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
56fbe93
fix(br_me_caged): migra para o poll novo (cobertura vs cobertura, não…
Winzen Aug 6, 2026
cdb1ed4
docs(br_me_caged): sinaliza onde trocar env para dev ao testar
Winzen Aug 6, 2026
f3ba221
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 6, 2026
cdb1fdf
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 6, 2026
e17a6b9
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 7, 2026
234cf81
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 7, 2026
209f400
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 7, 2026
fa75f71
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 7, 2026
4e5900d
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 7, 2026
a794580
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 8, 2026
2601acf
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 8, 2026
d153efa
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 8, 2026
7467054
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 11, 2026
65b135b
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 11, 2026
fd39daf
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 11, 2026
ebd81da
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 11, 2026
a383204
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 12, 2026
d05f895
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 12, 2026
050c1f5
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 12, 2026
457cf10
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 12, 2026
3d41be8
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 12, 2026
8b220d5
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 12, 2026
5102003
Merge branch 'main' into fix/br_me_caged_poll_migration
mergify[bot] Aug 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 20 additions & 25 deletions pipelines/datasets/br_me_caged/flows.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
YearMonth,
)
from pipelines.utils.metadata.tasks import (
commit_source_update_task,
poll_source_for_update_task,
register_table_materialization_task,
check_source_is_ahead_of_table_task,
register_source_coverage_task,
sync_table_coverage_task,
)
from pipelines.utils.tasks import (
rename_flow_run_dataset_table,
Expand All @@ -46,17 +46,21 @@ def _run_me_caged(

source_last_date = get_source_last_date()

if not force_run:
has_new_data = poll_source_for_update_task(
dataset_id=dataset_id,
table_id=table_id,
source_max_date=source_last_date,
env="prod",
date_format="%Y-%m",
)
if not has_new_data:
print(f"No updates for table {table_id}!")
return
register_source_coverage_task(
dataset_id=dataset_id,
table_id=table_id,
source_max_date=source_last_date,
env="prod", # trocar para "dev" ao testar localmente
date_format="%Y-%m",
)

if not force_run and not check_source_is_ahead_of_table_task(
dataset_id=dataset_id,
table_id=table_id,
env="prod", # trocar para "dev" ao testar localmente
):
print(f"No updates for table {table_id}!")
return
Comment thread
coderabbitai[bot] marked this conversation as resolved.

table_last_date = get_table_last_date(dataset_id, table_id)
_input_dir, output_dir = build_table_paths(table_id)
Expand Down Expand Up @@ -104,26 +108,17 @@ def _run_me_caged(
)

if update_metadata:
register_table_materialization_task(
sync_table_coverage_task(
dataset_id=dataset_id,
table_id=table_id,
coverage=PartBdpro(
date_column=YearMonth(year="ano", month="mes"),
date_format=DateFormat.YEAR_MONTH,
),
env="prod",
env="prod", # trocar para "dev" ao testar localmente
bq_project="basedosdados",
)

if source_last_date is not None:
commit_source_update_task(
dataset_id=dataset_id,
table_id=table_id,
source_max_date=source_last_date,
env="prod",
date_format="%Y-%m",
)


def _caged_flow(table_id: str, cron: str):
@flow(
Expand Down
Loading