From 55ab33d9db3621709f66e66ee331cf3bef5acc90 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 9 Sep 2026 15:54:18 +1000 Subject: [PATCH 1/7] feat(us_census_bps): add architecture, download and cleaning transform Onboards the Census Building Permits Survey into ten long tables plus a dictionary: monthly and annual counts and valuation of new residential construction by permit-issuing place, county, metropolitan area and state. The survey's ASCII files changed layout six times between 1980 and 2026, so parsing is driven by the two header rows each file carries rather than by the file name or the year. All 3,409 published files resolve into 21 distinct layouts with no unknown columns. Source behaviours handled, each of which loses data silently otherwise: - www2.census.gov answers HTTP 200 with a firewall "Request Rejected" page for a few valid URLs. Treating that as a 404 dropped four place months and one metropolitan month; a cache-busting query string recovers them. - 977 pre-2000 files end with a DOS end-of-file byte, which lands in a full-width row of empty fields in two of them. - The county files list some counties twice under variant spellings, with identical figures. Duplicates are collapsed on geography identity only, since one 2000 duplicate carries a wrong region code, and a repeat whose figures differ raises rather than being collapsed. - The January and March 1998 metropolitan files append 58 state records that all carry an identical, wrong measure block; they are dropped. - Valuation is published in thousands of dollars at state and metropolitan level and in dollars at county and place level. Everything is normalised to dollars, which makes county and state dollars per housing unit agree exactly for 2020-2025. 25,681,364 rows. Every table's key is unique, the published regional and divisional totals reconcile to the national total, and place-level units sum to the county files exactly for 99 to 100 percent of counties. --- .../code/architecture/dicionario.csv | 6 + .../code/architecture/permit_cbsa_annual.csv | 14 + .../code/architecture/permit_cbsa_monthly.csv | 15 + .../architecture/permit_county_annual.csv | 14 + .../architecture/permit_county_monthly.csv | 15 + .../code/architecture/permit_msa_annual.csv | 12 + .../code/architecture/permit_msa_monthly.csv | 13 + .../code/architecture/permit_place_annual.csv | 27 + .../architecture/permit_place_monthly.csv | 25 + .../code/architecture/permit_state_annual.csv | 15 + .../architecture/permit_state_monthly.csv | 16 + .../us_census_bps/code/build_architecture.py | 651 ++++++++++++++ models/us_census_bps/code/clean.py | 48 ++ models/us_census_bps/code/download.py | 162 ++++ models/us_census_bps/code/validate.py | 164 ++++ pipelines/datasets/us_census_bps/__init__.py | 0 pipelines/datasets/us_census_bps/constants.py | 85 ++ pipelines/datasets/us_census_bps/utils.py | 802 ++++++++++++++++++ 18 files changed, 2084 insertions(+) create mode 100644 models/us_census_bps/code/architecture/dicionario.csv create mode 100644 models/us_census_bps/code/architecture/permit_cbsa_annual.csv create mode 100644 models/us_census_bps/code/architecture/permit_cbsa_monthly.csv create mode 100644 models/us_census_bps/code/architecture/permit_county_annual.csv create mode 100644 models/us_census_bps/code/architecture/permit_county_monthly.csv create mode 100644 models/us_census_bps/code/architecture/permit_msa_annual.csv create mode 100644 models/us_census_bps/code/architecture/permit_msa_monthly.csv create mode 100644 models/us_census_bps/code/architecture/permit_place_annual.csv create mode 100644 models/us_census_bps/code/architecture/permit_place_monthly.csv create mode 100644 models/us_census_bps/code/architecture/permit_state_annual.csv create mode 100644 models/us_census_bps/code/architecture/permit_state_monthly.csv create mode 100644 models/us_census_bps/code/build_architecture.py create mode 100644 models/us_census_bps/code/clean.py create mode 100644 models/us_census_bps/code/download.py create mode 100644 models/us_census_bps/code/validate.py create mode 100644 pipelines/datasets/us_census_bps/__init__.py create mode 100644 pipelines/datasets/us_census_bps/constants.py create mode 100644 pipelines/datasets/us_census_bps/utils.py diff --git a/models/us_census_bps/code/architecture/dicionario.csv b/models/us_census_bps/code/architecture/dicionario.csv new file mode 100644 index 0000000000..0133da4682 --- /dev/null +++ b/models/us_census_bps/code/architecture/dicionario.csv @@ -0,0 +1,6 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +id_tabela,STRING,Slug of the us_census_bps table the dictionary entry describes,,no,,,no,,id_tabela +nome_coluna,STRING,Name of the column the dictionary entry describes,,no,,,no,,nome_coluna +chave,STRING,Coded value (key) exactly as stored in the data,,no,,,no,,chave +cobertura_temporal,STRING,Temporal coverage of the key,,no,,,no,,cobertura_temporal +valor,STRING,Human-readable label corresponding to the coded value,,no,,,no,,valor diff --git a/models/us_census_bps/code/architecture/permit_cbsa_annual.csv b/models/us_census_bps/code/architecture/permit_cbsa_annual.csv new file mode 100644 index 0000000000..40f1b6743f --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_cbsa_annual.csv @@ -0,0 +1,14 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,2003(1)2025,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +cbsa_id,STRING,Five-digit Core Based Statistical Area code,,no,diretorios_us.cbsa_2023:id_cbsa,,no,,CBSA Code +csa_id,STRING,Three-digit Combined Statistical Area code the CBSA belongs to,,no,,,no,"Null where the source publishes 999, meaning the CBSA is not part of a CSA",CSA Code +cbsa_type,STRING,Header code distinguishing metropolitan from micropolitan areas,2024(1)2025,yes,,,no,"Published from January 2024 only, when micropolitan areas were added to the file. Null before that",HHEADER +full_monthly_coverage,STRING,Whether the area is completely covered by monthly reporting permit-issuing places,2003(1)2023,yes,,,no,"Published up to 2023 only, in the field replaced by cbsa_type from January 2024. Null from 2024 on",MONCOV +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +cbsa_name,STRING,"Name of the Core Based Statistical Area, as published",,no,,,no,,CBSA Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_cbsa_monthly.csv b/models/us_census_bps/code/architecture/permit_cbsa_monthly.csv new file mode 100644 index 0000000000..4cca893870 --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_cbsa_monthly.csv @@ -0,0 +1,15 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,2004(1)2026,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +month,INT64,"Reference month of the survey period, from 1 to 12",2004(1)2026,no,br_bd_diretorios_data_tempo.mes:mes,month,no,,Survey Date +cbsa_id,STRING,Five-digit Core Based Statistical Area code,,no,diretorios_us.cbsa_2023:id_cbsa,,no,,CBSA Code +csa_id,STRING,Three-digit Combined Statistical Area code the CBSA belongs to,,no,,,no,"Null where the source publishes 999, meaning the CBSA is not part of a CSA",CSA Code +cbsa_type,STRING,Header code distinguishing metropolitan from micropolitan areas,2024(1)2025,yes,,,no,"Published from January 2024 only, when micropolitan areas were added to the file. Null before that",HHEADER +full_monthly_coverage,STRING,Whether the area is completely covered by monthly reporting permit-issuing places,2003(1)2023,yes,,,no,"Published up to 2023 only, in the field replaced by cbsa_type from January 2024. Null from 2024 on",MONCOV +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +cbsa_name,STRING,"Name of the Core Based Statistical Area, as published",,no,,,no,,CBSA Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_county_annual.csv b/models/us_census_bps/code/architecture/permit_county_annual.csv new file mode 100644 index 0000000000..b51283dad1 --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_county_annual.csv @@ -0,0 +1,14 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,1990(1)2025,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +county_id,STRING,"Five-digit FIPS county code, state code followed by county code",,no,diretorios_us.county:id_county,,no,,FIPS State + FIPS County +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,,FIPS State +region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code +division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +county_name,STRING,"Name of the county, as published",,no,,,no,,County Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_county_monthly.csv b/models/us_census_bps/code/architecture/permit_county_monthly.csv new file mode 100644 index 0000000000..5c02464d7b --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_county_monthly.csv @@ -0,0 +1,15 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,2000(1)2026,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +month,INT64,"Reference month of the survey period, from 1 to 12",2000(1)2026,no,br_bd_diretorios_data_tempo.mes:mes,month,no,,Survey Date +county_id,STRING,"Five-digit FIPS county code, state code followed by county code",,no,diretorios_us.county:id_county,,no,,FIPS State + FIPS County +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,,FIPS State +region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code +division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +county_name,STRING,"Name of the county, as published",,no,,,no,,County Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_msa_annual.csv b/models/us_census_bps/code/architecture/permit_msa_annual.csv new file mode 100644 index 0000000000..c8a224536f --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_msa_annual.csv @@ -0,0 +1,12 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,1980(1)2002,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +msa_cmsa_id,STRING,Four-digit Metropolitan or Consolidated Metropolitan Statistical Area code,,no,,,no,"The pre-2004 MSA/CMSA code system, superseded by CBSA codes. The two are different geographies, not a renaming, so this table does not join to the CBSA tables or to the CBSA directory",MSA/CMSA Code +pmsa_id,STRING,Four-digit Primary Metropolitan Statistical Area code,,no,,,no,"Null where the source publishes 9999, meaning the area is not divided into PMSAs",PMSA Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +msa_name,STRING,"Name of the metropolitan area, as published",,no,,,no,,MA Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_msa_monthly.csv b/models/us_census_bps/code/architecture/permit_msa_monthly.csv new file mode 100644 index 0000000000..e63618c39f --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_msa_monthly.csv @@ -0,0 +1,13 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,1988(1)2003,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +month,INT64,"Reference month of the survey period, from 1 to 12",1988(1)2003,no,br_bd_diretorios_data_tempo.mes:mes,month,no,,Survey Date +msa_cmsa_id,STRING,Four-digit Metropolitan or Consolidated Metropolitan Statistical Area code,,no,,,no,"The pre-2004 MSA/CMSA code system, superseded by CBSA codes. The two are different geographies, not a renaming, so this table does not join to the CBSA tables or to the CBSA directory",MSA/CMSA Code +pmsa_id,STRING,Four-digit Primary Metropolitan Statistical Area code,,no,,,no,"Null where the source publishes 9999, meaning the area is not divided into PMSAs",PMSA Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +msa_name,STRING,"Name of the metropolitan area, as published",,no,,,no,,MA Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_place_annual.csv b/models/us_census_bps/code/architecture/permit_place_annual.csv new file mode 100644 index 0000000000..aa6294d0ea --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_place_annual.csv @@ -0,0 +1,27 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,1980(1)2025,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,,State Code +county_id,STRING,"Five-digit FIPS county code, state code followed by county code",,no,diretorios_us.county:id_county,,no,,State Code + County Code +place_id,STRING,"Seven-digit FIPS place code, state code followed by place code",2008(1)2026,no,diretorios_us.place:id_place,,no,"Published from 2008 only. Null for the roughly 28 percent of permit offices that are minor civil divisions or county-part records rather than places, and for the balance-of-state sentinel the source writes as 99990. Where present it matches the 2020-vintage place directory for 99.6 percent of offices",FIPS Place Code +mcd_id,STRING,"Seven-digit FIPS minor civil division code, state code followed by MCD code",2008(1)2026,no,,,no,"Published from 2008 only. Identifies township and other minor civil division permit offices, which have no FIPS place code",FIPS MCD Code +cbsa_id,STRING,Five-digit Core Based Statistical Area code,2005(1)2026,no,diretorios_us.cbsa_2023:id_cbsa,,no,"Published from 2005 only, when CBSA codes replaced MSA/PMSA codes. Null where the source publishes 99999",CBSA Code +csa_id,STRING,Three-digit Combined Statistical Area code,2005(1)2026,no,,,no,Published from 2005 only. Null where the source publishes 999,CSA Code +msa_cmsa_id,STRING,Four-digit Metropolitan or Consolidated Metropolitan Statistical Area code,1988(1)2004,no,,,no,"The pre-2005 code system, superseded by cbsa_id. Null from 2005 on",MSA/CMSA Code +pmsa_id,STRING,Four-digit Primary Metropolitan Statistical Area code,1988(1)2004,no,,,no,"The pre-2005 code system, superseded by cbsa_id. Null from 2005 on and where the source publishes 9999",PMSA Code +permit_office_id,STRING,"Six-digit Building Permit Survey identifier of the permit-issuing office, unique within a state and survey period",,no,,,no,"The Census Bureau assigns this code to sort offices alphabetically within a state, so it is reassigned when offices are added and is not stable over time. Addison village, Illinois, is 001000 in 1988 and 002800 from 1995. Do not use it to build a panel; use place_id or mcd_id",6-Digit ID +census_place_id,STRING,Four-digit Census place code,2000(1)2026,no,,,no,"Published from 2000 only. A Census Bureau internal place code, not the FIPS place code",Census Place Code +region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code +division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +place_name,STRING,"Name of the permit-issuing place or office, as published",,no,,,no,,Place Name +central_city,STRING,Whether the place is a central city of a metropolitan area,2000(1)2026,yes,,,no,,Central City +footnote_code,STRING,Whether the place carries an explanatory footnote in the source release,2005(1)2026,yes,,,no,"The footnote text itself is published in a separate monthly file, not loaded here",Footnote Code +zip_code,STRING,ZIP code of the permit office or official,2000(1)2026,no,,,no,,Zip Code +population,INT64,Population of the place as carried in the source file,2008(1)2026,no,,person,no,"The Census layout document labels this the 2000 population, but the values change from release to release, so it is a current population estimate. Published from 2008 only",Pop +months_reported,INT64,Number of months of the year for which the permit office reported residential data,,no,,month,no,,Number of Months Rep +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_place_monthly.csv b/models/us_census_bps/code/architecture/permit_place_monthly.csv new file mode 100644 index 0000000000..13103dc0a1 --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_place_monthly.csv @@ -0,0 +1,25 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,1988(1)2026,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +month,INT64,"Reference month of the survey period, from 1 to 12",1988(1)2026,no,br_bd_diretorios_data_tempo.mes:mes,month,no,,Survey Date +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,,State Code +county_id,STRING,"Five-digit FIPS county code, state code followed by county code",,no,diretorios_us.county:id_county,,no,,State Code + County Code +place_id,STRING,"Seven-digit FIPS place code, state code followed by place code",2008(1)2026,no,diretorios_us.place:id_place,,no,"Published from 2008 only. Null for the roughly 28 percent of permit offices that are minor civil divisions or county-part records rather than places, and for the balance-of-state sentinel the source writes as 99990. Where present it matches the 2020-vintage place directory for 99.6 percent of offices",FIPS Place Code +mcd_id,STRING,"Seven-digit FIPS minor civil division code, state code followed by MCD code",2008(1)2026,no,,,no,"Published from 2008 only. Identifies township and other minor civil division permit offices, which have no FIPS place code",FIPS MCD Code +cbsa_id,STRING,Five-digit Core Based Statistical Area code,2005(1)2026,no,diretorios_us.cbsa_2023:id_cbsa,,no,"Published from 2005 only, when CBSA codes replaced MSA/PMSA codes. Null where the source publishes 99999",CBSA Code +csa_id,STRING,Three-digit Combined Statistical Area code,2005(1)2026,no,,,no,Published from 2005 only. Null where the source publishes 999,CSA Code +msa_cmsa_id,STRING,Four-digit Metropolitan or Consolidated Metropolitan Statistical Area code,1988(1)2004,no,,,no,"The pre-2005 code system, superseded by cbsa_id. Null from 2005 on",MSA/CMSA Code +pmsa_id,STRING,Four-digit Primary Metropolitan Statistical Area code,1988(1)2004,no,,,no,"The pre-2005 code system, superseded by cbsa_id. Null from 2005 on and where the source publishes 9999",PMSA Code +permit_office_id,STRING,"Six-digit Building Permit Survey identifier of the permit-issuing office, unique within a state and survey period",,no,,,no,"The Census Bureau assigns this code to sort offices alphabetically within a state, so it is reassigned when offices are added and is not stable over time. Addison village, Illinois, is 001000 in 1988 and 002800 from 1995. Do not use it to build a panel; use place_id or mcd_id",6-Digit ID +census_place_id,STRING,Four-digit Census place code,2000(1)2026,no,,,no,"Published from 2000 only. A Census Bureau internal place code, not the FIPS place code",Census Place Code +region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code +division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +place_name,STRING,"Name of the permit-issuing place or office, as published",,no,,,no,,Place Name +central_city,STRING,Whether the place is a central city of a metropolitan area,2000(1)2026,yes,,,no,,Central City +footnote_code,STRING,Whether the place carries an explanatory footnote in the source release,2005(1)2026,yes,,,no,"The footnote text itself is published in a separate monthly file, not loaded here",Footnote Code +zip_code,STRING,ZIP code of the permit office or official,2000(1)2026,no,,,no,,Zip Code +population,INT64,Population of the place as carried in the source file,2008(1)2026,no,,person,no,"The Census layout document labels this the 2000 population, but the values change from release to release, so it is a current population estimate. Published from 2008 only",Pop +source_code,STRING,How the data for the permit office was obtained for the month,,yes,,,no,"Codes 1 to 4 mark reported data, 5 marks imputed data and 9 marks a month with no report and no imputation. The monthly place file carries no separate reported-only block, so this column is how a reported subset is selected",Source Code +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value diff --git a/models/us_census_bps/code/architecture/permit_state_annual.csv b/models/us_census_bps/code/architecture/permit_state_annual.csv new file mode 100644 index 0000000000..40844c7bc4 --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_state_annual.csv @@ -0,0 +1,15 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,1980(1)2025,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +geography_level,STRING,Level of the geography the row describes,,yes,,,no,"The published state file also carries the national, regional and divisional totals. Filter on this column before aggregating, or the same permits are counted several times",FIPS State +geography_id,STRING,"Code identifying the geography, as published",,no,,,no,"Two-digit FIPS state code for states and territories, R1-R4 for regions, D1-D9 for divisions, US for the national total",FIPS State +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,"Null on the national, regional and divisional total rows",FIPS State +region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code +division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +geography_name,STRING,"Name of the geography, as published",,no,,,no,,State Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_state_monthly.csv b/models/us_census_bps/code/architecture/permit_state_monthly.csv new file mode 100644 index 0000000000..2282a11279 --- /dev/null +++ b/models/us_census_bps/code/architecture/permit_state_monthly.csv @@ -0,0 +1,16 @@ +name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory_column,measurement_unit,has_sensitive_data,observations,original_name +year,INT64,Reference year of the survey period,1988(1)2026,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date +month,INT64,"Reference month of the survey period, from 1 to 12",1988(1)2026,no,br_bd_diretorios_data_tempo.mes:mes,month,no,,Survey Date +geography_level,STRING,Level of the geography the row describes,,yes,,,no,"The published state file also carries the national, regional and divisional totals. Filter on this column before aggregating, or the same permits are counted several times",FIPS State +geography_id,STRING,"Code identifying the geography, as published",,no,,,no,"Two-digit FIPS state code for states and territories, R1-R4 for regions, D1-D9 for divisions, US for the national total",FIPS State +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,"Null on the national, regional and divisional total rows",FIPS State +region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code +division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code +structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block +geography_name,STRING,"Name of the geography, as published",,no,,,no,,State Name +buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/build_architecture.py b/models/us_census_bps/code/build_architecture.py new file mode 100644 index 0000000000..05babbcf35 --- /dev/null +++ b/models/us_census_bps/code/build_architecture.py @@ -0,0 +1,651 @@ +"""Emit the architecture CSVs for us_census_bps. + +The CSVs under ``architecture/`` are the single source of truth for column +names, order and BigQuery types. The cleaning transform in +``pipelines/datasets/us_census_bps/utils.py`` reads them, and the dbt models +mirror them. +""" + +from __future__ import annotations + +import csv +from pathlib import Path + +ARCH_DIR = Path(__file__).parent / "architecture" + +HEADER = [ + "name", + "bigquery_type", + "description", + "temporal_coverage", + "covered_by_dictionary", + "directory_column", + "measurement_unit", + "has_sensitive_data", + "observations", + "original_name", +] + +DIR_YEAR = "br_bd_diretorios_data_tempo.ano:ano" +DIR_MONTH = "br_bd_diretorios_data_tempo.mes:mes" +DIR_STATE = "diretorios_us.state:id_state" +DIR_COUNTY = "diretorios_us.county:id_county" +DIR_PLACE = "diretorios_us.place:id_place" +DIR_CBSA = "diretorios_us.cbsa_2023:id_cbsa" + +IMPUTED_NOTE = ( + "Estimate with imputation: reported data for responding permit offices " + "plus imputed data for non-respondents. This is the series to use by " + "default" +) +REPORTED_NOTE = ( + "Reported only: counts data from responding permit offices and makes no " + "imputation for non-respondents. Lower than the estimate with imputation" +) + + +def col( + name: str, + btype: str, + description: str, + *, + coverage: str = "", + dictionary: str = "no", + directory: str = "", + unit: str = "", + observations: str = "", + original: str = "", +) -> list[str]: + """Build one architecture row.""" + return [ + name, + btype, + description, + coverage, + dictionary, + directory, + unit, + "no", + observations, + original, + ] + + +def year_col(coverage: str) -> list[str]: + return col( + "year", + "INT64", + "Reference year of the survey period", + coverage=coverage, + directory=DIR_YEAR, + unit="year", + observations="Partition column", + original="Survey Date", + ) + + +def month_col(coverage: str) -> list[str]: + return col( + "month", + "INT64", + "Reference month of the survey period, from 1 to 12", + coverage=coverage, + directory=DIR_MONTH, + unit="month", + original="Survey Date", + ) + + +def structure_type_col() -> list[str]: + return col( + "structure_type", + "STRING", + "Type of residential structure, by the number of housing units in " + "the building", + dictionary="yes", + observations=( + "Census structure type codes: 101 single-family, 103 two-family, " + "104 three- and four-family, 105 five-or-more-family. Summing the " + "four codes reproduces the published total for the geography" + ), + original="column block", + ) + + +def measures(*, reported: bool) -> list[list[str]]: + """Buildings, units and valuation, optionally with the reported-only block.""" + rows = [ + col( + "buildings", + "INT64", + "Number of buildings authorized by building permits", + observations=IMPUTED_NOTE + ". Counted in buildings", + original="Bldgs", + ), + col( + "units", + "INT64", + "Number of housing units authorized by building permits", + observations=IMPUTED_NOTE + ". Counted in housing units", + original="Units", + ), + col( + "valuation", + "INT64", + "Construction valuation of the buildings authorized, in US dollars", + unit="usd", + observations=IMPUTED_NOTE, + original="Value", + ), + ] + if not reported: + return rows + rows += [ + col( + "buildings_reported", + "INT64", + "Number of buildings authorized, counting reporting permit " + "offices only", + observations=REPORTED_NOTE + ". Counted in buildings", + original="Bldgs rep", + ), + col( + "units_reported", + "INT64", + "Number of housing units authorized, counting reporting permit " + "offices only", + observations=REPORTED_NOTE + ". Counted in housing units", + original="Units rep", + ), + col( + "valuation_reported", + "INT64", + "Construction valuation of the buildings authorized, in US " + "dollars, counting reporting permit offices only", + unit="usd", + observations=REPORTED_NOTE, + original="Value rep", + ), + ] + return rows + + +def region_division(coverage: str = "") -> list[list[str]]: + return [ + col( + "region_id", + "STRING", + "Census region code, from 1 to 4", + coverage=coverage, + dictionary="yes", + original="Region Code", + ), + col( + "division_id", + "STRING", + "Census division code, from 1 to 9", + coverage=coverage, + dictionary="yes", + original="Division Code", + ), + ] + + +def state_table(monthly: bool) -> list[list[str]]: + cov = "1988(1)2026" if monthly else "1980(1)2025" + rows = [year_col(cov)] + if monthly: + rows.append(month_col(cov)) + rows += [ + col( + "geography_level", + "STRING", + "Level of the geography the row describes", + dictionary="yes", + observations=( + "The published state file also carries the national, " + "regional and divisional totals. Filter on this column before " + "aggregating, or the same permits are counted several times" + ), + original="FIPS State", + ), + col( + "geography_id", + "STRING", + "Code identifying the geography, as published", + observations=( + "Two-digit FIPS state code for states and territories, " + "R1-R4 for regions, D1-D9 for divisions, US for the national " + "total" + ), + original="FIPS State", + ), + col( + "state_id", + "STRING", + "Two-digit FIPS state code", + directory=DIR_STATE, + observations=( + "Null on the national, regional and divisional total rows" + ), + original="FIPS State", + ), + ] + rows += region_division() + rows += [ + structure_type_col(), + col( + "geography_name", + "STRING", + "Name of the geography, as published", + original="State Name", + ), + ] + rows += measures(reported=True) + return rows + + +def county_table(monthly: bool) -> list[list[str]]: + cov = "2000(1)2026" if monthly else "1990(1)2025" + rows = [year_col(cov)] + if monthly: + rows.append(month_col(cov)) + rows += [ + col( + "county_id", + "STRING", + "Five-digit FIPS county code, state code followed by county code", + directory=DIR_COUNTY, + original="FIPS State + FIPS County", + ), + col( + "state_id", + "STRING", + "Two-digit FIPS state code", + directory=DIR_STATE, + original="FIPS State", + ), + ] + rows += region_division() + rows += [ + structure_type_col(), + col( + "county_name", + "STRING", + "Name of the county, as published", + original="County Name", + ), + ] + rows += measures(reported=True) + return rows + + +def cbsa_table(monthly: bool) -> list[list[str]]: + cov = "2004(1)2026" if monthly else "2003(1)2025" + rows = [year_col(cov)] + if monthly: + rows.append(month_col(cov)) + rows += [ + col( + "cbsa_id", + "STRING", + "Five-digit Core Based Statistical Area code", + directory=DIR_CBSA, + original="CBSA Code", + ), + col( + "csa_id", + "STRING", + "Three-digit Combined Statistical Area code the CBSA belongs to", + observations=( + "Null where the source publishes 999, meaning the CBSA is not " + "part of a CSA" + ), + original="CSA Code", + ), + col( + "cbsa_type", + "STRING", + "Header code distinguishing metropolitan from micropolitan areas", + coverage="2024(1)2025", + dictionary="yes", + observations=( + "Published from January 2024 only, when micropolitan areas " + "were added to the file. Null before that" + ), + original="HHEADER", + ), + col( + "full_monthly_coverage", + "STRING", + "Whether the area is completely covered by monthly reporting " + "permit-issuing places", + coverage="2003(1)2023", + dictionary="yes", + observations=( + "Published up to 2023 only, in the field replaced by " + "cbsa_type from January 2024. Null from 2024 on" + ), + original="MONCOV", + ), + structure_type_col(), + col( + "cbsa_name", + "STRING", + "Name of the Core Based Statistical Area, as published", + original="CBSA Name", + ), + ] + rows += measures(reported=True) + return rows + + +def msa_table(monthly: bool) -> list[list[str]]: + cov = "1988(1)2003" if monthly else "1980(1)2002" + rows = [year_col(cov)] + if monthly: + rows.append(month_col(cov)) + rows += [ + col( + "msa_cmsa_id", + "STRING", + "Four-digit Metropolitan or Consolidated Metropolitan Statistical " + "Area code", + observations=( + "The pre-2004 MSA/CMSA code system, superseded by CBSA codes. " + "The two are different geographies, not a renaming, so this " + "table does not join to the CBSA tables or to the CBSA " + "directory" + ), + original="MSA/CMSA Code", + ), + col( + "pmsa_id", + "STRING", + "Four-digit Primary Metropolitan Statistical Area code", + observations=( + "Null where the source publishes 9999, meaning the area is " + "not divided into PMSAs" + ), + original="PMSA Code", + ), + structure_type_col(), + col( + "msa_name", + "STRING", + "Name of the metropolitan area, as published", + original="MA Name", + ), + ] + rows += measures(reported=True) + return rows + + +PLACE_GEO = [ + col( + "state_id", + "STRING", + "Two-digit FIPS state code", + directory=DIR_STATE, + original="State Code", + ), + col( + "county_id", + "STRING", + "Five-digit FIPS county code, state code followed by county code", + directory=DIR_COUNTY, + original="State Code + County Code", + ), + col( + "place_id", + "STRING", + "Seven-digit FIPS place code, state code followed by place code", + coverage="2008(1)2026", + directory=DIR_PLACE, + observations=( + "Published from 2008 only. Null for the roughly 28 percent of " + "permit offices that are minor civil divisions or county-part " + "records rather than places, and for the balance-of-state " + "sentinel the source writes as 99990. Where present it matches " + "the 2020-vintage place directory for 99.6 percent of offices" + ), + original="FIPS Place Code", + ), + col( + "mcd_id", + "STRING", + "Seven-digit FIPS minor civil division code, state code followed by " + "MCD code", + coverage="2008(1)2026", + observations=( + "Published from 2008 only. Identifies township and other minor " + "civil division permit offices, which have no FIPS place code" + ), + original="FIPS MCD Code", + ), + col( + "cbsa_id", + "STRING", + "Five-digit Core Based Statistical Area code", + coverage="2005(1)2026", + directory=DIR_CBSA, + observations=( + "Published from 2005 only, when CBSA codes replaced MSA/PMSA " + "codes. Null where the source publishes 99999" + ), + original="CBSA Code", + ), + col( + "csa_id", + "STRING", + "Three-digit Combined Statistical Area code", + coverage="2005(1)2026", + observations=( + "Published from 2005 only. Null where the source publishes 999" + ), + original="CSA Code", + ), + col( + "msa_cmsa_id", + "STRING", + "Four-digit Metropolitan or Consolidated Metropolitan Statistical " + "Area code", + coverage="1988(1)2004", + observations=( + "The pre-2005 code system, superseded by cbsa_id. Null from 2005 " + "on" + ), + original="MSA/CMSA Code", + ), + col( + "pmsa_id", + "STRING", + "Four-digit Primary Metropolitan Statistical Area code", + coverage="1988(1)2004", + observations=( + "The pre-2005 code system, superseded by cbsa_id. Null from 2005 " + "on and where the source publishes 9999" + ), + original="PMSA Code", + ), + col( + "permit_office_id", + "STRING", + "Six-digit Building Permit Survey identifier of the permit-issuing " + "office, unique within a state and survey period", + observations=( + "The Census Bureau assigns this code to sort offices " + "alphabetically within a state, so it is reassigned when offices " + "are added and is not stable over time. Addison village, " + "Illinois, is 001000 in 1988 and 002800 from 1995. Do not use it " + "to build a panel; use place_id or mcd_id" + ), + original="6-Digit ID", + ), + col( + "census_place_id", + "STRING", + "Four-digit Census place code", + coverage="2000(1)2026", + observations=( + "Published from 2000 only. A Census Bureau internal place code, " + "not the FIPS place code" + ), + original="Census Place Code", + ), +] + +PLACE_DESCRIPTIVE = [ + col( + "place_name", + "STRING", + "Name of the permit-issuing place or office, as published", + original="Place Name", + ), + col( + "central_city", + "STRING", + "Whether the place is a central city of a metropolitan area", + coverage="2000(1)2026", + dictionary="yes", + original="Central City", + ), + col( + "footnote_code", + "STRING", + "Whether the place carries an explanatory footnote in the source " + "release", + coverage="2005(1)2026", + dictionary="yes", + observations=( + "The footnote text itself is published in a separate monthly " + "file, not loaded here" + ), + original="Footnote Code", + ), + col( + "zip_code", + "STRING", + "ZIP code of the permit office or official", + coverage="2000(1)2026", + original="Zip Code", + ), + col( + "population", + "INT64", + "Population of the place as carried in the source file", + coverage="2008(1)2026", + unit="person", + observations=( + "The Census layout document labels this the 2000 population, but " + "the values change from release to release, so it is a current " + "population estimate. Published from 2008 only" + ), + original="Pop", + ), +] + + +def place_table(monthly: bool) -> list[list[str]]: + cov = "1988(1)2026" if monthly else "1980(1)2025" + rows = [year_col(cov)] + if monthly: + rows.append(month_col(cov)) + rows += PLACE_GEO + rows += region_division() + rows.append(structure_type_col()) + rows += PLACE_DESCRIPTIVE + if monthly: + rows.append( + col( + "source_code", + "STRING", + "How the data for the permit office was obtained for the " + "month", + dictionary="yes", + observations=( + "Codes 1 to 4 mark reported data, 5 marks imputed data " + "and 9 marks a month with no report and no imputation. " + "The monthly place file carries no separate reported-only " + "block, so this column is how a reported subset is " + "selected" + ), + original="Source Code", + ) + ) + else: + rows.append( + col( + "months_reported", + "INT64", + "Number of months of the year for which the permit office " + "reported residential data", + unit="month", + original="Number of Months Rep", + ) + ) + rows += measures(reported=not monthly) + return rows + + +DICIONARIO = [ + col( + "id_tabela", + "STRING", + "Slug of the us_census_bps table the dictionary entry describes", + original="id_tabela", + ), + col( + "nome_coluna", + "STRING", + "Name of the column the dictionary entry describes", + original="nome_coluna", + ), + col( + "chave", + "STRING", + "Coded value (key) exactly as stored in the data", + original="chave", + ), + col( + "cobertura_temporal", + "STRING", + "Temporal coverage of the key", + original="cobertura_temporal", + ), + col( + "valor", + "STRING", + "Human-readable label corresponding to the coded value", + original="valor", + ), +] + +TABLES = { + "permit_place_monthly": place_table(monthly=True), + "permit_place_annual": place_table(monthly=False), + "permit_county_monthly": county_table(monthly=True), + "permit_county_annual": county_table(monthly=False), + "permit_cbsa_monthly": cbsa_table(monthly=True), + "permit_cbsa_annual": cbsa_table(monthly=False), + "permit_msa_monthly": msa_table(monthly=True), + "permit_msa_annual": msa_table(monthly=False), + "permit_state_monthly": state_table(monthly=True), + "permit_state_annual": state_table(monthly=False), + "dicionario": DICIONARIO, +} + + +def main() -> None: + ARCH_DIR.mkdir(parents=True, exist_ok=True) + for table, rows in TABLES.items(): + path = ARCH_DIR / f"{table}.csv" + with path.open("w", newline="", encoding="utf-8") as fh: + writer = csv.writer(fh, lineterminator="\n") + writer.writerow(HEADER) + writer.writerows(rows) + print(f"{table}: {len(rows)} columns -> {path.name}") + + +if __name__ == "__main__": + main() diff --git a/models/us_census_bps/code/clean.py b/models/us_census_bps/code/clean.py new file mode 100644 index 0000000000..d0b8a10d5f --- /dev/null +++ b/models/us_census_bps/code/clean.py @@ -0,0 +1,48 @@ +"""Clean the downloaded Census Building Permits Survey files into Parquet. + +Reads from ``$BPS_DATA_DIR/input`` and writes ``$BPS_DATA_DIR/output`` +(default ``~/Downloads/us_census_bps_data``). The transform itself lives in +``pipelines.datasets.us_census_bps.utils`` so the recurring pipeline and this +one-shot bootstrap cannot drift apart. +""" + +from __future__ import annotations + +import argparse +import logging +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from pipelines.datasets.us_census_bps.utils import clean_all + +DATA_DIR = Path( + os.environ.get( + "BPS_DATA_DIR", Path.home() / "Downloads/us_census_bps_data" + ) +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("tables", nargs="*", help="restrict to these tables") + args = parser.parse_args() + logging.basicConfig( + level=logging.INFO, format="%(asctime)s %(message)s", stream=sys.stdout + ) + totals = clean_all( + DATA_DIR / "input", + DATA_DIR / "output", + only=set(args.tables) or None, + ) + print("\n=== rows written ===") + for table, rows in sorted(totals.items()): + print(f" {table:24s} {rows:>12,}") + print(f" {'TOTAL':24s} {sum(totals.values()):>12,}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/models/us_census_bps/code/download.py b/models/us_census_bps/code/download.py new file mode 100644 index 0000000000..85d1093790 --- /dev/null +++ b/models/us_census_bps/code/download.py @@ -0,0 +1,162 @@ +"""Download the Census Building Permits Survey ASCII files. + +Writes to ``$BPS_DATA_DIR`` (default ``~/Downloads/us_census_bps_data/input``). + +Two Census server behaviours make a naive download lose data silently: + +* A file that does not exist is served as an HTML page. It does carry HTTP + 404, but the body must still be checked so an HTML page is never written to + disk as if it were data. +* A small number of perfectly valid URLs are rejected by the site firewall, + which answers **HTTP 200** with a "Request Rejected" HTML page. Treating + that as a missing file drops whole months without any error — four place + months and one metropolitan month were lost this way on the first run. + The rejection is cached against the exact URL, so a cache-busting query + string gets the real file. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures as cf +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from pipelines.datasets.us_census_bps.constants import constants + +BASE = constants.BASE_URL.value +HEADERS = constants.HEADERS.value +THIS_YEAR = time.gmtime().tm_year + +DATA_DIR = Path( + os.environ.get( + "BPS_DATA_DIR", Path.home() / "Downloads/us_census_bps_data" + ) +) +INPUT_DIR = DATA_DIR / "input" + + +def targets() -> list[tuple[str, Path]]: + """Build the (url, destination) list for every published BPS file.""" + out: list[tuple[str, Path]] = [] + for level, directory in constants.GEO_DIRS.value.items(): + prefix = constants.GEO_PREFIXES.value[level] + for kind in ("monthly", "annual"): + first = constants.FIRST_YEAR.value[(level, kind)] + for year in range(first, THIS_YEAR + 1): + names = ( + [ + f"{prefix}{year % 100:02d}{m:02d}c.txt" + for m in range(1, 13) + ] + if kind == "monthly" + else [f"{prefix}{year}a.txt"] + ) + for name in names: + url = BASE + urllib.parse.quote(f"{directory}/{name}") + out.append((url, INPUT_DIR / level / name)) + for directory, prefix in constants.PLACE_REGIONS.value.items(): + for kind in ("monthly", "annual"): + first = constants.FIRST_YEAR.value[("place", kind)] + for year in range(first, THIS_YEAR + 1): + names = ( + [ + f"{prefix}{year % 100:02d}{m:02d}c.txt" + for m in range(1, 13) + ] + if kind == "monthly" + else [f"{prefix}{year}a.txt"] + ) + for name in names: + url = BASE + urllib.parse.quote( + f"Place/{directory}/{name}" + ) + out.append((url, INPUT_DIR / "place" / name)) + return out + + +def fetch(url: str, dest: Path, tries: int = 5) -> tuple[str, str]: + """Download one file. + + Args: + url: Absolute URL of the Census file. + dest: Destination path; parent directories are created. + tries: Attempts before giving up. + + Returns: + ``(status, detail)`` where status is "ok", "cached", "missing" or + "failed". A "missing" result means the server answered HTTP 404. + """ + if dest.exists() and dest.stat().st_size > 0: + return "cached", "" + detail = "" + for attempt in range(tries): + # The firewall caches its rejection against the exact URL, so every + # retry after the first carries a different cache-busting parameter. + target = url if attempt == 0 else f"{url}?attempt={attempt}" + req = urllib.request.Request(target, headers=HEADERS) + try: + with urllib.request.urlopen(req, timeout=300) as resp: + body = resp.read() + except urllib.error.HTTPError as exc: + if exc.code == 404: + return "missing", "HTTP 404" + detail = f"HTTP {exc.code}" + time.sleep(2 * (attempt + 1)) + continue + except Exception as exc: + detail = f"{type(exc).__name__}: {exc}" + time.sleep(2 * (attempt + 1)) + continue + head = body[:1000].lower() + if b" int: + parser = argparse.ArgumentParser() + parser.add_argument("--workers", type=int, default=6) + args = parser.parse_args() + + jobs = targets() + print(f"{len(jobs)} candidate files -> {INPUT_DIR}") + counts = {"ok": 0, "cached": 0, "missing": 0, "failed": 0} + failures: list[str] = [] + with cf.ThreadPoolExecutor(max_workers=args.workers) as pool: + futures = {pool.submit(fetch, u, d): u for u, d in jobs} + for i, fut in enumerate(cf.as_completed(futures), start=1): + status, detail = fut.result() + counts[status] += 1 + if status == "failed": + failures.append(f"{futures[fut]} -- {detail}") + if i % 250 == 0: + print(f" {i}/{len(jobs)} {counts}", flush=True) + print(f"done: {counts}") + if failures: + print("FAILED (not a 404 -- do not treat as missing):") + for line in failures[:30]: + print(" ", line) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/models/us_census_bps/code/validate.py b/models/us_census_bps/code/validate.py new file mode 100644 index 0000000000..60ec0233d7 --- /dev/null +++ b/models/us_census_bps/code/validate.py @@ -0,0 +1,164 @@ +"""Validate the cleaned us_census_bps Parquet against the survey's own totals. + +Runs entirely on the local Parquet with DuckDB, before anything is uploaded. +Checks key uniqueness, the aggregation identities the survey publishes, and +the share of place records that reach the US place directory. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import duckdb + +DATA_DIR = Path( + os.environ.get( + "BPS_DATA_DIR", Path.home() / "Downloads/us_census_bps_data" + ) +) +OUT = DATA_DIR / "output" + + +def src(table: str) -> str: + """Return a DuckDB scan expression for a table's hive-partitioned Parquet.""" + return ( + f"read_parquet('{OUT}/{table}/**/*.parquet', " + f"hive_partitioning=false, union_by_name=true)" + ) + + +KEYS = { + "permit_place_monthly": ["year", "month", "state_id", "permit_office_id"], + "permit_place_annual": ["year", "state_id", "permit_office_id"], + # The county files carry one "Balance of State" record per state, with no + # FIPS county code, so the state is part of the key. + "permit_county_monthly": ["year", "month", "state_id", "county_id"], + "permit_county_annual": ["year", "state_id", "county_id"], + "permit_cbsa_monthly": ["year", "month", "cbsa_id"], + "permit_cbsa_annual": ["year", "cbsa_id"], + "permit_msa_monthly": ["year", "month", "msa_cmsa_id", "pmsa_id"], + "permit_msa_annual": ["year", "msa_cmsa_id", "pmsa_id"], + "permit_state_monthly": ["year", "month", "geography_id"], + "permit_state_annual": ["year", "geography_id"], +} + + +def main() -> int: + con = duckdb.connect() + failures: list[str] = [] + + print("=== row counts and key uniqueness ===") + for table, key in KEYS.items(): + cols = ", ".join([*key, "structure_type"]) + counts = con.execute( + f"select count(*), count(distinct ({cols})) from {src(table)}" + ).fetchone() + assert counts is not None + n, k = counts + flag = "OK" if n == k else "DUPLICATE KEYS" + if n != k: + failures.append(f"{table}: {n - k:,} duplicate key rows") + print(f" {table:24s} {n:>12,} rows key={flag}") + + print("\n=== survey aggregation identities (units, latest 5 years) ===") + # Regions and divisions must sum to the published national total. + rows = con.execute(f""" + with a as ( + select year, geography_level, sum(cast(units as bigint)) u + from {src("permit_state_annual")} group by 1, 2 + ) + select year, + max(u) filter (where geography_level='nation') as nation, + max(u) filter (where geography_level='region') as regions, + max(u) filter (where geography_level='division') as divisions, + max(u) filter (where geography_level='state') as states + from (select year, geography_level, sum(u) u from a group by 1, 2) + group by 1 order by year desc limit 5 + """).fetchall() + for year, nation, regions, divisions, states in rows: + ok = nation == regions == divisions + if not ok: + failures.append(f"state_annual {year}: region/division != nation") + print( + f" {year} nation={nation:>9,} regions={regions:>9,} " + f"divisions={divisions:>9,} states={states:>9,} " + f"{'OK' if ok else 'MISMATCH'}" + ) + + print("\n=== place records reaching the FIPS place directory ===") + rows = con.execute(f""" + select year, + count(distinct permit_office_id || state_id) offices, + count(distinct case when place_id is not null + then permit_office_id || state_id end) with_place + from {src("permit_place_annual")} + where cast(year as int) in (2008, 2015, 2025) group by 1 order by 1 + """).fetchall() + for year, offices, with_place in rows: + print( + f" {year}: {with_place:,}/{offices:,} offices carry a FIPS place " + f"code ({with_place / offices * 100:.1f}%)" + ) + + print("\n=== valuation sanity (annual, USD per housing unit) ===") + rows = con.execute(f""" + select 'state' lvl, year, + sum(cast(valuation as bigint)) / nullif(sum(cast(units as bigint)), 0) v + from {src("permit_state_annual")} where geography_level='nation' and cast(year as int)>=2020 + group by 1,2 + union all + select 'county', year, + sum(cast(valuation as bigint)) / nullif(sum(cast(units as bigint)), 0) + from {src("permit_county_annual")} where cast(year as int)>=2020 group by 1,2 + order by 2, 1 + """).fetchall() + for lvl, year, value in rows: + print(f" {lvl:7s} {year} ${value:,.0f} per unit") + + print("\n=== cross-level agreement, annual units ===") + rows = con.execute(f""" + with p as ( + select year, county_id, sum(cast(units as bigint)) u + from {src("permit_place_annual")} where county_id is not null + group by 1, 2 + ), + c as ( + select year, county_id, sum(cast(units as bigint)) u + from {src("permit_county_annual")} where county_id is not null + group by 1, 2 + ), + j as (select p.year, p.u pu, c.u cu from p join c using (year, county_id)) + select year, count(*) counties, sum(pu) place_units, sum(cu) county_units, + count(*) filter (where pu = cu) exact + from j where cast(year as int) in (2000, 2010, 2020, 2025) + group by 1 order by 1 + """).fetchall() + for year, counties, place_units, county_units, exact in rows: + gap = (place_units - county_units) / county_units * 100 + print( + f" {year}: {counties:,} counties place={place_units:,} " + f"county={county_units:,} gap={gap:+.3f}% " + f"exact match on {exact:,} ({exact / counties * 100:.1f}%)" + ) + + print("\n=== structure types present ===") + print( + " ", + con.execute( + f"select distinct structure_type from {src('permit_state_annual')} " + "order by 1" + ).fetchall(), + ) + + if failures: + print(f"\nFAILURES ({len(failures)}):") + for line in failures: + print(" ", line) + return 1 + print("\nAll checks passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pipelines/datasets/us_census_bps/__init__.py b/pipelines/datasets/us_census_bps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pipelines/datasets/us_census_bps/constants.py b/pipelines/datasets/us_census_bps/constants.py new file mode 100644 index 0000000000..d35246686f --- /dev/null +++ b/pipelines/datasets/us_census_bps/constants.py @@ -0,0 +1,85 @@ +"""Constants for the us_census_bps pipeline.""" + +from enum import Enum +from pathlib import Path + + +class constants(Enum): + """Constants for the Census Building Permits Survey pipeline.""" + + DATASET_ID = "us_census_bps" + + BASE_URL = "https://www2.census.gov/econ/bps/" + + HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36" + ) + } + + # Region directories and their file-name prefixes for the place files. + PLACE_REGIONS = { + "Northeast Region": "ne", + "Midwest Region": "mw", + "South Region": "so", + "West Region": "we", + } + + # Directory on the Census server for each non-place geography level. + GEO_DIRS = { + "state": "State", + "county": "County", + "metro": "Metro (ending 2023)", + "cbsa": "CBSA (beginning Jan 2024)", + } + + # File-name prefix for each non-place geography level. + GEO_PREFIXES = { + "state": "st", + "county": "co", + "metro": "ma", + "cbsa": "cbsa", + } + + # First survey year available per level and periodicity. + FIRST_YEAR = { + ("state", "monthly"): 1988, + ("state", "annual"): 1980, + ("county", "monthly"): 2000, + ("county", "annual"): 1990, + ("metro", "monthly"): 1988, + ("metro", "annual"): 1980, + ("cbsa", "monthly"): 2024, + ("cbsa", "annual"): 2024, + ("place", "monthly"): 1988, + ("place", "annual"): 1980, + } + + # Valuation is published in thousands of dollars at these levels. + THOUSANDS_LEVELS = ("state", "metro", "cbsa") + + # Census structure type codes, in published column order. + STRUCTURE_TYPES = ("101", "103", "104", "105") + + ARCHITECTURE_DIR = ( + Path(__file__).resolve().parents[3] + / "models" + / "us_census_bps" + / "code" + / "architecture" + ) + + TABLES = ( + "permit_place_monthly", + "permit_place_annual", + "permit_county_monthly", + "permit_county_annual", + "permit_cbsa_monthly", + "permit_cbsa_annual", + "permit_msa_monthly", + "permit_msa_annual", + "permit_state_monthly", + "permit_state_annual", + "dicionario", + ) diff --git a/pipelines/datasets/us_census_bps/utils.py b/pipelines/datasets/us_census_bps/utils.py new file mode 100644 index 0000000000..42819617bb --- /dev/null +++ b/pipelines/datasets/us_census_bps/utils.py @@ -0,0 +1,802 @@ +"""Download and cleaning transform for us_census_bps. + +Shared by the recurring pipeline and the one-shot bootstrap in +``models/us_census_bps/code/``. Pure functions, no Prefect, so they are +importable and unit-testable. + +The Census Building Permits Survey publishes comma-delimited ASCII files whose +column layout changed six times between 1980 and 2026 — fields were added, +moved and redefined. Parsing is therefore driven by the two header rows each +file carries, never by the file name or the year, so a further change is a loud +failure rather than silently shifted data. +""" + +from __future__ import annotations + +import csv +import logging +import re +import shutil +from collections.abc import Iterator +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from pipelines.datasets.us_census_bps.constants import constants + +log = logging.getLogger("us_census_bps") + +PA = {"STRING": pa.string(), "INT64": pa.int64(), "FLOAT64": pa.float64()} +_ARCH = constants.ARCHITECTURE_DIR.value +STRUCTURE_TYPES = constants.STRUCTURE_TYPES.value + +# ── header resolution ─────────────────────────────────────────────────────── + +# (header row 1, header row 2) -> canonical field name. +_PAIR: dict[tuple[str, str], str] = { + ("survey", "date"): "survey_date", + ("fips", "state"): "state", + ("fips", "county"): "county", + ("state", "code"): "state", + ("county", "code"): "county", + ("place", "code"): "census_place", + ("census place", "code"): "census_place", + ("fips place", "code"): "fips_place", + ("fips mcd", "code"): "fips_mcd", + ("csa", "code"): "csa", + ("cbsa", "code"): "cbsa", + ("msa/", "cmsa"): "msa_cmsa", + ("pmsa", "code"): "pmsa", + ("footnote", "code"): "footnote", + ("central", "city"): "central_city", + ("ccity", "code"): "central_city", + ("zip", "code"): "zip", + ("region", "code"): "region", + ("division", "code"): "division", + ("source", "code"): "source", + ("6-digit", "id"): "office", + ("number of", "months rep"): "months_reported", +} + +# Header row 1 alone, for the layouts where row 2 is blank or shifted. +_SINGLE: dict[str, str] = { + "survey": "survey_date", + "state": "state", + "county": "county", + "place": "census_place", + "census place": "census_place", + "fips place": "fips_place", + "fips mcd": "fips_mcd", + "pop": "pop", + "csa": "csa", + "cbsa": "cbsa", + "msa/": "msa_cmsa", + "pmsa": "pmsa", + "footnote": "footnote", + "central": "central_city", + "ccity": "central_city", + "zip": "zip", + "region": "region", + "division": "division", + "source": "source", + "6-digit": "office", + "number of": "months_reported", + "moncov": "moncov", + "hheader": "hheader", +} + + +def _norm(value: str) -> str: + """Lowercase, strip and collapse whitespace in a header cell.""" + return re.sub(r"\s+", " ", value.replace("\xa0", " ")).strip().lower() + + +def resolve_fields(head1: list[str], head2: list[str], n_id: int) -> list[str]: + """Map the identifier columns of a BPS file to canonical field names. + + The last identifier column is the geography name in every published + layout, so it is assigned directly rather than looked up. + + Args: + head1: First header row, split on commas. + head2: Second header row, split on commas. + n_id: Number of identifier columns, i.e. the index of the first + measure column. + + Returns: + ``n_id`` field names; ``""`` marks a column the file leaves blank. + + Raises: + ValueError: If a labelled column cannot be resolved, which means the + Census Bureau changed the layout again. + """ + fields: list[str] = [] + for i in range(n_id): + if i == n_id - 1: + fields.append("name") + continue + a = _norm(head1[i]) if i < len(head1) else "" + b = _norm(head2[i]) if i < len(head2) else "" + if not a and not b: + fields.append("") + continue + name = _PAIR.get((a, b)) or _SINGLE.get(a) or _SINGLE.get(b) + if name is None: + raise ValueError( + f"unknown BPS header column {i}: row1={a!r} row2={b!r}" + ) + fields.append(name) + return fields + + +# ── file identity ─────────────────────────────────────────────────────────── + +_FILE_RE = re.compile( + r"^(?Pst|co|cbsa|ma|ne|mw|so|we)" + r"(?:(?P\d{2})(?P\d{2})c|(?P\d{4})a)\.txt$", + re.IGNORECASE, +) + +_PREFIX_LEVEL = { + "st": "state", + "co": "county", + "cbsa": "cbsa", + "ma": "metro", + "ne": "place", + "mw": "place", + "so": "place", + "we": "place", +} + + +def parse_filename(name: str) -> tuple[str, str, int, int | None]: + """Read the geography level, periodicity and period from a file name. + + Args: + name: Bare file name, e.g. ``"mw2507c.txt"``. + + Returns: + ``(level, periodicity, year, month)``; ``month`` is None for annual + files. + + Raises: + ValueError: If the name does not match a published BPS file name. + """ + m = _FILE_RE.match(name) + if not m: + raise ValueError(f"unrecognised BPS file name: {name}") + level = _PREFIX_LEVEL[m.group("prefix").lower()] + if m.group("yyyy"): + return level, "annual", int(m.group("yyyy")), None + yy = int(m.group("yy")) + year = 1900 + yy if yy >= 80 else 2000 + yy + return level, "monthly", year, int(m.group("mm")) + + +def check_survey_date(raw: str, year: int, month: int | None) -> None: + """Assert a row's survey-date field agrees with the file name's period. + + The Census Bureau writes this field five different ways (``YYMM``, + ``YYYYMM``, ``YYYY``, ``YYYY99`` and ``YY99``), so it is validated against + the file name rather than parsed as the authority. + + Args: + raw: Survey date exactly as published. + year: Year taken from the file name. + month: Month taken from the file name, or None for annual files. + + Raises: + ValueError: If the field cannot be reconciled with the file name. + """ + digits = raw.strip() + if not digits.isdigit(): + raise ValueError(f"non-numeric survey date {raw!r}") + if month is None: + ok = digits in { + str(year), + f"{year}99", + f"{year % 100:02d}99", + f"{year % 100:02d}{99}", + } + else: + ok = digits in {f"{year}{month:02d}", f"{year % 100:02d}{month:02d}"} + if not ok: + raise ValueError( + f"survey date {raw!r} does not match file period " + f"{year}-{month if month else 'annual'}" + ) + + +# ── value cleaning ────────────────────────────────────────────────────────── + +# Values the source uses to mean "not applicable" for a given code column. +_SENTINEL = { + "csa": {"999", "0999"}, + "cbsa": {"99999", "0"}, + "pmsa": {"9999"}, + "msa_cmsa": {"9999"}, + "region": {"0"}, + "division": {"0"}, +} + + +def _clean_code(field: str, raw: str) -> str | None: + """Strip a code field and map the source's not-applicable sentinels to None.""" + value = raw.strip().lstrip("-").strip() + if not value: + return None + if value in _SENTINEL.get(field, frozenset()): + return None + return value + + +def _clean_int(raw: str) -> int | None: + """Parse an integer measure, returning None when the field is blank.""" + value = raw.strip().replace(",", "") + if not value or value in {"-", "."}: + return None + try: + return int(float(value)) + except ValueError: + return None + + +_NAME_DOTS = re.compile(r"[.\s]+$") + + +def _clean_name(raw: str) -> str | None: + """Strip the dot leaders the pre-1996 files pad geography names with.""" + value = _NAME_DOTS.sub("", raw.strip()) + return value or None + + +def _pad(value: str | None, width: int) -> str | None: + """Zero-pad a numeric code, leaving a non-numeric code untouched.""" + if value is None: + return None + return value.zfill(width) if value.isdigit() else value + + +# ── parsing ───────────────────────────────────────────────────────────────── + + +def read_file(path: Path) -> tuple[list[str], list[list[str]]]: + """Read one BPS ASCII file into its identifier field names and data rows. + + Args: + path: Path to the downloaded ``.txt`` file. + + Returns: + ``(fields, rows)`` where ``fields`` names the identifier columns and + ``rows`` holds every non-blank data row, already split on commas. + + Raises: + ValueError: If the header cannot be located or the measure block is + neither 12 nor 24 columns wide. + """ + # 977 of the pre-2000 files end with a DOS end-of-file byte. In two of + # them it lands in a full-width row of empty fields, which would otherwise + # be parsed as data. + text = path.read_bytes().decode("latin-1").replace("\x1a", "") + lines = text.splitlines() + if len(lines) < 3: + raise ValueError(f"{path.name}: fewer than 3 lines") + head1 = next(csv.reader([lines[0]])) + head2 = next(csv.reader([lines[1]])) + rows = [ + r for r in csv.reader(lines[2:]) if r and any(c.strip() for c in r) + ] + if not rows: + return [], [] + n_id_candidates = [i for i, c in enumerate(head2) if _norm(c) == "bldgs"] + if not n_id_candidates: + raise ValueError(f"{path.name}: no 'Bldgs' column in header row 2") + n_id = n_id_candidates[0] + width = len(rows[0]) + n_measure = width - n_id + if n_measure not in (12, 24): + raise ValueError( + f"{path.name}: {n_measure} measure columns (expected 12 or 24)" + ) + return resolve_fields(head1, head2, n_id), rows + + +# Fields that describe a geography rather than identify it. They are excluded +# from the duplicate key because the source sometimes contradicts itself on +# them: St. Clair County, Illinois is listed twice in eight 2000 county files, +# once with its correct Midwest region code and once with a wrong one, and the +# figures on both rows are identical. +_NOT_IDENTITY = frozenset( + { + "survey_date", + "name", + "region", + "division", + "source", + "months_reported", + "pop", + "zip", + "footnote", + "central_city", + "moncov", + "hheader", + "csa", + "census_place", + "fips_place", + "fips_mcd", + } +) + + +def melt_rows( + fields: list[str], + rows: list[list[str]], + *, + level: str, + year: int, + month: int | None, +) -> Iterator[dict]: + """Turn one file's rows into long records, one per structure type. + + Valuation is normalised to US dollars: the state and metropolitan files + publish it in thousands, the county and place files in dollars. + + Args: + fields: Identifier field names from :func:`read_file`. + rows: Data rows from :func:`read_file`. + level: Geography level of the file. + year: Survey year from the file name. + month: Survey month, or None for annual files. + + Yields: + One record per source row and structure type. + + Raises: + ValueError: If a row's survey date contradicts the file name. + """ + n_id = len(fields) + scale = 1000 if level in constants.THOUSANDS_LEVELS.value else 1 + # Some county files list the same county twice, usually under two + # spellings of its name ("Anchorage Borough" and "Anchorage + # Municipality"), with identical figures. Keeping both would double the + # county. The measures are compared before the repeat is discarded, so a + # genuine split would raise instead of being silently collapsed. + seen: dict[tuple[str, ...], tuple[str, ...]] = {} + identity = [f for f in fields if f and f not in _NOT_IDENTITY] + for row in rows: + if len(row) < n_id + 12: + continue + ident = {f: row[i] for i, f in enumerate(fields) if f} + check_survey_date(ident.get("survey_date", ""), year, month) + measures = row[n_id:] + key = tuple(ident[f].strip() for f in identity) + stripped = tuple(v.strip() for v in measures) + if key in seen: + if seen[key] != stripped: + raise ValueError( + f"repeated key {key} with different figures in " + f"{year}-{month}; the source split a record" + ) + continue + seen[key] = stripped + has_reported = len(measures) >= 24 + base: dict[str, object] = { + f: _clean_code(f, v) + for f, v in ident.items() + if f not in {"survey_date", "name"} + } + base["name"] = _clean_name(ident.get("name", "")) + base["year"] = year + base["month"] = month + for k, code in enumerate(STRUCTURE_TYPES): + rec: dict[str, object] = dict(base) + rec["structure_type"] = code + rec["buildings"] = _clean_int(measures[3 * k]) + rec["units"] = _clean_int(measures[3 * k + 1]) + value = _clean_int(measures[3 * k + 2]) + rec["valuation"] = None if value is None else value * scale + if has_reported: + rec["buildings_reported"] = _clean_int(measures[12 + 3 * k]) + rec["units_reported"] = _clean_int(measures[12 + 3 * k + 1]) + value_rep = _clean_int(measures[12 + 3 * k + 2]) + rec["valuation_reported"] = ( + None if value_rep is None else value_rep * scale + ) + yield rec + + +def target_table(level: str, periodicity: str, fields: list[str]) -> str: + """Choose the destination table for a parsed file. + + The metropolitan files switched from the MSA/PMSA code system to CBSA + codes mid-series — annual 2003 and monthly January 2004 — so the split is + decided by which code columns the header declares, never by the year. + + Args: + level: Geography level from the file name. + periodicity: ``"monthly"`` or ``"annual"``. + fields: Identifier field names from :func:`read_file`. + + Returns: + The destination table slug. + """ + if level in {"metro", "cbsa"}: + geo = "cbsa" if "cbsa" in fields else "msa" + else: + geo = level + return f"permit_{geo}_{periodicity}" + + +# ── field mapping to the architecture columns ─────────────────────────────── + +_PLACE_NULL_CODES = {"00000", "99990", "99999", "0", "000"} + + +def _geography_level(code: str | None) -> str | None: + """Classify a state-file geography code as nation, region, division or state.""" + if not code: + return None + if code == "US": + return "nation" + if code.startswith("R"): + return "region" + if code.startswith("D"): + return "division" + return "state" + + +def _compose(prefix: str | None, suffix: str | None, width: int) -> str | None: + """Join a state code to a sub-state code, or return None if either is absent.""" + if not prefix or not suffix or suffix in _PLACE_NULL_CODES: + return None + return f"{prefix}{suffix.zfill(width)}" + + +def to_columns(rec: dict, table: str) -> dict: + """Rename one long record's fields to the architecture column names. + + Args: + rec: A record from :func:`melt_rows`. + table: Destination table slug. + + Returns: + The record keyed by architecture column name. + """ + out: dict = { + "year": rec["year"], + "structure_type": rec["structure_type"], + "buildings": rec.get("buildings"), + "units": rec.get("units"), + "valuation": rec.get("valuation"), + } + if table.endswith("_monthly"): + out["month"] = rec["month"] + if "buildings_reported" in rec: + out["buildings_reported"] = rec.get("buildings_reported") + out["units_reported"] = rec.get("units_reported") + out["valuation_reported"] = rec.get("valuation_reported") + + state = _pad(rec.get("state"), 2) + region = rec.get("region") + division = rec.get("division") + + if "_state_" in table: + code = rec.get("state") + out["geography_level"] = _geography_level(code) + out["geography_id"] = _pad(code, 2) + out["state_id"] = state if code and code.isdigit() else None + out["region_id"] = region + out["division_id"] = division + out["geography_name"] = rec.get("name") + elif "_county_" in table: + out["county_id"] = _compose(state, rec.get("county"), 3) + out["state_id"] = state + out["region_id"] = region + out["division_id"] = division + out["county_name"] = rec.get("name") + elif "_cbsa_" in table: + out["cbsa_id"] = _pad(rec.get("cbsa"), 5) + out["csa_id"] = _pad(rec.get("csa"), 3) + out["cbsa_type"] = rec.get("hheader") + out["full_monthly_coverage"] = rec.get("moncov") + out["cbsa_name"] = rec.get("name") + elif "_msa_" in table: + out["msa_cmsa_id"] = _pad(rec.get("msa_cmsa"), 4) + out["pmsa_id"] = _pad(rec.get("pmsa"), 4) + out["msa_name"] = rec.get("name") + else: # place + out["state_id"] = state + out["county_id"] = _compose(state, rec.get("county"), 3) + out["place_id"] = _compose(state, rec.get("fips_place"), 5) + out["mcd_id"] = _compose(state, rec.get("fips_mcd"), 5) + out["cbsa_id"] = _pad(rec.get("cbsa"), 5) + out["csa_id"] = _pad(rec.get("csa"), 3) + out["msa_cmsa_id"] = _pad(rec.get("msa_cmsa"), 4) + out["pmsa_id"] = _pad(rec.get("pmsa"), 4) + out["permit_office_id"] = _pad(rec.get("office"), 6) + out["census_place_id"] = _pad(rec.get("census_place"), 4) + out["region_id"] = region + out["division_id"] = division + out["place_name"] = rec.get("name") + out["central_city"] = rec.get("central_city") + out["footnote_code"] = rec.get("footnote") + out["zip_code"] = rec.get("zip") + out["population"] = _clean_int(rec.get("pop") or "") + if table.endswith("_monthly"): + out["source_code"] = rec.get("source") + else: + out["months_reported"] = _clean_int( + rec.get("months_reported") or "" + ) + return out + + +def keep_row(row: dict, table: str) -> bool: + """Reject rows that do not belong in a metropolitan-area table. + + The January and March 1998 metropolitan files append 58 state and + balance-of-state records coded with the not-applicable metro sentinel + 9999. They are not metropolitan areas and their figures do not match the + state file, so they are dropped rather than left to corrupt any total + taken over the table. + + Args: + row: A record from :func:`to_columns`. + table: Destination table slug. + + Returns: + Whether the row should be written. + """ + if "_msa_" in table: + return row.get("msa_cmsa_id") is not None + if "_cbsa_" in table: + return row.get("cbsa_id") is not None + return True + + +# ── schema ────────────────────────────────────────────────────────────────── + + +def read_arch(table: str) -> list[dict]: + """Read a table's architecture CSV — the schema source of truth. + + Args: + table: Table slug, matching the CSV file name. + + Returns: + One dict per column, in architecture order. + """ + with (_ARCH / f"{table}.csv").open(encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def _schemas(table: str) -> tuple[list[str], pa.Schema, pa.Schema]: + """Return the column order and the typed and all-string arrow schemas.""" + arch = read_arch(table) + order = [a["name"] for a in arch] + typed = pa.schema( + [pa.field(a["name"], PA[a["bigquery_type"]]) for a in arch] + ) + strings = pa.schema([pa.field(a["name"], pa.string()) for a in arch]) + return order, typed, strings + + +def write_parquet( + records: list[dict], table: str, out_dir: Path, part: int +) -> int: + """Write one batch of records as all-STRING Snappy Parquet. + + Staging is all-STRING by Data Basis convention: ``gcs.dump_header`` + stringifies the header BigQuery infers the staging schema from, so typed + parquet is rejected on read. Values pass through the architecture's real + types first, then cast to string via arrow — never ``astype(str)``, which + would write a NULL as the literal ``"nan"`` and defeat the dbt + ``safe_cast``. + + Args: + records: Records for one table and one year. + table: Table slug. + out_dir: Root output directory. + part: Sequence number for this batch within the year partition. + + Returns: + Number of rows written. + """ + if not records: + return 0 + order, typed, strings = _schemas(table) + frame = pd.DataFrame.from_records(records) + for name in order: + if name not in frame.columns: + frame[name] = None + year = int(frame["year"].iloc[0]) + pdir = out_dir / table / f"year={year}" + pdir.mkdir(parents=True, exist_ok=True) + at = pa.Table.from_pandas(frame[order], schema=typed, preserve_index=False) + at = at.cast(strings) + pq.write_table(at, pdir / f"part_{part:04d}.parquet", compression="snappy") + return len(records) + + +# ── dictionary ────────────────────────────────────────────────────────────── + +_STRUCTURE_LABELS = { + "101": "Single-family building (1 housing unit)", + "103": "Two-family building (2 housing units)", + "104": "Three- and four-family building (3 to 4 housing units)", + "105": "Multifamily building (5 or more housing units)", +} +_GEOGRAPHY_LEVEL_LABELS = { + "nation": "United States total", + "region": "Census region total", + "division": "Census division total", + "state": "State, District of Columbia or territory", +} +_REGION_LABELS = { + "1": "Northeast", + "2": "Midwest", + "3": "South", + "4": "West", +} +_DIVISION_LABELS = { + "1": "New England", + "2": "Middle Atlantic", + "3": "East North Central", + "4": "West North Central", + "5": "South Atlantic", + "6": "East South Central", + "7": "West South Central", + "8": "Mountain", + "9": "Pacific", +} +_SOURCE_LABELS = { + "1": "Building Permits C-404 survey form", + "2": "Received data in electronic format", + "3": "Received residential data via online reporting", + "4": "Received residential data from another source, equivalent to " + "reported data", + "5": "No report received, imputed data", + "9": "No report received and no imputed data", +} +_CBSA_TYPE_LABELS = { + "2": "Metropolitan area that is part of a Combined Statistical Area", + "4": "Metropolitan area that is not part of a Combined Statistical Area", + "5": "Micropolitan area", +} +_COVERAGE_LABELS = { + "C": "Area completely covered by monthly reporting permit-issuing places" +} +_FOOTNOTE_LABELS = { + "2": "Place carries an explanatory footnote in the source release" +} +_CENTRAL_CITY_LABELS = {"1": "Place is a central city of a metropolitan area"} + +# column name -> (label map, temporal coverage) for every coded column. +_DICTIONARY: dict[str, tuple[dict[str, str], str]] = { + "structure_type": (_STRUCTURE_LABELS, ""), + "geography_level": (_GEOGRAPHY_LEVEL_LABELS, ""), + "region_id": (_REGION_LABELS, ""), + "division_id": (_DIVISION_LABELS, ""), + "source_code": (_SOURCE_LABELS, ""), + "cbsa_type": (_CBSA_TYPE_LABELS, "2024(1)2025"), + "full_monthly_coverage": (_COVERAGE_LABELS, "2003(1)2023"), + "footnote_code": (_FOOTNOTE_LABELS, "2005(1)2026"), + "central_city": (_CENTRAL_CITY_LABELS, "2000(1)2026"), +} + + +def build_dicionario() -> list[dict]: + """Build the dictionary rows from the columns each table actually carries. + + Derived from the architecture CSVs rather than from a hand-kept list, so a + column flagged ``covered_by_dictionary`` can never end up without labels. + + Returns: + Dictionary rows, one per table, column and coded value. + + Raises: + KeyError: If a column is flagged as dictionary-covered but has no + label map above. + """ + rows: list[dict] = [] + for table in constants.TABLES.value: + if table == "dicionario": + continue + for column in read_arch(table): + if column["covered_by_dictionary"] != "yes": + continue + labels, coverage = _DICTIONARY[column["name"]] + for key, value in labels.items(): + rows.append( + { + "id_tabela": table, + "nome_coluna": column["name"], + "chave": key, + "cobertura_temporal": coverage + or column["temporal_coverage"], + "valor": value, + } + ) + return rows + + +# ── driver ────────────────────────────────────────────────────────────────── + +FLUSH_ROWS = 400_000 + + +def clean_all( + input_dir: Path, output_dir: Path, *, only: set[str] | None = None +) -> dict[str, int]: + """Parse every downloaded BPS file into the partitioned Parquet tables. + + Files are grouped by destination table and year so each year partition is + written in a few flushes, keeping peak memory to one batch rather than one + table. + + Args: + input_dir: Directory holding the downloaded ``.txt`` files. + output_dir: Root directory to write ``/year=/`` under. + only: Optional set of table slugs to restrict the run to. + + Returns: + Rows written per table. + """ + grouped: dict[tuple[str, int], list[Path]] = {} + for path in sorted(input_dir.rglob("*.txt")): + level, periodicity, year, _ = parse_filename(path.name) + with path.open("rb") as fh: + header = fh.readline().decode("latin-1") + # Only the metropolitan files need the full header to be routed, and + # only to tell the MSA era from the CBSA era. + fields = ["cbsa"] if "CBSA" in header else [] + table = target_table(level, periodicity, fields) + if only and table not in only: + continue + grouped.setdefault((table, year), []).append(path) + + # A rerun must not leave part files behind from a previous run that + # produced more of them, or the stale rows are read back as real data. + for table in {t for t, _ in grouped}: + shutil.rmtree(output_dir / table, ignore_errors=True) + + totals: dict[str, int] = {} + dropped: dict[str, int] = {} + for (table, year), paths in sorted(grouped.items()): + buffer: list[dict] = [] + part = 0 + written = 0 + for path in paths: + level, periodicity, file_year, month = parse_filename(path.name) + fields, rows = read_file(path) + for rec in melt_rows( + fields, rows, level=level, year=file_year, month=month + ): + row = to_columns(rec, table) + if keep_row(row, table): + buffer.append(row) + else: + dropped[table] = dropped.get(table, 0) + 1 + if len(buffer) >= FLUSH_ROWS: + written += write_parquet(buffer, table, output_dir, part) + part += 1 + buffer = [] + written += write_parquet(buffer, table, output_dir, part) + totals[table] = totals.get(table, 0) + written + log.info(f"{table} {year}: {written:,} rows") + + if not only or "dicionario" in (only or set()): + rows = build_dicionario() + shutil.rmtree(output_dir / "dicionario", ignore_errors=True) + order, _, strings = _schemas("dicionario") + frame = pd.DataFrame.from_records(rows)[order] + target = output_dir / "dicionario" + target.mkdir(parents=True, exist_ok=True) + at = pa.Table.from_pandas(frame, schema=strings, preserve_index=False) + pq.write_table(at, target / "data.parquet", compression="snappy") + totals["dicionario"] = len(rows) + for table, count in sorted(dropped.items()): + log.info(f"{table}: dropped {count:,} rows with no geography code") + return totals From 6480ce17074dff37c3ca0393a377a559bb15a5c0 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 9 Sep 2026 16:35:11 +1000 Subject: [PATCH 2/7] feat(us_census_bps): add dbt models, tests and the staging upload Eleven models over the all-STRING staging tables, partitioned on year and clustered on the geography and structure type. dbt run and dbt test both pass against dev: 11 models, 80 tests, no failures. The uploader streams each Parquet file to GCS and defines the staging table as an external Parquet table over the prefix. That reproduces exactly what basedosdados.Table.create produces, without its pandas step, which reads the whole file into memory and would need tens of gigabytes for the 19-million-row place table. Keeping it all-STRING also matches what the recurring pipeline's upload_to_gcs will later write over the same prefix. Referential allowances are measured against the US geography directory, not guessed, and each is explained in the model description: - state_id matches completely, so it takes the plain relationships test. It only does because the survey identified the territories by their old Census codes until 2021 and by FIPS from 2022, which the transform now normalises: Puerto Rico is 43 through 2021 and 72 after, and geography_id still carries the code as published. - county_id is 0.4 to 1.4 percent unmatched, almost all Connecticut's eight counties replaced by planning regions in 2022 and the split Alaska census areas. - place_id is 0.3 percent unmatched, from place code drift between decennial vintages against the 2020-vintage directory. - cbsa_id is 2.7 percent unmatched, from CBSA redelineations against the 2023-vintage directory. --- dbt_project.yml | 3 + .../code/architecture/permit_state_annual.csv | 2 +- .../architecture/permit_state_monthly.csv | 2 +- .../us_census_bps/code/build_architecture.py | 7 +- models/us_census_bps/code/build_dbt.py | 343 +++++++ models/us_census_bps/code/upload.py | 148 +++ models/us_census_bps/schema.yml | 919 ++++++++++++++++++ .../us_census_bps__dicionario.sql | 16 + .../us_census_bps__permit_cbsa_annual.sql | 30 + .../us_census_bps__permit_cbsa_monthly.sql | 31 + .../us_census_bps__permit_county_annual.sql | 30 + .../us_census_bps__permit_county_monthly.sql | 31 + .../us_census_bps__permit_msa_annual.sql | 28 + .../us_census_bps__permit_msa_monthly.sql | 29 + .../us_census_bps__permit_place_annual.sql | 43 + .../us_census_bps__permit_place_monthly.sql | 41 + .../us_census_bps__permit_state_annual.sql | 31 + .../us_census_bps__permit_state_monthly.sql | 32 + pipelines/datasets/us_census_bps/utils.py | 7 + 19 files changed, 1770 insertions(+), 3 deletions(-) create mode 100644 models/us_census_bps/code/build_dbt.py create mode 100644 models/us_census_bps/code/upload.py create mode 100644 models/us_census_bps/schema.yml create mode 100644 models/us_census_bps/us_census_bps__dicionario.sql create mode 100644 models/us_census_bps/us_census_bps__permit_cbsa_annual.sql create mode 100644 models/us_census_bps/us_census_bps__permit_cbsa_monthly.sql create mode 100644 models/us_census_bps/us_census_bps__permit_county_annual.sql create mode 100644 models/us_census_bps/us_census_bps__permit_county_monthly.sql create mode 100644 models/us_census_bps/us_census_bps__permit_msa_annual.sql create mode 100644 models/us_census_bps/us_census_bps__permit_msa_monthly.sql create mode 100644 models/us_census_bps/us_census_bps__permit_place_annual.sql create mode 100644 models/us_census_bps/us_census_bps__permit_place_monthly.sql create mode 100644 models/us_census_bps/us_census_bps__permit_state_annual.sql create mode 100644 models/us_census_bps/us_census_bps__permit_state_monthly.sql diff --git a/dbt_project.yml b/dbt_project.yml index 3a0a21876e..cb8b32f1aa 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -183,6 +183,9 @@ models: us_bls_oes: +materialized: table +schema: us_bls_oes + us_census_bps: + +materialized: table + +schema: us_census_bps us_fdic_bankfind: +materialized: table +schema: us_fdic_bankfind diff --git a/models/us_census_bps/code/architecture/permit_state_annual.csv b/models/us_census_bps/code/architecture/permit_state_annual.csv index 40844c7bc4..f6bea8e677 100644 --- a/models/us_census_bps/code/architecture/permit_state_annual.csv +++ b/models/us_census_bps/code/architecture/permit_state_annual.csv @@ -2,7 +2,7 @@ name,bigquery_type,description,temporal_coverage,covered_by_dictionary,directory year,INT64,Reference year of the survey period,1980(1)2025,no,br_bd_diretorios_data_tempo.ano:ano,year,no,Partition column,Survey Date geography_level,STRING,Level of the geography the row describes,,yes,,,no,"The published state file also carries the national, regional and divisional totals. Filter on this column before aggregating, or the same permits are counted several times",FIPS State geography_id,STRING,"Code identifying the geography, as published",,no,,,no,"Two-digit FIPS state code for states and territories, R1-R4 for regions, D1-D9 for divisions, US for the national total",FIPS State -state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,"Null on the national, regional and divisional total rows",FIPS State +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,"Null on the national, regional and divisional total rows. Until 2021 the survey identified the territories by their old Census codes rather than by FIPS, so Puerto Rico is published as 43 through 2021 and as 72 from 2022; this column carries the FIPS code throughout, and geography_id carries the code as published",FIPS State region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block diff --git a/models/us_census_bps/code/architecture/permit_state_monthly.csv b/models/us_census_bps/code/architecture/permit_state_monthly.csv index 2282a11279..36163b3e16 100644 --- a/models/us_census_bps/code/architecture/permit_state_monthly.csv +++ b/models/us_census_bps/code/architecture/permit_state_monthly.csv @@ -3,7 +3,7 @@ year,INT64,Reference year of the survey period,1988(1)2026,no,br_bd_diretorios_d month,INT64,"Reference month of the survey period, from 1 to 12",1988(1)2026,no,br_bd_diretorios_data_tempo.mes:mes,month,no,,Survey Date geography_level,STRING,Level of the geography the row describes,,yes,,,no,"The published state file also carries the national, regional and divisional totals. Filter on this column before aggregating, or the same permits are counted several times",FIPS State geography_id,STRING,"Code identifying the geography, as published",,no,,,no,"Two-digit FIPS state code for states and territories, R1-R4 for regions, D1-D9 for divisions, US for the national total",FIPS State -state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,"Null on the national, regional and divisional total rows",FIPS State +state_id,STRING,Two-digit FIPS state code,,no,diretorios_us.state:id_state,,no,"Null on the national, regional and divisional total rows. Until 2021 the survey identified the territories by their old Census codes rather than by FIPS, so Puerto Rico is published as 43 through 2021 and as 72 from 2022; this column carries the FIPS code throughout, and geography_id carries the code as published",FIPS State region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block diff --git a/models/us_census_bps/code/build_architecture.py b/models/us_census_bps/code/build_architecture.py index 05babbcf35..81def2ec33 100644 --- a/models/us_census_bps/code/build_architecture.py +++ b/models/us_census_bps/code/build_architecture.py @@ -226,7 +226,12 @@ def state_table(monthly: bool) -> list[list[str]]: "Two-digit FIPS state code", directory=DIR_STATE, observations=( - "Null on the national, regional and divisional total rows" + "Null on the national, regional and divisional total rows. " + "Until 2021 the survey identified the territories by their " + "old Census codes rather than by FIPS, so Puerto Rico is " + "published as 43 through 2021 and as 72 from 2022; this " + "column carries the FIPS code throughout, and geography_id " + "carries the code as published" ), original="FIPS State", ), diff --git a/models/us_census_bps/code/build_dbt.py b/models/us_census_bps/code/build_dbt.py new file mode 100644 index 0000000000..c50202d52d --- /dev/null +++ b/models/us_census_bps/code/build_dbt.py @@ -0,0 +1,343 @@ +"""Emit the dbt models and schema.yml for us_census_bps. + +Column order and types come from the architecture CSVs, so the models cannot +drift from the schema the cleaning transform writes. The referential tests' +allowances are the shares measured against the US geography directory in +BigQuery, not guesses; each is explained in the model description. +""" + +from __future__ import annotations + +import csv +import sys +from pathlib import Path + +MODEL_DIR = Path(__file__).resolve().parent.parent +ARCH = MODEL_DIR / "code" / "architecture" +DATASET = "us_census_bps" + +# table -> (partition start, partition end, cluster columns) +PARTITION = { + "permit_place_monthly": (1988, 2031, ["state_id", "structure_type"]), + "permit_place_annual": (1980, 2030, ["state_id", "structure_type"]), + "permit_county_monthly": (2000, 2031, ["state_id", "structure_type"]), + "permit_county_annual": (1990, 2030, ["state_id", "structure_type"]), + "permit_cbsa_monthly": (2004, 2031, ["cbsa_id", "structure_type"]), + "permit_cbsa_annual": (2003, 2030, ["cbsa_id", "structure_type"]), + "permit_msa_monthly": (1988, 2008, ["msa_cmsa_id", "structure_type"]), + "permit_msa_annual": (1980, 2007, ["msa_cmsa_id", "structure_type"]), + "permit_state_monthly": ( + 1988, + 2031, + ["geography_level", "structure_type"], + ), + "permit_state_annual": (1980, 2030, ["geography_level", "structure_type"]), +} + +KEY = { + "permit_place_monthly": ["year", "month", "state_id", "permit_office_id"], + "permit_place_annual": ["year", "state_id", "permit_office_id"], + "permit_county_monthly": ["year", "month", "state_id", "county_id"], + "permit_county_annual": ["year", "state_id", "county_id"], + "permit_cbsa_monthly": ["year", "month", "cbsa_id"], + "permit_cbsa_annual": ["year", "cbsa_id"], + "permit_msa_monthly": ["year", "month", "msa_cmsa_id", "pmsa_id"], + "permit_msa_annual": ["year", "msa_cmsa_id", "pmsa_id"], + "permit_state_monthly": ["year", "month", "geography_id"], + "permit_state_annual": ["year", "geography_id"], +} + +NOT_NULL = { + "permit_place_monthly": ["year", "month", "state_id", "permit_office_id"], + "permit_place_annual": ["year", "state_id", "permit_office_id"], + "permit_county_monthly": ["year", "month", "state_id"], + "permit_county_annual": ["year", "state_id"], + "permit_cbsa_monthly": ["year", "month", "cbsa_id"], + "permit_cbsa_annual": ["year", "cbsa_id"], + "permit_msa_monthly": ["year", "month", "msa_cmsa_id"], + "permit_msa_annual": ["year", "msa_cmsa_id"], + "permit_state_monthly": [ + "year", + "month", + "geography_id", + "geography_level", + ], + "permit_state_annual": ["year", "geography_id", "geography_level"], +} + +# Columns below the 5 percent non-null floor, measured in BigQuery. +SPARSE = { + "permit_place_monthly": ["central_city"], + "permit_place_annual": ["central_city"], +} + +DIRECTORY = "br_bd_diretorios_us__" + +# column -> (directory model, key column). state_id matches completely, so it +# takes the plain relationships test; the rest carry a measured allowance. +STRICT_FK = {"state_id": ("state", "id_state")} +LOOSE_FK = { + "county_id": ("county", "id_county", 0.02), + "place_id": ("place", "id_place", 0.02), + "cbsa_id": ("cbsa_2023", "id_cbsa", 0.05), +} + +DESCRIPTIONS = { + "permit_place_monthly": ( + "Monthly counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, for " + "each permit-issuing place, from January 1988. One row per place, " + "month and structure type. Figures include imputed values for " + "non-responding permit offices; source_code marks how each office's " + "month was obtained, and the monthly place file carries no separate " + "reported-only block. The monthly universe was a sample of roughly " + "9,000 permit offices until it was widened to the full universe of " + "about 20,000 in 2024, so counts of offices are not comparable across " + "that break; the annual table covers the full universe throughout." + ), + "permit_place_annual": ( + "Annual counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, for " + "each permit-issuing place, from 1980. One row per place, year and " + "structure type, for the full universe of about 20,000 permit " + "offices. Both the estimate including imputation for non-responding " + "offices and the reported-only figures are given; use the former by " + "default." + ), + "permit_county_monthly": ( + "Monthly counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "county, from January 2000. One row per county, month and structure " + "type. Both the estimate including imputation for non-responding " + "permit offices and the reported-only figures are given; use the " + "former by default." + ), + "permit_county_annual": ( + "Annual counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "county, from 1990. One row per county, year and structure type. Both " + "the estimate including imputation for non-responding permit offices " + "and the reported-only figures are given; use the former by default." + ), + "permit_cbsa_monthly": ( + "Monthly counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "Core Based Statistical Area, from January 2004. One row per area, " + "month and structure type. Micropolitan areas were added to the " + "series in January 2024. For 1988 to 2003 the survey used the older " + "MSA and PMSA definitions, which are a different geography and are " + "held in permit_msa_monthly." + ), + "permit_cbsa_annual": ( + "Annual counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "Core Based Statistical Area, from 2003. One row per area, year and " + "structure type. Micropolitan areas were added to the series in 2024. " + "For 1980 to 2002 the survey used the older MSA and PMSA definitions, " + "which are a different geography and are held in permit_msa_annual." + ), + "permit_msa_monthly": ( + "Monthly counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "Metropolitan Statistical Area, from January 1988 to December 2003. " + "One row per area, month and structure type. This table uses the " + "pre-2004 MSA and PMSA code system, which was superseded by CBSA " + "codes rather than renamed, so it does not join to permit_cbsa_" + "monthly or to the CBSA directory." + ), + "permit_msa_annual": ( + "Annual counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "Metropolitan Statistical Area, from 1980 to 2002. One row per area, " + "year and structure type. This table uses the pre-2004 MSA and PMSA " + "code system, which was superseded by CBSA codes rather than renamed, " + "so it does not join to permit_cbsa_annual or to the CBSA directory." + ), + "permit_state_monthly": ( + "Monthly counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "state, from January 1988. The published file also carries the " + "national, regional and divisional totals, so filter on " + "geography_level before aggregating or the same permits are counted " + "several times. One row per geography, month and structure type." + ), + "permit_state_annual": ( + "Annual counts of new privately-owned residential buildings, housing " + "units and construction valuation authorized by building permits, by " + "state, from 1980. The published file also carries the national, " + "regional and divisional totals, so filter on geography_level before " + "aggregating or the same permits are counted several times. One row " + "per geography, year and structure type." + ), + "dicionario": ( + "Dictionary of the coded values used across the us_census_bps tables, " + "one row per table, column and key." + ), +} + +EXCEPTIONS = { + "county_id": ( + "Exception: the county_id referential test allows up to 2 percent " + "unmatched because the survey reports historical counties that the " + "current FIPS directory no longer lists, chiefly Connecticut's eight " + "counties replaced by planning regions in 2022 and the Alaska census " + "areas that have since been split." + ), + "place_id": ( + "Exception: the place_id referential test allows up to 2 percent " + "unmatched because place FIPS codes drift between decennial " + "vintages while the directory holds the 2020 vintage. Measured " + "unmatched share is 0.3 percent." + ), + "cbsa_id": ( + "Exception: the cbsa_id referential test allows up to 5 percent " + "unmatched because CBSA delineations are revised every few years " + "while the directory holds the 2023 vintage; areas such as " + "Los Angeles-Long Beach-Santa Ana (31100) were redelineated. " + "Measured unmatched share is 2.7 percent." + ), +} + + +def read_arch(table: str) -> list[dict]: + """Read one table's architecture CSV.""" + with (ARCH / f"{table}.csv").open(encoding="utf-8") as fh: + return list(csv.DictReader(fh)) + + +def wrap(text: str, indent: str, width: int = 79) -> list[str]: + """Wrap text to the repo's line length at a given indent.""" + words, lines, line = text.split(), [], indent + for word in words: + if len(line) + len(word) + 1 > width and line.strip(): + lines.append(line.rstrip()) + line = indent + word + else: + line = f"{line} {word}" if line.strip() else indent + word + if line.strip(): + lines.append(line.rstrip()) + return lines + + +def build_sql(table: str) -> str: + """Render one dbt model.""" + arch = read_arch(table) + casts = ",\n".join( + f" safe_cast({c['name']} as {c['bigquery_type'].lower()}) " + f"{c['name']}" + for c in arch + ) + if table == "dicionario": + config = ( + ' schema="us_census_bps",\n' + ' alias="dicionario",\n' + ' materialized="table",\n' + ) + else: + start, end, cluster = PARTITION[table] + cluster_sql = ", ".join(f'"{c}"' for c in cluster) + config = ( + ' schema="us_census_bps",\n' + f' alias="{table}",\n' + ' materialized="table",\n' + " partition_by={\n" + ' "field": "year",\n' + ' "data_type": "int64",\n' + f' "range": {{"start": {start}, "end": {end}, ' + '"interval": 1},\n' + " },\n" + f" cluster_by=[{cluster_sql}],\n" + ) + return ( + "{{\n config(\n" + + config + + " )\n}}\n\n\nselect\n" + + casts + + f'\nfrom {{{{ set_datalake_project("{DATASET}_staging.{table}") }}}}' + " as t\n" + ) + + +def build_schema() -> str: + """Render schema.yml for every model.""" + out = ["---", "version: 2", "models:"] + for table in [*PARTITION, "dicionario"]: + arch = read_arch(table) + names = [c["name"] for c in arch] + description = DESCRIPTIONS[table] + for column, note in EXCEPTIONS.items(): + if column in names and column in LOOSE_FK: + description = f"{description} {note}" + out.append(f" - name: {DATASET}__{table}") + out.append(" description: >-") + out += wrap(description, " " * 6) + if table != "dicionario": + out.append(" tests:") + out.append(" - dbt_utils.unique_combination_of_columns:") + out.append(" combination_of_columns:") + for column in [*KEY[table], "structure_type"]: + out.append(f" - {column}") + out.append(" - not_null_proportion_multiple_columns:") + out.append(" at_least: 0.05") + if table in SPARSE: + out.append(" ignore_values:") + for column in SPARSE[table]: + out.append(f" - {column}") + out.append(" columns:") + for column in arch: + name = column["name"] + out.append(f" - name: {name}") + out.append(" description: >-") + out += wrap(column["description"], " " * 10) + tests: list[str] = [] + if name in NOT_NULL.get(table, []): + tests.append(" - not_null") + if name == "year" and table != "dicionario": + tests += [ + " - relationships:", + " to: ref('br_bd_diretorios_data_tempo__ano')", + " field: ano.ano", + ] + if name == "month": + tests += [ + " - relationships:", + " to: ref('br_bd_diretorios_data_tempo__mes')", + " field: mes.mes", + ] + if name in STRICT_FK: + model, field = STRICT_FK[name] + tests += [ + " - relationships:", + f" to: ref('{DIRECTORY}{model}')", + f" field: {field}", + ] + if name in LOOSE_FK: + model, field, allowed = LOOSE_FK[name] + tests += [ + " - custom_relationships:", + f" to: ref('{DIRECTORY}{model}')", + f" field: {field}", + # The macro only filters nulls when ignore_values is set, + # and these columns are null by design for records the + # survey does not tie to that geography. + " ignore_values: ['']", + f" proportion_allowed_failures: {allowed}", + ] + if tests: + out.append(" tests:") + out += tests + return "\n".join(out) + "\n" + + +def main() -> int: + for table in [*PARTITION, "dicionario"]: + path = MODEL_DIR / f"{DATASET}__{table}.sql" + path.write_text(build_sql(table), encoding="utf-8") + print(f"wrote {path.name}") + (MODEL_DIR / "schema.yml").write_text(build_schema(), encoding="utf-8") + print("wrote schema.yml") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/models/us_census_bps/code/upload.py b/models/us_census_bps/code/upload.py new file mode 100644 index 0000000000..839d6bdbf4 --- /dev/null +++ b/models/us_census_bps/code/upload.py @@ -0,0 +1,148 @@ +"""Upload the cleaned us_census_bps Parquet to BigQuery staging. + +Streams each file to GCS and then defines the staging table as an EXTERNAL +Parquet table over the uploaded prefix. This reproduces exactly what +``basedosdados.Table.create`` produces — an all-STRING external table, +hive-partitioned on ``year`` — without its pandas step, which reads the whole +file into memory and would need tens of gigabytes for the 19-million-row place +table. + +All-STRING matters beyond memory: the recurring pipeline's ``upload_to_gcs`` +writes an all-STRING staging table over the same prefix, so a typed table left +here would collide with it on the first pipeline run. + +Usage: + uv run python models/us_census_bps/code/upload.py [table ...] +""" + +from __future__ import annotations + +import argparse +import os +import sys +import tomllib +from pathlib import Path + +from google.cloud import bigquery, storage +from google.oauth2 import service_account + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from pipelines.datasets.us_census_bps.constants import constants + +DATASET_ID = constants.DATASET_ID.value +PROJECT = "basedosdados-dev" +BUCKET = "basedosdados-dev" +DATA_DIR = Path( + os.environ.get( + "BPS_DATA_DIR", Path.home() / "Downloads/us_census_bps_data" + ) +) +OUTPUT = DATA_DIR / "output" + + +def clients() -> tuple[storage.Client, bigquery.Client]: + """Build GCS and BigQuery clients from the basedosdados service account.""" + config = tomllib.loads( + (Path.home() / ".basedosdados/config.toml").read_text() + ) + path = config["gcloud-projects"]["staging"]["credentials_path"] + creds = service_account.Credentials.from_service_account_file(path) + return ( + storage.Client(credentials=creds, project=PROJECT), + bigquery.Client(credentials=creds, project=PROJECT), + ) + + +def upload_table( + gcs: storage.Client, bq: bigquery.Client, table: str, expected: int +) -> int: + """Stream one table's Parquet to GCS and redefine its staging table. + + Args: + gcs: Storage client. + bq: BigQuery client. + table: Table slug. + expected: Row count the cleaning step reported, asserted after load. + + Returns: + The row count read back from BigQuery. + + Raises: + FileNotFoundError: If the table has no local output. + ValueError: If the loaded row count differs from ``expected``. + """ + local = OUTPUT / table + if not local.exists(): + raise FileNotFoundError(local) + files = sorted(local.rglob("*.parquet")) + prefix = f"staging/{DATASET_ID}/{table}/" + # The bucket is requester-pays, so every call needs a billing project. + bucket = gcs.bucket(BUCKET, user_project=PROJECT) + + stale = list(bucket.list_blobs(prefix=prefix)) + for blob in stale: + blob.delete() + print(f" cleared {len(stale)} stale objects under {prefix}") + + for i, path in enumerate(files, start=1): + blob = bucket.blob(prefix + str(path.relative_to(local))) + blob.chunk_size = 64 * 1024 * 1024 + blob.upload_from_filename(str(path)) + if i % 25 == 0 or i == len(files): + print(f" uploaded {i}/{len(files)}", flush=True) + + ref = f"{PROJECT}.{DATASET_ID}_staging.{table}" + bq.query(f"drop table if exists `{ref}`").result() + config = bigquery.ExternalConfig("PARQUET") + config.source_uris = [f"gs://{BUCKET}/{prefix}*"] + if table != "dicionario": + hive = bigquery.HivePartitioningOptions() + hive.mode = "STRINGS" + hive.source_uri_prefix = f"gs://{BUCKET}/{prefix}" + config.hive_partitioning = hive + definition = bigquery.Table(ref) + definition.external_data_configuration = config + bq.create_table(definition) + + row = next(iter(bq.query(f"select count(*) as n from `{ref}`").result())) + if row.n != expected: + raise ValueError( + f"{table}: loaded {row.n:,} rows, expected {expected:,}" + ) + print(f" {table}: {row.n:,} rows — OK") + return int(row.n) + + +# Row counts reported by models/us_census_bps/code/clean.py, smallest first. +TABLES = [ + ("dicionario", 144), + ("permit_state_annual", 12_136), + ("permit_state_monthly", 120_380), + ("permit_msa_annual", 31_428), + ("permit_msa_monthly", 266_304), + ("permit_cbsa_annual", 38_720), + ("permit_cbsa_monthly", 466_820), + ("permit_county_annual", 435_772), + ("permit_county_monthly", 1_510_676), + ("permit_place_annual", 3_515_268), + ("permit_place_monthly", 19_283_716), +] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("tables", nargs="*") + args = parser.parse_args() + gcs, bq = clients() + bq.create_dataset(f"{PROJECT}.{DATASET_ID}_staging", exists_ok=True) + todo = [t for t in TABLES if not args.tables or t[0] in args.tables] + for table, expected in todo: + print(f"=== {table} ===", flush=True) + upload_table(gcs, bq, table, expected) + print("ALL TABLES UPLOADED") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/models/us_census_bps/schema.yml b/models/us_census_bps/schema.yml new file mode 100644 index 0000000000..d4dd96fe41 --- /dev/null +++ b/models/us_census_bps/schema.yml @@ -0,0 +1,919 @@ +--- +version: 2 +models: + - name: us_census_bps__permit_place_monthly + description: >- + Monthly counts of new privately-owned residential buildings, housing + units and construction valuation authorized by building permits, for each + permit-issuing place, from January 1988. One row per place, month and + structure type. Figures include imputed values for non-responding permit + offices; source_code marks how each office's month was obtained, and the + monthly place file carries no separate reported-only block. The monthly + universe was a sample of roughly 9,000 permit offices until it was + widened to the full universe of about 20,000 in 2024, so counts of + offices are not comparable across that break; the annual table covers the + full universe throughout. Exception: the county_id referential test + allows up to 2 percent unmatched because the survey reports historical + counties that the current FIPS directory no longer lists, chiefly + Connecticut's eight counties replaced by planning regions in 2022 and the + Alaska census areas that have since been split. Exception: the place_id + referential test allows up to 2 percent unmatched because place FIPS + codes drift between decennial vintages while the directory holds the 2020 + vintage. Measured unmatched share is 0.3 percent. Exception: the cbsa_id + referential test allows up to 5 percent unmatched because CBSA + delineations are revised every few years while the directory holds the + 2023 vintage; areas such as Los Angeles-Long Beach-Santa Ana (31100) were + redelineated. Measured unmatched share is 2.7 percent. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - year + - month + - state_id + - permit_office_id + - structure_type + - not_null_proportion_multiple_columns: + at_least: 0.05 + ignore_values: [central_city] + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: month + description: >- + Reference month of the survey period, from 1 to 12 + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__mes') + field: mes.mes + - name: state_id + description: >- + Two-digit FIPS state code + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_us__state') + field: id_state + - name: county_id + description: >- + Five-digit FIPS county code, state code followed by county code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__county') + field: id_county + ignore_values: [''] + proportion_allowed_failures: 0.02 + - name: place_id + description: >- + Seven-digit FIPS place code, state code followed by place code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__place') + field: id_place + ignore_values: [''] + proportion_allowed_failures: 0.02 + - name: mcd_id + description: >- + Seven-digit FIPS minor civil division code, state code followed by + MCD code + - name: cbsa_id + description: >- + Five-digit Core Based Statistical Area code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__cbsa_2023') + field: id_cbsa + ignore_values: [''] + proportion_allowed_failures: 0.05 + - name: csa_id + description: >- + Three-digit Combined Statistical Area code + - name: msa_cmsa_id + description: >- + Four-digit Metropolitan or Consolidated Metropolitan Statistical Area + code + - name: pmsa_id + description: >- + Four-digit Primary Metropolitan Statistical Area code + - name: permit_office_id + description: >- + Six-digit Building Permit Survey identifier of the permit-issuing + office, unique within a state and survey period + tests: [not_null] + - name: census_place_id + description: >- + Four-digit Census place code + - name: region_id + description: >- + Census region code, from 1 to 4 + - name: division_id + description: >- + Census division code, from 1 to 9 + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: place_name + description: >- + Name of the permit-issuing place or office, as published + - name: central_city + description: >- + Whether the place is a central city of a metropolitan area + - name: footnote_code + description: >- + Whether the place carries an explanatory footnote in the source + release + - name: zip_code + description: >- + ZIP code of the permit office or official + - name: population + description: >- + Population of the place as carried in the source file + - name: source_code + description: >- + How the data for the permit office was obtained for the month + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: us_census_bps__permit_place_annual + description: >- + Annual counts of new privately-owned residential buildings, housing units + and construction valuation authorized by building permits, for each + permit-issuing place, from 1980. One row per place, year and structure + type, for the full universe of about 20,000 permit offices. Both the + estimate including imputation for non-responding offices and the + reported-only figures are given; use the former by default. Exception: + the county_id referential test allows up to 2 percent unmatched because + the survey reports historical counties that the current FIPS directory no + longer lists, chiefly Connecticut's eight counties replaced by planning + regions in 2022 and the Alaska census areas that have since been split. + Exception: the place_id referential test allows up to 2 percent unmatched + because place FIPS codes drift between decennial vintages while the + directory holds the 2020 vintage. Measured unmatched share is 0.3 + percent. Exception: the cbsa_id referential test allows up to 5 percent + unmatched because CBSA delineations are revised every few years while the + directory holds the 2023 vintage; areas such as Los Angeles-Long + Beach-Santa Ana (31100) were redelineated. Measured unmatched share is + 2.7 percent. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - year + - state_id + - permit_office_id + - structure_type + - not_null_proportion_multiple_columns: + at_least: 0.05 + ignore_values: [central_city] + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: state_id + description: >- + Two-digit FIPS state code + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_us__state') + field: id_state + - name: county_id + description: >- + Five-digit FIPS county code, state code followed by county code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__county') + field: id_county + ignore_values: [''] + proportion_allowed_failures: 0.02 + - name: place_id + description: >- + Seven-digit FIPS place code, state code followed by place code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__place') + field: id_place + ignore_values: [''] + proportion_allowed_failures: 0.02 + - name: mcd_id + description: >- + Seven-digit FIPS minor civil division code, state code followed by + MCD code + - name: cbsa_id + description: >- + Five-digit Core Based Statistical Area code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__cbsa_2023') + field: id_cbsa + ignore_values: [''] + proportion_allowed_failures: 0.05 + - name: csa_id + description: >- + Three-digit Combined Statistical Area code + - name: msa_cmsa_id + description: >- + Four-digit Metropolitan or Consolidated Metropolitan Statistical Area + code + - name: pmsa_id + description: >- + Four-digit Primary Metropolitan Statistical Area code + - name: permit_office_id + description: >- + Six-digit Building Permit Survey identifier of the permit-issuing + office, unique within a state and survey period + tests: [not_null] + - name: census_place_id + description: >- + Four-digit Census place code + - name: region_id + description: >- + Census region code, from 1 to 4 + - name: division_id + description: >- + Census division code, from 1 to 9 + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: place_name + description: >- + Name of the permit-issuing place or office, as published + - name: central_city + description: >- + Whether the place is a central city of a metropolitan area + - name: footnote_code + description: >- + Whether the place carries an explanatory footnote in the source + release + - name: zip_code + description: >- + ZIP code of the permit office or official + - name: population + description: >- + Population of the place as carried in the source file + - name: months_reported + description: >- + Number of months of the year for which the permit office reported + residential data + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_county_monthly + description: >- + Monthly counts of new privately-owned residential buildings, housing + units and construction valuation authorized by building permits, by + county, from January 2000. One row per county, month and structure type. + Both the estimate including imputation for non-responding permit offices + and the reported-only figures are given; use the former by default. + Exception: the county_id referential test allows up to 2 percent + unmatched because the survey reports historical counties that the current + FIPS directory no longer lists, chiefly Connecticut's eight counties + replaced by planning regions in 2022 and the Alaska census areas that + have since been split. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - year + - month + - state_id + - county_id + - structure_type + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: month + description: >- + Reference month of the survey period, from 1 to 12 + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__mes') + field: mes.mes + - name: county_id + description: >- + Five-digit FIPS county code, state code followed by county code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__county') + field: id_county + ignore_values: [''] + proportion_allowed_failures: 0.02 + - name: state_id + description: >- + Two-digit FIPS state code + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_us__state') + field: id_state + - name: region_id + description: >- + Census region code, from 1 to 4 + - name: division_id + description: >- + Census division code, from 1 to 9 + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: county_name + description: >- + Name of the county, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_county_annual + description: >- + Annual counts of new privately-owned residential buildings, housing units + and construction valuation authorized by building permits, by county, + from 1990. One row per county, year and structure type. Both the estimate + including imputation for non-responding permit offices and the + reported-only figures are given; use the former by default. Exception: + the county_id referential test allows up to 2 percent unmatched because + the survey reports historical counties that the current FIPS directory no + longer lists, chiefly Connecticut's eight counties replaced by planning + regions in 2022 and the Alaska census areas that have since been split. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [year, state_id, county_id, structure_type] + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: county_id + description: >- + Five-digit FIPS county code, state code followed by county code + tests: + - custom_relationships: + to: ref('br_bd_diretorios_us__county') + field: id_county + ignore_values: [''] + proportion_allowed_failures: 0.02 + - name: state_id + description: >- + Two-digit FIPS state code + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_us__state') + field: id_state + - name: region_id + description: >- + Census region code, from 1 to 4 + - name: division_id + description: >- + Census division code, from 1 to 9 + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: county_name + description: >- + Name of the county, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_cbsa_monthly + description: >- + Monthly counts of new privately-owned residential buildings, housing + units and construction valuation authorized by building permits, by Core + Based Statistical Area, from January 2004. One row per area, month and + structure type. Micropolitan areas were added to the series in January + 2024. For 1988 to 2003 the survey used the older MSA and PMSA + definitions, which are a different geography and are held in + permit_msa_monthly. Exception: the cbsa_id referential test allows up to + 5 percent unmatched because CBSA delineations are revised every few years + while the directory holds the 2023 vintage; areas such as Los + Angeles-Long Beach-Santa Ana (31100) were redelineated. Measured + unmatched share is 2.7 percent. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [year, month, cbsa_id, structure_type] + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: month + description: >- + Reference month of the survey period, from 1 to 12 + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__mes') + field: mes.mes + - name: cbsa_id + description: >- + Five-digit Core Based Statistical Area code + tests: + - not_null + - custom_relationships: + to: ref('br_bd_diretorios_us__cbsa_2023') + field: id_cbsa + ignore_values: [''] + proportion_allowed_failures: 0.05 + - name: csa_id + description: >- + Three-digit Combined Statistical Area code the CBSA belongs to + - name: cbsa_type + description: >- + Header code distinguishing metropolitan from micropolitan areas + - name: full_monthly_coverage + description: >- + Whether the area is completely covered by monthly reporting + permit-issuing places + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: cbsa_name + description: >- + Name of the Core Based Statistical Area, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_cbsa_annual + description: >- + Annual counts of new privately-owned residential buildings, housing units + and construction valuation authorized by building permits, by Core Based + Statistical Area, from 2003. One row per area, year and structure type. + Micropolitan areas were added to the series in 2024. For 1980 to 2002 the + survey used the older MSA and PMSA definitions, which are a different + geography and are held in permit_msa_annual. Exception: the cbsa_id + referential test allows up to 5 percent unmatched because CBSA + delineations are revised every few years while the directory holds the + 2023 vintage; areas such as Los Angeles-Long Beach-Santa Ana (31100) were + redelineated. Measured unmatched share is 2.7 percent. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [year, cbsa_id, structure_type] + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: cbsa_id + description: >- + Five-digit Core Based Statistical Area code + tests: + - not_null + - custom_relationships: + to: ref('br_bd_diretorios_us__cbsa_2023') + field: id_cbsa + ignore_values: [''] + proportion_allowed_failures: 0.05 + - name: csa_id + description: >- + Three-digit Combined Statistical Area code the CBSA belongs to + - name: cbsa_type + description: >- + Header code distinguishing metropolitan from micropolitan areas + - name: full_monthly_coverage + description: >- + Whether the area is completely covered by monthly reporting + permit-issuing places + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: cbsa_name + description: >- + Name of the Core Based Statistical Area, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_msa_monthly + description: >- + Monthly counts of new privately-owned residential buildings, housing + units and construction valuation authorized by building permits, by + Metropolitan Statistical Area, from January 1988 to December 2003. One + row per area, month and structure type. This table uses the pre-2004 MSA + and PMSA code system, which was superseded by CBSA codes rather than + renamed, so it does not join to permit_cbsa_monthly or to the CBSA + directory. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - year + - month + - msa_cmsa_id + - pmsa_id + - structure_type + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: month + description: >- + Reference month of the survey period, from 1 to 12 + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__mes') + field: mes.mes + - name: msa_cmsa_id + description: >- + Four-digit Metropolitan or Consolidated Metropolitan Statistical Area + code + tests: [not_null] + - name: pmsa_id + description: >- + Four-digit Primary Metropolitan Statistical Area code + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: msa_name + description: >- + Name of the metropolitan area, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_msa_annual + description: >- + Annual counts of new privately-owned residential buildings, housing units + and construction valuation authorized by building permits, by + Metropolitan Statistical Area, from 1980 to 2002. One row per area, year + and structure type. This table uses the pre-2004 MSA and PMSA code + system, which was superseded by CBSA codes rather than renamed, so it + does not join to permit_cbsa_annual or to the CBSA directory. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [year, msa_cmsa_id, pmsa_id, structure_type] + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: msa_cmsa_id + description: >- + Four-digit Metropolitan or Consolidated Metropolitan Statistical Area + code + tests: [not_null] + - name: pmsa_id + description: >- + Four-digit Primary Metropolitan Statistical Area code + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: msa_name + description: >- + Name of the metropolitan area, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_state_monthly + description: >- + Monthly counts of new privately-owned residential buildings, housing + units and construction valuation authorized by building permits, by + state, from January 1988. The published file also carries the national, + regional and divisional totals, so filter on geography_level before + aggregating or the same permits are counted several times. One row per + geography, month and structure type. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [year, month, geography_id, structure_type] + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: month + description: >- + Reference month of the survey period, from 1 to 12 + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__mes') + field: mes.mes + - name: geography_level + description: >- + Level of the geography the row describes + tests: [not_null] + - name: geography_id + description: >- + Code identifying the geography, as published + tests: [not_null] + - name: state_id + description: >- + Two-digit FIPS state code + tests: + - relationships: + to: ref('br_bd_diretorios_us__state') + field: id_state + - name: region_id + description: >- + Census region code, from 1 to 4 + - name: division_id + description: >- + Census division code, from 1 to 9 + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: geography_name + description: >- + Name of the geography, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__permit_state_annual + description: >- + Annual counts of new privately-owned residential buildings, housing units + and construction valuation authorized by building permits, by state, from + 1980. The published file also carries the national, regional and + divisional totals, so filter on geography_level before aggregating or the + same permits are counted several times. One row per geography, year and + structure type. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [year, geography_id, structure_type] + - not_null_proportion_multiple_columns: + at_least: 0.05 + columns: + - name: year + description: >- + Reference year of the survey period + tests: + - not_null + - relationships: + to: ref('br_bd_diretorios_data_tempo__ano') + field: ano.ano + - name: geography_level + description: >- + Level of the geography the row describes + tests: [not_null] + - name: geography_id + description: >- + Code identifying the geography, as published + tests: [not_null] + - name: state_id + description: >- + Two-digit FIPS state code + tests: + - relationships: + to: ref('br_bd_diretorios_us__state') + field: id_state + - name: region_id + description: >- + Census region code, from 1 to 4 + - name: division_id + description: >- + Census division code, from 1 to 9 + - name: structure_type + description: >- + Type of residential structure, by the number of housing units in the + building + - name: geography_name + description: >- + Name of the geography, as published + - name: buildings + description: >- + Number of buildings authorized by building permits + - name: units + description: >- + Number of housing units authorized by building permits + - name: valuation + description: >- + Construction valuation of the buildings authorized, in US dollars + - name: buildings_reported + description: >- + Number of buildings authorized, counting reporting permit offices + only + - name: units_reported + description: >- + Number of housing units authorized, counting reporting permit offices + only + - name: valuation_reported + description: >- + Construction valuation of the buildings authorized, in US dollars, + counting reporting permit offices only + - name: us_census_bps__dicionario + description: >- + Dictionary of the coded values used across the us_census_bps tables, one + row per table, column and key. + columns: + - name: id_tabela + description: >- + Slug of the us_census_bps table the dictionary entry describes + - name: nome_coluna + description: >- + Name of the column the dictionary entry describes + - name: chave + description: >- + Coded value (key) exactly as stored in the data + - name: cobertura_temporal + description: >- + Temporal coverage of the key + - name: valor + description: >- + Human-readable label corresponding to the coded value diff --git a/models/us_census_bps/us_census_bps__dicionario.sql b/models/us_census_bps/us_census_bps__dicionario.sql new file mode 100644 index 0000000000..22fe2d8387 --- /dev/null +++ b/models/us_census_bps/us_census_bps__dicionario.sql @@ -0,0 +1,16 @@ +{{ + config( + schema="us_census_bps", + alias="dicionario", + materialized="table", + ) +}} + + +select + safe_cast(id_tabela as string) id_tabela, + safe_cast(nome_coluna as string) nome_coluna, + safe_cast(chave as string) chave, + safe_cast(cobertura_temporal as string) cobertura_temporal, + safe_cast(valor as string) valor +from {{ set_datalake_project("us_census_bps_staging.dicionario") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_cbsa_annual.sql b/models/us_census_bps/us_census_bps__permit_cbsa_annual.sql new file mode 100644 index 0000000000..9ff60e5ed8 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_cbsa_annual.sql @@ -0,0 +1,30 @@ +{{ + config( + schema="us_census_bps", + alias="permit_cbsa_annual", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 2003, "end": 2030, "interval": 1}, + }, + cluster_by=["cbsa_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(cbsa_id as string) cbsa_id, + safe_cast(csa_id as string) csa_id, + safe_cast(cbsa_type as string) cbsa_type, + safe_cast(full_monthly_coverage as string) full_monthly_coverage, + safe_cast(structure_type as string) structure_type, + safe_cast(cbsa_name as string) cbsa_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_cbsa_annual") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_cbsa_monthly.sql b/models/us_census_bps/us_census_bps__permit_cbsa_monthly.sql new file mode 100644 index 0000000000..0a0b0f6678 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_cbsa_monthly.sql @@ -0,0 +1,31 @@ +{{ + config( + schema="us_census_bps", + alias="permit_cbsa_monthly", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 2004, "end": 2031, "interval": 1}, + }, + cluster_by=["cbsa_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(month as int64) month, + safe_cast(cbsa_id as string) cbsa_id, + safe_cast(csa_id as string) csa_id, + safe_cast(cbsa_type as string) cbsa_type, + safe_cast(full_monthly_coverage as string) full_monthly_coverage, + safe_cast(structure_type as string) structure_type, + safe_cast(cbsa_name as string) cbsa_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_cbsa_monthly") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_county_annual.sql b/models/us_census_bps/us_census_bps__permit_county_annual.sql new file mode 100644 index 0000000000..fcd6bd6142 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_county_annual.sql @@ -0,0 +1,30 @@ +{{ + config( + schema="us_census_bps", + alias="permit_county_annual", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 1990, "end": 2030, "interval": 1}, + }, + cluster_by=["state_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(county_id as string) county_id, + safe_cast(state_id as string) state_id, + safe_cast(region_id as string) region_id, + safe_cast(division_id as string) division_id, + safe_cast(structure_type as string) structure_type, + safe_cast(county_name as string) county_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_county_annual") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_county_monthly.sql b/models/us_census_bps/us_census_bps__permit_county_monthly.sql new file mode 100644 index 0000000000..80f8ff4578 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_county_monthly.sql @@ -0,0 +1,31 @@ +{{ + config( + schema="us_census_bps", + alias="permit_county_monthly", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 2000, "end": 2031, "interval": 1}, + }, + cluster_by=["state_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(month as int64) month, + safe_cast(county_id as string) county_id, + safe_cast(state_id as string) state_id, + safe_cast(region_id as string) region_id, + safe_cast(division_id as string) division_id, + safe_cast(structure_type as string) structure_type, + safe_cast(county_name as string) county_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_county_monthly") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_msa_annual.sql b/models/us_census_bps/us_census_bps__permit_msa_annual.sql new file mode 100644 index 0000000000..cf34102fed --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_msa_annual.sql @@ -0,0 +1,28 @@ +{{ + config( + schema="us_census_bps", + alias="permit_msa_annual", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 1980, "end": 2007, "interval": 1}, + }, + cluster_by=["msa_cmsa_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(msa_cmsa_id as string) msa_cmsa_id, + safe_cast(pmsa_id as string) pmsa_id, + safe_cast(structure_type as string) structure_type, + safe_cast(msa_name as string) msa_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_msa_annual") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_msa_monthly.sql b/models/us_census_bps/us_census_bps__permit_msa_monthly.sql new file mode 100644 index 0000000000..d8a7f06795 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_msa_monthly.sql @@ -0,0 +1,29 @@ +{{ + config( + schema="us_census_bps", + alias="permit_msa_monthly", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 1988, "end": 2008, "interval": 1}, + }, + cluster_by=["msa_cmsa_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(month as int64) month, + safe_cast(msa_cmsa_id as string) msa_cmsa_id, + safe_cast(pmsa_id as string) pmsa_id, + safe_cast(structure_type as string) structure_type, + safe_cast(msa_name as string) msa_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_msa_monthly") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_place_annual.sql b/models/us_census_bps/us_census_bps__permit_place_annual.sql new file mode 100644 index 0000000000..041a63b27e --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_place_annual.sql @@ -0,0 +1,43 @@ +{{ + config( + schema="us_census_bps", + alias="permit_place_annual", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 1980, "end": 2030, "interval": 1}, + }, + cluster_by=["state_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(state_id as string) state_id, + safe_cast(county_id as string) county_id, + safe_cast(place_id as string) place_id, + safe_cast(mcd_id as string) mcd_id, + safe_cast(cbsa_id as string) cbsa_id, + safe_cast(csa_id as string) csa_id, + safe_cast(msa_cmsa_id as string) msa_cmsa_id, + safe_cast(pmsa_id as string) pmsa_id, + safe_cast(permit_office_id as string) permit_office_id, + safe_cast(census_place_id as string) census_place_id, + safe_cast(region_id as string) region_id, + safe_cast(division_id as string) division_id, + safe_cast(structure_type as string) structure_type, + safe_cast(place_name as string) place_name, + safe_cast(central_city as string) central_city, + safe_cast(footnote_code as string) footnote_code, + safe_cast(zip_code as string) zip_code, + safe_cast(population as int64) population, + safe_cast(months_reported as int64) months_reported, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_place_annual") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_place_monthly.sql b/models/us_census_bps/us_census_bps__permit_place_monthly.sql new file mode 100644 index 0000000000..4d7943a4f4 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_place_monthly.sql @@ -0,0 +1,41 @@ +{{ + config( + schema="us_census_bps", + alias="permit_place_monthly", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 1988, "end": 2031, "interval": 1}, + }, + cluster_by=["state_id", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(month as int64) month, + safe_cast(state_id as string) state_id, + safe_cast(county_id as string) county_id, + safe_cast(place_id as string) place_id, + safe_cast(mcd_id as string) mcd_id, + safe_cast(cbsa_id as string) cbsa_id, + safe_cast(csa_id as string) csa_id, + safe_cast(msa_cmsa_id as string) msa_cmsa_id, + safe_cast(pmsa_id as string) pmsa_id, + safe_cast(permit_office_id as string) permit_office_id, + safe_cast(census_place_id as string) census_place_id, + safe_cast(region_id as string) region_id, + safe_cast(division_id as string) division_id, + safe_cast(structure_type as string) structure_type, + safe_cast(place_name as string) place_name, + safe_cast(central_city as string) central_city, + safe_cast(footnote_code as string) footnote_code, + safe_cast(zip_code as string) zip_code, + safe_cast(population as int64) population, + safe_cast(source_code as string) source_code, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation +from {{ set_datalake_project("us_census_bps_staging.permit_place_monthly") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_state_annual.sql b/models/us_census_bps/us_census_bps__permit_state_annual.sql new file mode 100644 index 0000000000..d30250a7c2 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_state_annual.sql @@ -0,0 +1,31 @@ +{{ + config( + schema="us_census_bps", + alias="permit_state_annual", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 1980, "end": 2030, "interval": 1}, + }, + cluster_by=["geography_level", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(geography_level as string) geography_level, + safe_cast(geography_id as string) geography_id, + safe_cast(state_id as string) state_id, + safe_cast(region_id as string) region_id, + safe_cast(division_id as string) division_id, + safe_cast(structure_type as string) structure_type, + safe_cast(geography_name as string) geography_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_state_annual") }} as t diff --git a/models/us_census_bps/us_census_bps__permit_state_monthly.sql b/models/us_census_bps/us_census_bps__permit_state_monthly.sql new file mode 100644 index 0000000000..f88415f627 --- /dev/null +++ b/models/us_census_bps/us_census_bps__permit_state_monthly.sql @@ -0,0 +1,32 @@ +{{ + config( + schema="us_census_bps", + alias="permit_state_monthly", + materialized="table", + partition_by={ + "field": "year", + "data_type": "int64", + "range": {"start": 1988, "end": 2031, "interval": 1}, + }, + cluster_by=["geography_level", "structure_type"], + ) +}} + + +select + safe_cast(year as int64) year, + safe_cast(month as int64) month, + safe_cast(geography_level as string) geography_level, + safe_cast(geography_id as string) geography_id, + safe_cast(state_id as string) state_id, + safe_cast(region_id as string) region_id, + safe_cast(division_id as string) division_id, + safe_cast(structure_type as string) structure_type, + safe_cast(geography_name as string) geography_name, + safe_cast(buildings as int64) buildings, + safe_cast(units as int64) units, + safe_cast(valuation as int64) valuation, + safe_cast(buildings_reported as int64) buildings_reported, + safe_cast(units_reported as int64) units_reported, + safe_cast(valuation_reported as int64) valuation_reported +from {{ set_datalake_project("us_census_bps_staging.permit_state_monthly") }} as t diff --git a/pipelines/datasets/us_census_bps/utils.py b/pipelines/datasets/us_census_bps/utils.py index 42819617bb..317f2cec23 100644 --- a/pipelines/datasets/us_census_bps/utils.py +++ b/pipelines/datasets/us_census_bps/utils.py @@ -433,6 +433,12 @@ def target_table(level: str, periodicity: str, fields: list[str]) -> str: _PLACE_NULL_CODES = {"00000", "99990", "99999", "0", "000"} +# Until 2021 the survey identified the territories by their old Census codes +# rather than by FIPS, so Puerto Rico appears as 43 through 2021 and as 72 +# from 2022. Normalising state_id keeps a territory on one identifier across +# the whole panel; geography_id still carries the code exactly as published. +_LEGACY_STATE_CODES = {"43": "72", "52": "78"} + def _geography_level(code: str | None) -> str | None: """Classify a state-file geography code as nation, region, division or state.""" @@ -479,6 +485,7 @@ def to_columns(rec: dict, table: str) -> dict: out["valuation_reported"] = rec.get("valuation_reported") state = _pad(rec.get("state"), 2) + state = _LEGACY_STATE_CODES.get(state or "", state) region = rec.get("region") division = rec.get("division") From de90f05290f979f7916d86d3bd4b20cd01a22a4d Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 9 Sep 2026 16:58:50 +1000 Subject: [PATCH 3/7] feat(us_census_bps): register metadata on staging and publish there Dataset building_permits_survey_bps: 11 tables, 5 raw data sources (one per geography level, so every table links exactly one), 171 columns with types, units, dictionary flags and directory links, observation levels, coverage, refresh records and per-table auxiliary-file bundles. Every write is read back and checked by verify_metadata.py, which passes. Two backend behaviours the registration has to work around: - CreateUpdateTable fails with "'TableForm' has no field named 'coverages_areas'" on any table that already has a coverage, so all table writes, including the deferred raw-source link, run before any coverage is created. - get_dataset does not return Coverage.isClosed, so the free and BD Pro coverages cannot be told apart through it. Reading them back through GraphQL instead; doing it the other way had already produced a duplicate free coverage, which was deleted. The monthly tables carry the BD Pro split the house rule calls for on any table refreshing monthly or more often: free through 2026-01, pro 2026-02 to 2026-07, on a six-month lag. Nothing is paywalled until a pipeline applies the row access policies. The annual tables and the two closed MSA series are entirely free. Auxiliary bundles hold the Census record layout for each geography level plus a README covering the citation, provenance and every transformation applied. They sit in basedosdados-dev because that is the only bucket this service account can write, and all ten URLs return HTTP 400 anonymously: both data buckets are requester-pays. That is the open defect PR #1928 fixes by moving bundles to basedosdados-public, not something specific to this dataset. New tag building-permit was created; nothing in the existing vocabulary covered the subject. --- models/us_census_bps/code/auxiliary_files.py | 216 +++++++++ models/us_census_bps/code/metadata.py | 448 +++++++++++++++++++ models/us_census_bps/code/metadata_stage2.py | 255 +++++++++++ models/us_census_bps/code/publish.py | 78 ++++ models/us_census_bps/code/verify_metadata.py | 186 ++++++++ pyproject.toml | 4 + 6 files changed, 1187 insertions(+) create mode 100644 models/us_census_bps/code/auxiliary_files.py create mode 100644 models/us_census_bps/code/metadata.py create mode 100644 models/us_census_bps/code/metadata_stage2.py create mode 100644 models/us_census_bps/code/publish.py create mode 100644 models/us_census_bps/code/verify_metadata.py diff --git a/models/us_census_bps/code/auxiliary_files.py b/models/us_census_bps/code/auxiliary_files.py new file mode 100644 index 0000000000..258c21e635 --- /dev/null +++ b/models/us_census_bps/code/auxiliary_files.py @@ -0,0 +1,216 @@ +"""Build and upload the per-table auxiliary-file bundles for us_census_bps. + +Each bundle holds the Census record-layout document for that geography level +plus a README naming the citation, provenance and download date. + +The bundles go to ``basedosdados-dev`` because that is the only bucket the +onboarding service account can write. Both data buckets are requester-pays, so +the published URLs return HTTP 400 to an anonymous reader; the fix is the +pending move to ``gs://basedosdados-public``, not a per-dataset workaround. +This script fetches every URL it registers with no credentials and reports the +status it actually gets. +""" + +from __future__ import annotations + +import datetime +import os +import sys +import tomllib +import urllib.request +import zipfile +from pathlib import Path + +from google.cloud import storage +from google.oauth2 import service_account + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +from pipelines.datasets.us_census_bps.constants import constants + +DATASET_ID = constants.DATASET_ID.value +BUCKET = "basedosdados-dev" +DOC_BASE = constants.BASE_URL.value + "Documentation/" +DATA_DIR = Path( + os.environ.get( + "BPS_DATA_DIR", Path.home() / "Downloads/us_census_bps_data" + ) +) +WORK = DATA_DIR / "auxiliary" + +CITATION = ( + "U.S. Census Bureau, Building Permits Survey. " + "https://www.census.gov/construction/bps/" +) + +# table group -> (layout document, human name, what the document covers) +DOCS = { + ("permit_place_monthly", "permit_place_annual"): ( + "placeasc.pdf", + "place_record_layout.pdf", + "Record layout of the place-level ASCII files, including the field " + "order for each of the layouts used since 1988, the source codes and " + "the footnotes appended to permit office names.", + ), + ("permit_county_monthly", "permit_county_annual"): ( + "cntyasc.pdf", + "county_record_layout.pdf", + "Record layout of the county-level ASCII files, including the Census " + "region and division codes.", + ), + ("permit_state_monthly", "permit_state_annual"): ( + "stateasc.pdf", + "state_record_layout.pdf", + "Record layout of the state-level ASCII files. Note that the source " + "publishes valuation in thousands of dollars at this level; the Data " + "Basis tables carry dollars.", + ), + ("permit_cbsa_monthly", "permit_cbsa_annual"): ( + "cbsaasc.pdf", + "cbsa_record_layout.pdf", + "Record layout of the Core Based Statistical Area files, including " + "the header code that distinguishes metropolitan from micropolitan " + "areas from January 2024. Valuation is published in thousands of " + "dollars; the Data Basis tables carry dollars.", + ), + ("permit_msa_monthly", "permit_msa_annual"): ( + "msaasc.pdf", + "msa_record_layout.pdf", + "Record layout of the pre-2004 Metropolitan Statistical Area files, " + "including the header coverage code. Valuation is published in " + "thousands of dollars; the Data Basis tables carry dollars.", + ), +} + +LINK_ONLY = [ + ( + "Importing Permits Files to Microsoft Excel", + DOC_BASE + "Importing%20Permits%20Files%20to%20Microsoft%20Excel.docx", + "Publisher guide to opening the raw ASCII files in Excel. Not needed " + "to use the Data Basis tables, which are already parsed.", + ), + ( + "Building Permits Survey landing page", + "https://www.census.gov/construction/bps/", + "Survey methodology, release schedule and the current release.", + ), + ( + "Compiled data file (all geography levels in one CSV)", + constants.BASE_URL.value + "Master%20Data%20Set/", + "The Census Bureau's own combined extract. Data Basis builds from the " + "per-level ASCII files instead, which go back further.", + ), +] + + +def readme( + tables: tuple[str, ...], doc_name: str, covers: str, remote: str +) -> str: + """Render the bundle README.""" + today = datetime.date.today().isoformat() + lines = [ + f"# Auxiliary files — us_census_bps.{tables[0].rsplit('_', 1)[0]}", + "", + "## Citation", + "", + CITATION, + "", + "The Building Permits Survey is a work of the United States", + "Government and is in the public domain (17 U.S.C. s.105).", + "", + "## Tables these documents describe", + "", + *[f"- `basedosdados.{DATASET_ID}.{t}`" for t in tables], + "", + "## Bundled files", + "", + f"### `{doc_name}`", + "", + covers, + "", + f"- Source: {DOC_BASE}{remote}", + f"- Downloaded: {today}", + "", + "## What Data Basis changed from the raw files", + "", + "- The raw files carry one row per geography and period with four", + " blocks of columns, one per structure type. The Data Basis tables", + " are long: one row per geography, period and structure type, with", + " the structure type in `structure_type` and its label in the", + " `dicionario` table.", + "- Valuation is normalised to US dollars everywhere. The state and", + " metropolitan files publish it in thousands, so those figures are", + " multiplied by 1,000 and are therefore precise to $1,000.", + "- `buildings`, `units` and `valuation` are the estimates including", + " imputation for non-responding permit offices, which is the series", + " to use by default. The `_reported` columns count only offices that", + " reported.", + "- Codes the source uses to mean 'not applicable' are stored as NULL:", + " 999 for CSA, 99999 for CBSA, 9999 for PMSA and MSA, and 00000,", + " 000 and 99990 for place, county and minor civil division codes.", + "- Until 2021 the survey identified the territories by their old", + " Census codes rather than by FIPS. `state_id` carries the FIPS code", + " throughout; the state tables' `geography_id` keeps the code as", + " published.", + "", + "## Further reading, not bundled", + "", + ] + for title, url, note in LINK_ONLY: + lines += [f"- **{title}** — {url}", f" {note}", ""] + return "\n".join(lines) + + +def main() -> int: + WORK.mkdir(parents=True, exist_ok=True) + config = tomllib.loads( + (Path.home() / ".basedosdados/config.toml").read_text() + ) + creds = service_account.Credentials.from_service_account_file( + config["gcloud-projects"]["staging"]["credentials_path"] + ) + gcs = storage.Client(credentials=creds, project=BUCKET) + bucket = gcs.bucket(BUCKET, user_project=BUCKET) + + registered: dict[str, str] = {} + for tables, (remote, local_name, covers) in DOCS.items(): + pdf = WORK / local_name + if not pdf.exists(): + request = urllib.request.Request( + DOC_BASE + remote, headers=constants.HEADERS.value + ) + with urllib.request.urlopen(request, timeout=180) as response: + pdf.write_bytes(response.read()) + archive = WORK / f"{tables[0].rsplit('_', 1)[0]}_auxiliary_files.zip" + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr( + "README.md", readme(tables, local_name, covers, remote) + ) + zf.write(pdf, local_name) + for table in tables: + key = f"auxiliary_files/{DATASET_ID}/{table}/auxiliary_files.zip" + bucket.blob(key).upload_from_filename(str(archive)) + url = f"https://storage.googleapis.com/{BUCKET}/{key}" + registered[table] = url + print(f"{table}: {archive.stat().st_size:,} bytes -> {url}") + + print("\n=== anonymous fetch of every registered URL ===") + for table, url in registered.items(): + request = urllib.request.Request(url, method="HEAD") + try: + with urllib.request.urlopen(request, timeout=60) as response: + status = str(response.status) + except urllib.error.HTTPError as exc: + status = f"{exc.code} {exc.reason}" + except Exception as exc: + status = f"{type(exc).__name__}" + print(f" {table:24s} HTTP {status}") + + (WORK / "urls.txt").write_text( + "\n".join(f"{t}\t{u}" for t, u in registered.items()), encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/models/us_census_bps/code/metadata.py b/models/us_census_bps/code/metadata.py new file mode 100644 index 0000000000..c5c8b9a43f --- /dev/null +++ b/models/us_census_bps/code/metadata.py @@ -0,0 +1,448 @@ +"""Register the us_census_bps metadata in the Data Basis backend. + +Idempotent: every record is looked up before it is written, and the existing +id is passed back, because create_update_* duplicates a record when called +without one. + +Usage: + python models/us_census_bps/code/metadata.py --env staging +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +sys.path.insert( + 0, "/Users/rdahis/Monash Uni Enterprise Dropbox/Ricardo Dahis/BD/mcp" +) +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) + +import server + +from pipelines.datasets.us_census_bps.constants import constants + +ARCH = Path(__file__).resolve().parent / "architecture" +DATASET_SLUG = "building_permits_survey_bps" +GCP_DATASET = constants.DATASET_ID.value +BASE = constants.BASE_URL.value + +NAME_PT = "Pesquisa de Alvarás de Construção (BPS)" +NAME_EN = "Building Permits Survey (BPS)" +NAME_ES = "Encuesta de Permisos de Construcción (BPS)" + +DESC_PT = ( + "Contagens mensais e anuais de edificações, unidades habitacionais e " + "valor de construção de novas residências privadas autorizadas por " + "alvarás de construção nos Estados Unidos, desde 1980. Os dados são " + "publicados por jurisdição emissora de alvarás, condado, área " + "estatística metropolitana e estado, separados por tipo de edificação " + "(1 unidade, 2 unidades, 3 a 4 unidades e 5 ou mais unidades). As " + "estimativas incluem imputação para jurisdições que não responderam à " + "pesquisa, e as tabelas trazem também os valores apenas reportados." +) +DESC_EN = ( + "Monthly and annual counts of buildings, housing units and construction " + "valuation for new privately-owned residential construction authorized " + "by building permits in the United States, from 1980. The data are " + "published by permit-issuing jurisdiction, county, metropolitan " + "statistical area and state, split by structure type (1 unit, 2 units, " + "3 to 4 units and 5 or more units). Estimates include imputation for " + "jurisdictions that did not respond to the survey, and the tables also " + "carry the reported-only figures." +) +DESC_ES = ( + "Recuentos mensuales y anuales de edificaciones, viviendas y valor de " + "construcción de nuevas obras residenciales privadas autorizadas por " + "permisos de construcción en Estados Unidos, desde 1980. Los datos se " + "publican por jurisdicción emisora de permisos, condado, área " + "estadística metropolitana y estado, separados por tipo de edificación " + "(1 unidad, 2 unidades, 3 a 4 unidades y 5 o más unidades). Las " + "estimaciones incluyen imputación para las jurisdicciones que no " + "respondieron a la encuesta, y las tablas traen también las cifras " + "solamente reportadas." +) + +TAG_SLUGS = ["housing", "construcao", "real_estate", "regulacao"] +NEW_TAGS = [ + { + "slug": "building-permit", + "name_pt": "alvará de construção", + "name_en": "building permit", + "name_es": "permiso de construcción", + } +] +THEME_SLUGS = ["economics", "urbanization"] + +# geography level -> (source directory, table slugs it feeds) +SOURCES = { + "place": ( + "Place/", + ("permit_place_monthly", "permit_place_annual"), + "Place-level files", + "Arquivos por jurisdição emissora de alvarás", + "Archivos por jurisdicción emisora de permisos", + ), + "county": ( + "County/", + ("permit_county_monthly", "permit_county_annual"), + "County-level files", + "Arquivos por condado", + "Archivos por condado", + ), + "cbsa": ( + "CBSA%20(beginning%20Jan%202024)/", + ("permit_cbsa_monthly", "permit_cbsa_annual"), + "Core Based Statistical Area files", + "Arquivos por área estatística baseada em núcleo", + "Archivos por área estadística basada en núcleo", + ), + "msa": ( + "Metro%20(ending%202023)/", + ("permit_msa_monthly", "permit_msa_annual"), + "Metropolitan Statistical Area files, through 2023", + "Arquivos por área estatística metropolitana, até 2023", + "Archivos por área estadística metropolitana, hasta 2023", + ), + "state": ( + "State/", + ("permit_state_monthly", "permit_state_annual"), + "State-level files", + "Arquivos por estado", + "Archivos por estado", + ), +} + +SOURCE_DESC_EN = ( + "Comma-delimited ASCII files published by the U.S. Census Bureau. Works " + "of the United States Government are not subject to copyright (17 U.S.C. " + "s.105) and are in the public domain. Record layouts for every geography " + "level are published at " + BASE + "Documentation/ and are bundled with " + "each table's auxiliary files." +) +SOURCE_DESC_PT = ( + "Arquivos ASCII separados por vírgula publicados pelo U.S. Census " + "Bureau. Obras do Governo dos Estados Unidos não estão sujeitas a " + "direitos autorais (17 U.S.C. s.105) e são de domínio público. Os " + "layouts de registro de cada nível geográfico estão em " + + BASE + + "Documentation/ e acompanham os arquivos auxiliares de cada tabela." +) +SOURCE_DESC_ES = ( + "Archivos ASCII separados por comas publicados por el U.S. Census " + "Bureau. Las obras del Gobierno de los Estados Unidos no están sujetas a " + "derechos de autor (17 U.S.C. s.105) y son de dominio público. Los " + "diseños de registro de cada nivel geográfico están en " + + BASE + + "Documentation/ y acompañan los archivos auxiliares de cada tabla." +) + +TABLE_NAMES = { + "permit_place_monthly": ( + "Alvarás por jurisdição, mensal", + "Permits by place, monthly", + "Permisos por jurisdicción, mensual", + ), + "permit_place_annual": ( + "Alvarás por jurisdição, anual", + "Permits by place, annual", + "Permisos por jurisdicción, anual", + ), + "permit_county_monthly": ( + "Alvarás por condado, mensal", + "Permits by county, monthly", + "Permisos por condado, mensual", + ), + "permit_county_annual": ( + "Alvarás por condado, anual", + "Permits by county, annual", + "Permisos por condado, anual", + ), + "permit_cbsa_monthly": ( + "Alvarás por área estatística baseada em núcleo, mensal", + "Permits by Core Based Statistical Area, monthly", + "Permisos por área estadística basada en núcleo, mensual", + ), + "permit_cbsa_annual": ( + "Alvarás por área estatística baseada em núcleo, anual", + "Permits by Core Based Statistical Area, annual", + "Permisos por área estadística basada en núcleo, anual", + ), + "permit_msa_monthly": ( + "Alvarás por área estatística metropolitana, mensal", + "Permits by Metropolitan Statistical Area, monthly", + "Permisos por área estadística metropolitana, mensual", + ), + "permit_msa_annual": ( + "Alvarás por área estatística metropolitana, anual", + "Permits by Metropolitan Statistical Area, annual", + "Permisos por área estadística metropolitana, anual", + ), + "permit_state_monthly": ( + "Alvarás por estado, mensal", + "Permits by state, monthly", + "Permisos por estado, mensual", + ), + "permit_state_annual": ( + "Alvarás por estado, anual", + "Permits by state, annual", + "Permisos por estado, anual", + ), + "dicionario": ("Dicionário", "Dictionary", "Diccionario"), +} + +# table -> (entity slug, identifying column) for each observation level +OBSERVATION_LEVELS = { + "permit_place_monthly": [ + ("city", "place_id"), + ("construction", "structure_type"), + ], + "permit_place_annual": [ + ("city", "place_id"), + ("construction", "structure_type"), + ], + "permit_county_monthly": [ + ("county", "county_id"), + ("construction", "structure_type"), + ], + "permit_county_annual": [ + ("county", "county_id"), + ("construction", "structure_type"), + ], + "permit_cbsa_monthly": [ + ("metropolitan_area", "cbsa_id"), + ("construction", "structure_type"), + ], + "permit_cbsa_annual": [ + ("metropolitan_area", "cbsa_id"), + ("construction", "structure_type"), + ], + "permit_msa_monthly": [ + ("metropolitan_area", "msa_cmsa_id"), + ("construction", "structure_type"), + ], + "permit_msa_annual": [ + ("metropolitan_area", "msa_cmsa_id"), + ("construction", "structure_type"), + ], + "permit_state_monthly": [ + ("state", "state_id"), + ("construction", "structure_type"), + ], + "permit_state_annual": [ + ("state", "state_id"), + ("construction", "structure_type"), + ], +} + +# table -> (start, end, free_end) where each is (year, month) or (year, None). +# free_end is the last period that stays public; None means the whole series +# is public. Data Basis paywalls the trailing window of any table that +# refreshes monthly or more often, with a six-month free lag. +COVERAGE = { + "permit_place_monthly": ((1988, 1), (2026, 7), (2026, 1)), + "permit_place_annual": ((1980, None), (2025, None), None), + "permit_county_monthly": ((2000, 1), (2026, 7), (2026, 1)), + "permit_county_annual": ((1990, None), (2025, None), None), + "permit_cbsa_monthly": ((2004, 1), (2026, 7), (2026, 1)), + "permit_cbsa_annual": ((2003, None), (2025, None), None), + "permit_msa_monthly": ((1988, 1), (2003, 12), None), + "permit_msa_annual": ((1980, None), (2002, None), None), + "permit_state_monthly": ((1988, 1), (2026, 7), (2026, 1)), + "permit_state_annual": ((1980, None), (2025, None), None), +} + +TABLE_ORDER = [ + "permit_place_monthly", + "permit_place_annual", + "permit_county_monthly", + "permit_county_annual", + "permit_cbsa_monthly", + "permit_cbsa_annual", + "permit_msa_monthly", + "permit_msa_annual", + "permit_state_monthly", + "permit_state_annual", + "dicionario", +] + +AUX_URL = ( + "https://storage.googleapis.com/basedosdados-dev/auxiliary_files/" + f"{GCP_DATASET}/{{table}}/auxiliary_files.zip" +) + +# Refresh cadence recorded against each table. +UPDATE = {"monthly": ("month", 1, 1), "annual": ("year", 1, 4)} +TODAY = "2026-09-09T00:00:00+00:00" + + +def next_period(year: int, month: int | None) -> tuple[int, int | None]: + """Return the period after the given one, so free and pro never overlap.""" + if month is None: + return year + 1, None + return (year + 1, 1) if month == 12 else (year, month + 1) + + +DICIONARIO_DESC = ( + "Dicionário dos valores codificados usados nas tabelas de " + "us_census_bps, com uma linha por tabela, coluna e chave." +) + + +def table_description(table: str) -> str: + """Return the table description used in both the dbt schema and here.""" + from build_dbt import DESCRIPTIONS + + return DESCRIPTIONS[table] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--env", default="staging") + parser.add_argument("--gcp-project", default="basedosdados-dev") + args = parser.parse_args() + env = args.env + + ids = server.discover_ids( + env=env, keys=["status", "theme", "entity", "license", "availability"] + ) + status = ids["status"] + account = server.get_authenticated_account(env=env)["id"] + org = server.lookup_id("organization", "census_bureau", env=env)["id"] + + tag_ids = [] + for slug in TAG_SLUGS: + tag_ids.append(server.lookup_id("tag", slug, env=env)["id"]) + for tag in NEW_TAGS: + try: + tag_ids.append(server.lookup_id("tag", tag["slug"], env=env)["id"]) + except RuntimeError: + created = server.create_update_tag(env=env, **tag) + print(f"created tag {tag['slug']}: {created}") + tag_ids.append(server.lookup_id("tag", tag["slug"], env=env)["id"]) + + existing = server.get_dataset(DATASET_SLUG, env=env) + dataset_id = existing.get("id") if existing.get("found") else None + result = server.create_update_dataset( + slug=DATASET_SLUG, + name_pt=NAME_PT, + name_en=NAME_EN, + name_es=NAME_ES, + description_pt=DESC_PT, + description_en=DESC_EN, + description_es=DESC_ES, + organization_ids=[org], + theme_ids=[ids["theme"][t] for t in THEME_SLUGS], + tag_ids=tag_ids, + status_id=status["under_review"], + id=dataset_id, + env=env, + ) + dataset_id = result.get("id") or dataset_id + print(f"dataset {DATASET_SLUG}: {dataset_id}") + + prior_sources = server.get_raw_data_sources(DATASET_SLUG, env=env) + if isinstance(prior_sources, dict): + prior_sources = prior_sources.get("raw_data_sources", []) + existing_sources = { + s["url"]: s["id"] for s in prior_sources if s.get("url") + } + source_ids: dict[str, str | None] = {} + for level, (path, tables, name_en, name_pt, name_es) in SOURCES.items(): + url = BASE + path + res = server.create_update_raw_data_source( + dataset_id=dataset_id, + name_pt=name_pt, + name_en=name_en, + name_es=name_es, + url=url, + license_id=ids["license"]["cc0"], + availability_id=ids["availability"]["online"], + description_pt=SOURCE_DESC_PT, + description_en=SOURCE_DESC_EN, + description_es=SOURCE_DESC_ES, + has_structured_data=True, + is_free=True, + contains_api=False, + requires_registration=False, + id=existing_sources.get(url), + env=env, + ) + source_ids[level] = res.get("id") or existing_sources.get(url) + print(f"raw source {level}: {source_ids[level]}") + for table in tables: + source_ids[table] = source_ids[level] + + state = server.get_dataset(DATASET_SLUG, env=env).get("tables", {}) + for table in TABLE_ORDER: + pt, en, es = TABLE_NAMES[table] + desc = ( + DICIONARIO_DESC + if table == "dicionario" + else table_description(table) + ) + prior = state.get(table, {}) + res = server.create_update_table( + slug=table, + name_pt=pt, + name_en=en, + name_es=es, + dataset_id=dataset_id, + status_id=status["published"], + published_by_ids=[account], + data_cleaned_by_ids=[account], + description_pt=desc, + description_en=desc, + description_es=desc, + auxiliary_files_url=( + "" if table == "dicionario" else AUX_URL.format(table=table) + ), + id=prior.get("id"), + env=env, + ) + table_id = res.get("id") or prior.get("id") + print(f"table {table}: {table_id}") + + server.create_update_cloud_table( + table_id=table_id, + gcp_project_id=args.gcp_project, + gcp_dataset_id=GCP_DATASET, + gcp_table_id=table, + id=(prior.get("cloud_tables") or [{}])[0].get("id"), + env=env, + ) + + columns = list(csv.DictReader((ARCH / f"{table}.csv").open())) + payload = [ + { + "name": c["name"], + "bigquery_type": c["bigquery_type"], + "description": c["description"], + "description_en": c["description"], + "temporal_coverage": c["temporal_coverage"], + "covered_by_dictionary": c["covered_by_dictionary"], + "directory_column": c["directory_column"], + "measurement_unit": c["measurement_unit"], + "has_sensitive_data": c["has_sensitive_data"], + "observations": c["observations"], + "observations_en": c["observations"], + } + for c in columns + ] + server.bulk_upsert_columns( + table_id=table_id, + columns_json=json.dumps(payload, ensure_ascii=False), + env=env, + ) + print(f" columns: {len(payload)}") + print( + "\nStage 1 complete (dataset, sources, tables, cloud tables, columns)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/models/us_census_bps/code/metadata_stage2.py b/models/us_census_bps/code/metadata_stage2.py new file mode 100644 index 0000000000..ed59604a51 --- /dev/null +++ b/models/us_census_bps/code/metadata_stage2.py @@ -0,0 +1,255 @@ +"""Second half of the us_census_bps metadata registration. + +Runs in three passes, in this order for a reason: + +1. Tables, with the deferred raw-data-source link. ``CreateUpdateTable`` + fails with ``'TableForm' has no field named 'coverages_areas'`` on any + table that already has a coverage, so every table write must happen + before any coverage is created. +2. Observation levels, partition flags, coverage, datetime ranges and the + refresh record. +3. Table order. + +Idempotent: existing records are read back and their ids reused, because +create_update_* duplicates a record when called without one. Coverage is read +through GraphQL rather than ``get_dataset``, which does not return +``is_closed`` and so cannot tell the free coverage from the BD Pro one. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert( + 0, "/Users/rdahis/Monash Uni Enterprise Dropbox/Ricardo Dahis/BD/mcp" +) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import server +from metadata import ( + AUX_URL, + BASE, + COVERAGE, + DATASET_SLUG, + DICIONARIO_DESC, + OBSERVATION_LEVELS, + SOURCES, + TABLE_NAMES, + TABLE_ORDER, + TODAY, + UPDATE, + next_period, + table_description, +) + +COVERAGE_QUERY = """ +{ allTable(id: "%s") { edges { node { + coverages { edges { node { id isClosed + datetimeRanges { edges { node { id } } } } } } + updates { edges { node { id } } } + observationLevels { edges { node { id entity { slug } } } } } } } } +""" + + +def read_state(table_id: str, env: str) -> dict: + """Read a table's coverages, updates and observation levels by id.""" + node = server._gql(COVERAGE_QUERY % table_id, {}, env=env)["allTable"][ + "edges" + ][0]["node"] + coverages = [] + for edge in node["coverages"]["edges"]: + c = edge["node"] + ranges = [ + server._strip_id(r["node"]["id"]) + for r in c["datetimeRanges"]["edges"] + ] + coverages.append( + { + "id": server._strip_id(c["id"]), + "is_closed": c["isClosed"], + "ranges": ranges, + } + ) + return { + "coverages": coverages, + "updates": [ + server._strip_id(u["node"]["id"]) for u in node["updates"]["edges"] + ], + "observation_levels": { + o["node"]["entity"]["slug"]: server._strip_id(o["node"]["id"]) + for o in node["observationLevels"]["edges"] + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--env", default="staging") + args = parser.parse_args() + env = args.env + + ids = server.discover_ids(env=env, keys=["entity", "status"]) + entity = ids["entity"] + published = ids["status"]["published"] + account = server.get_authenticated_account(env=env)["id"] + area = server.lookup_id("area", "us", env=env)["id"] + dataset = server.get_dataset(DATASET_SLUG, env=env) + tables = dataset["tables"] + + prior_sources = server.get_raw_data_sources(DATASET_SLUG, env=env) + if isinstance(prior_sources, dict): + prior_sources = prior_sources.get("raw_data_sources", []) + by_url = {s["url"]: s["id"] for s in prior_sources if s.get("url")} + source_by_table: dict[str, str] = {} + for _level, (path, table_slugs, *_rest) in SOURCES.items(): + for table in table_slugs: + source_by_table[table] = by_url[BASE + path] + + print("=== pass 1: tables and raw-source links ===") + for table in TABLE_ORDER: + info = tables[table] + pt, en, es = TABLE_NAMES[table] + desc = ( + DICIONARIO_DESC + if table == "dicionario" + else table_description(table) + ) + server.create_update_table( + slug=table, + name_pt=pt, + name_en=en, + name_es=es, + dataset_id=dataset["id"], + status_id=published, + published_by_ids=[account], + data_cleaned_by_ids=[account], + description_pt=desc, + description_en=desc, + description_es=desc, + raw_data_source_ids=( + [source_by_table[table]] if table in source_by_table else None + ), + auxiliary_files_url=( + "" if table == "dicionario" else AUX_URL.format(table=table) + ), + id=info["id"], + env=env, + ) + print(f" {table}: raw source {source_by_table.get(table, '-')}") + + print("\n=== pass 2: observation levels, coverage and updates ===") + for table in TABLE_ORDER: + if table == "dicionario": + continue + info = tables[table] + table_id = info["id"] + columns = {c["name"]: c["id"] for c in info["columns"]} + state = read_state(table_id, env) + print(f" --- {table} ---") + + ol_ids: list[str] = [] + for entity_slug, column_name in OBSERVATION_LEVELS[table]: + res = server.create_update_observation_level( + table_id=table_id, + entity_id=entity[entity_slug], + id=state["observation_levels"].get(entity_slug), + env=env, + ) + ol_id = res.get("id") or state["observation_levels"][entity_slug] + ol_ids.append(ol_id) + server.update_column( + column_id=columns[column_name], + column_name=column_name, + table_id=table_id, + observation_level_id=ol_id, + env=env, + ) + print(f" observation level {entity_slug} -> {column_name}") + server.reorder_observation_levels( + table_id=table_id, ol_ids=ol_ids, env=env + ) + + # Set last: update_column's booleans default to False, so the + # partition flag has to outlive the observation-level writes above. + server.update_column( + column_id=columns["year"], + column_name="year", + table_id=table_id, + is_partition=True, + env=env, + ) + + start, end, free_end = COVERAGE[table] + free_prior = next( + (c for c in state["coverages"] if not c["is_closed"]), None + ) + free_id = server.create_update_coverage( + table_id=table_id, + area_id=area, + is_closed=False, + id=(free_prior or {}).get("id"), + env=env, + ).get("id") or (free_prior or {}).get("id") + free_last = free_end or end + server.create_update_datetime_range( + coverage_id=free_id, + start_year=start[0], + start_month=start[1], + end_year=free_last[0], + end_month=free_last[1], + interval=1, + is_closed=False, + id=((free_prior or {}).get("ranges") or [None])[0], + env=env, + ) + print(f" free coverage {start} .. {free_last}") + + if free_end is not None: + pro_prior = next( + (c for c in state["coverages"] if c["is_closed"]), None + ) + pro_id = server.create_update_coverage( + table_id=table_id, + area_id=area, + is_closed=True, + id=(pro_prior or {}).get("id"), + env=env, + ).get("id") or (pro_prior or {}).get("id") + pro_start = next_period(*free_end) + server.create_update_datetime_range( + coverage_id=pro_id, + start_year=pro_start[0], + start_month=pro_start[1], + end_year=end[0], + end_month=end[1], + interval=1, + is_closed=True, + id=((pro_prior or {}).get("ranges") or [None])[0], + env=env, + ) + print(f" BD Pro coverage {pro_start} .. {end}") + + cadence = "monthly" if table.endswith("_monthly") else "annual" + entity_slug, frequency, lag = UPDATE[cadence] + server.create_update_update( + entity_id=entity[entity_slug], + frequency=frequency, + lag=lag, + latest=TODAY, + table_id=table_id, + id=(state["updates"] or [None])[0], + env=env, + ) + print(f" update: every {frequency} {entity_slug}, lag {lag}") + + server.reorder_tables( + dataset_slug=DATASET_SLUG, table_slugs=TABLE_ORDER, env=env + ) + print("\n=== pass 3: table order set ===") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/models/us_census_bps/code/publish.py b/models/us_census_bps/code/publish.py new file mode 100644 index 0000000000..cf26666d1f --- /dev/null +++ b/models/us_census_bps/code/publish.py @@ -0,0 +1,78 @@ +"""Flip the us_census_bps dataset status. + +Data Basis publishes the dev/staging dataset before promotion so a reviewer +sees it as it will appear, and publishes the production dataset only after the +onboarding PR is merged, the table-approve action has materialised the prod +tables, and those tables are verified. + +CreateUpdateDataset replaces rather than patches, so every required field is +re-sent from the same definitions the registration used. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +sys.path.insert( + 0, "/Users/rdahis/Monash Uni Enterprise Dropbox/Ricardo Dahis/BD/mcp" +) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import server +from metadata import ( + DATASET_SLUG, + DESC_EN, + DESC_ES, + DESC_PT, + NAME_EN, + NAME_ES, + NAME_PT, + NEW_TAGS, + TAG_SLUGS, + THEME_SLUGS, +) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--env", default="staging") + parser.add_argument( + "--status", default="published", choices=["published", "under_review"] + ) + args = parser.parse_args() + env = args.env + + ids = server.discover_ids(env=env, keys=["status", "theme"]) + org = server.lookup_id("organization", "census_bureau", env=env)["id"] + tag_ids = [ + server.lookup_id("tag", slug, env=env)["id"] + for slug in TAG_SLUGS + [t["slug"] for t in NEW_TAGS] + ] + dataset = server.get_dataset(DATASET_SLUG, env=env) + if not dataset.get("found"): + raise SystemExit(f"dataset {DATASET_SLUG} not found in {env}") + + server.create_update_dataset( + slug=DATASET_SLUG, + name_pt=NAME_PT, + name_en=NAME_EN, + name_es=NAME_ES, + description_pt=DESC_PT, + description_en=DESC_EN, + description_es=DESC_ES, + organization_ids=[org], + theme_ids=[ids["theme"][t] for t in THEME_SLUGS], + tag_ids=tag_ids, + status_id=ids["status"][args.status], + id=dataset["id"], + env=env, + ) + after = server.get_dataset(DATASET_SLUG, env=env) + print(f"{DATASET_SLUG} in {env}: status now {after.get('status')}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/models/us_census_bps/code/verify_metadata.py b/models/us_census_bps/code/verify_metadata.py new file mode 100644 index 0000000000..8555251186 --- /dev/null +++ b/models/us_census_bps/code/verify_metadata.py @@ -0,0 +1,186 @@ +"""Read the registered us_census_bps metadata back and check it is complete. + +Every write in this onboarding is verified by reading it back, not by trusting +the mutation's return value. +""" + +from __future__ import annotations + +import argparse +import csv +import sys +from pathlib import Path + +sys.path.insert( + 0, "/Users/rdahis/Monash Uni Enterprise Dropbox/Ricardo Dahis/BD/mcp" +) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import server +from metadata import DATASET_SLUG, TABLE_ORDER + +ARCH = Path(__file__).resolve().parent / "architecture" + +QUERY = """ +{ + allDataset(slug: "%s") { + edges { node { + slug status { slug } + themes { edges { node { slug } } } + tags { edges { node { slug } } } + organizations { edges { node { slug } } } + tables(first: 20) { edges { node { + slug order auxiliaryFilesUrl + status { slug } + rawDataSource { edges { node { url } } } + cloudTables { edges { node { gcpProjectId gcpDatasetId gcpTableId } } } + observationLevels { edges { node { + entity { slug } + columns { edges { node { name } } } + } } } + coverages { edges { node { + isClosed + datetimeRanges { edges { node { + startYear startMonth endYear endMonth isClosed + } } } + } } } + updates { edges { node { entity { slug } frequency lag latest } } } + columns(first: 60) { edges { node { + name isPartition coveredByDictionary measurementUnit + bigqueryType { name } + directoryPrimaryKey { name table { slug dataset { slug } } } + descriptionEn descriptionPt + } } } + } } } + } } + } +} +""" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--env", default="staging") + args = parser.parse_args() + + node = server._gql(QUERY % DATASET_SLUG, {}, env=args.env)["allDataset"][ + "edges" + ][0]["node"] + problems: list[str] = [] + + print(f"dataset {node['slug']} — status {node['status']['slug']}") + print( + f" organizations {[o['node']['slug'] for o in node['organizations']['edges']]}" + ) + print(f" themes {[t['node']['slug'] for t in node['themes']['edges']]}") + print(f" tags {[t['node']['slug'] for t in node['tags']['edges']]}") + if not node["tags"]["edges"]: + problems.append("dataset has no tags") + + tables = {t["node"]["slug"]: t["node"] for t in node["tables"]["edges"]} + if set(tables) != set(TABLE_ORDER): + problems.append(f"table set mismatch: {sorted(set(tables))}") + + for slug in TABLE_ORDER: + t = tables[slug] + expected = [ + c["name"] for c in csv.DictReader((ARCH / f"{slug}.csv").open()) + ] + got = {c["node"]["name"]: c["node"] for c in t["columns"]["edges"]} + missing = [c for c in expected if c not in got] + partitions = [n for n, c in got.items() if c["isPartition"]] + fks = { + n: f"{c['directoryPrimaryKey']['table']['dataset']['slug']}." + f"{c['directoryPrimaryKey']['table']['slug']}:" + f"{c['directoryPrimaryKey']['name']}" + for n, c in got.items() + if c["directoryPrimaryKey"] + } + dicts = sorted(n for n, c in got.items() if c["coveredByDictionary"]) + no_desc = sorted( + n for n, c in got.items() if not (c["descriptionEn"] or "").strip() + ) + ols = [ + ( + o["node"]["entity"]["slug"], + [c["node"]["name"] for c in o["node"]["columns"]["edges"]], + ) + for o in t["observationLevels"]["edges"] + ] + cov = [ + ( + c["node"]["isClosed"], + [ + ( + r["node"]["startYear"], + r["node"]["startMonth"], + r["node"]["endYear"], + r["node"]["endMonth"], + ) + for r in c["node"]["datetimeRanges"]["edges"] + ], + ) + for c in t["coverages"]["edges"] + ] + ct = [ + f"{c['node']['gcpProjectId']}.{c['node']['gcpDatasetId']}." + f"{c['node']['gcpTableId']}" + for c in t["cloudTables"]["edges"] + ] + sources = [s["node"]["url"] for s in t["rawDataSource"]["edges"]] + upd = [ + ( + u["node"]["entity"]["slug"], + u["node"]["frequency"], + u["node"]["lag"], + (u["node"]["latest"] or "")[:10], + ) + for u in t["updates"]["edges"] + ] + + print(f"\n{slug} (order {t['order']}, status {t['status']['slug']})") + print(f" columns {len(got)}/{len(expected)} partitions {partitions}") + print(f" observation levels {ols}") + print(f" directory links {fks}") + print(f" dictionary-covered {dicts}") + print(f" coverage {cov}") + print(f" update {upd}") + print(f" cloud table {ct} raw sources {len(sources)}") + print( + f" auxiliary files {'set' if t['auxiliaryFilesUrl'] else 'MISSING'}" + ) + + if missing: + problems.append(f"{slug}: missing columns {missing}") + if no_desc: + problems.append( + f"{slug}: columns without an English description {no_desc}" + ) + if len(ct) != 1: + problems.append(f"{slug}: {len(ct)} cloud tables") + if slug != "dicionario": + if partitions != ["year"]: + problems.append(f"{slug}: partitions are {partitions}") + if len(ols) != 2 or any(len(c) != 1 for _e, c in ols): + problems.append(f"{slug}: observation levels {ols}") + if not cov: + problems.append(f"{slug}: no coverage") + if len(upd) != 1: + problems.append(f"{slug}: {len(upd)} update records") + if len(sources) != 1: + problems.append(f"{slug}: {len(sources)} raw sources linked") + if not t["auxiliaryFilesUrl"]: + problems.append(f"{slug}: no auxiliary files URL") + + print("\n" + "=" * 60) + if problems: + print(f"PROBLEMS ({len(problems)}):") + for p in problems: + print(" -", p) + return 1 + print("All metadata checks passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 31739a5c94..8fdf93c6b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -334,6 +334,10 @@ project-excludes = [ # run via `uv run` with bare imports resolved through sys.path. The shared # transform lives in the typed pipelines/datasets/br_mgi_compras_publicas/. "models/br_mgi_compras_publicas/code", + # Standalone one-shot onboarding ETL: sibling-module and out-of-repo + # imports pyrefly cannot resolve. The reusable transform lives in + # pipelines/datasets/us_census_bps/utils.py, which is checked. + "models/us_census_bps/code", "models/br_tse_eleicoes/code/[[]dbt[]]br_tse_eleicoes.ipynb", "models/world_wb_mides/code/licitacao_item.ipynb", "models/world_olympedia_olympics/code/[[]code[]]world_olympedia_olympics.ipynb", From 8edea1f375651fb349c09af4c01bfc6b138f1e23 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 9 Sep 2026 19:13:28 +1000 Subject: [PATCH 4/7] feat(us_census_bps): rename the backend dataset to bps and register prod Three changes from the review, then the production promotion. - Backend slug is now `bps`, renamed in place on staging so the record keeps its id and history. The GCP dataset stays `us_census_bps`. - `buildings` and `units` carry measurement units. `Column.measurementUnit` is a free string, not a foreign key to the unit vocabulary, so `building` and `housing_unit` need no global vocabulary change. The unit is no longer repeated in the column observations. - Production metadata registered, dataset status `under_review` so it stays off the public site until the PR merges, table-approve materialises `basedosdados.us_census_bps.*`, and those tables are verified. Cloud tables point at `basedosdados`. The tag list is now per environment: staging still carries the older Portuguese slugs (`construcao`, `regulacao`, `real_estate`) while production uses English ones (`construction`, `regulation`, `real-estate`, the last of which has 10 datasets against 1 for the `real_estate` near-duplicate). `building-permit` was created in both. verify_metadata.py passes against staging and production. --- .../code/architecture/permit_cbsa_annual.csv | 8 +- .../code/architecture/permit_cbsa_monthly.csv | 8 +- .../architecture/permit_county_annual.csv | 8 +- .../architecture/permit_county_monthly.csv | 8 +- .../code/architecture/permit_msa_annual.csv | 8 +- .../code/architecture/permit_msa_monthly.csv | 8 +- .../code/architecture/permit_place_annual.csv | 8 +- .../architecture/permit_place_monthly.csv | 4 +- .../code/architecture/permit_state_annual.csv | 8 +- .../architecture/permit_state_monthly.csv | 8 +- .../us_census_bps/code/build_architecture.py | 12 ++- models/us_census_bps/code/metadata.py | 76 ++++++++++++++----- models/us_census_bps/code/publish.py | 2 +- 13 files changed, 103 insertions(+), 63 deletions(-) diff --git a/models/us_census_bps/code/architecture/permit_cbsa_annual.csv b/models/us_census_bps/code/architecture/permit_cbsa_annual.csv index 40f1b6743f..a7c1bd93db 100644 --- a/models/us_census_bps/code/architecture/permit_cbsa_annual.csv +++ b/models/us_census_bps/code/architecture/permit_cbsa_annual.csv @@ -6,9 +6,9 @@ cbsa_type,STRING,Header code distinguishing metropolitan from micropolitan areas full_monthly_coverage,STRING,Whether the area is completely covered by monthly reporting permit-issuing places,2003(1)2023,yes,,,no,"Published up to 2023 only, in the field replaced by cbsa_type from January 2024. Null from 2024 on",MONCOV structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block cbsa_name,STRING,"Name of the Core Based Statistical Area, as published",,no,,,no,,CBSA Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_cbsa_monthly.csv b/models/us_census_bps/code/architecture/permit_cbsa_monthly.csv index 4cca893870..a96ce36e33 100644 --- a/models/us_census_bps/code/architecture/permit_cbsa_monthly.csv +++ b/models/us_census_bps/code/architecture/permit_cbsa_monthly.csv @@ -7,9 +7,9 @@ cbsa_type,STRING,Header code distinguishing metropolitan from micropolitan areas full_monthly_coverage,STRING,Whether the area is completely covered by monthly reporting permit-issuing places,2003(1)2023,yes,,,no,"Published up to 2023 only, in the field replaced by cbsa_type from January 2024. Null from 2024 on",MONCOV structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block cbsa_name,STRING,"Name of the Core Based Statistical Area, as published",,no,,,no,,CBSA Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_county_annual.csv b/models/us_census_bps/code/architecture/permit_county_annual.csv index b51283dad1..a00f7049ec 100644 --- a/models/us_census_bps/code/architecture/permit_county_annual.csv +++ b/models/us_census_bps/code/architecture/permit_county_annual.csv @@ -6,9 +6,9 @@ region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block county_name,STRING,"Name of the county, as published",,no,,,no,,County Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_county_monthly.csv b/models/us_census_bps/code/architecture/permit_county_monthly.csv index 5c02464d7b..db18f91e9f 100644 --- a/models/us_census_bps/code/architecture/permit_county_monthly.csv +++ b/models/us_census_bps/code/architecture/permit_county_monthly.csv @@ -7,9 +7,9 @@ region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block county_name,STRING,"Name of the county, as published",,no,,,no,,County Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_msa_annual.csv b/models/us_census_bps/code/architecture/permit_msa_annual.csv index c8a224536f..483a37b873 100644 --- a/models/us_census_bps/code/architecture/permit_msa_annual.csv +++ b/models/us_census_bps/code/architecture/permit_msa_annual.csv @@ -4,9 +4,9 @@ msa_cmsa_id,STRING,Four-digit Metropolitan or Consolidated Metropolitan Statisti pmsa_id,STRING,Four-digit Primary Metropolitan Statistical Area code,,no,,,no,"Null where the source publishes 9999, meaning the area is not divided into PMSAs",PMSA Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block msa_name,STRING,"Name of the metropolitan area, as published",,no,,,no,,MA Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_msa_monthly.csv b/models/us_census_bps/code/architecture/permit_msa_monthly.csv index e63618c39f..fd919a4dcc 100644 --- a/models/us_census_bps/code/architecture/permit_msa_monthly.csv +++ b/models/us_census_bps/code/architecture/permit_msa_monthly.csv @@ -5,9 +5,9 @@ msa_cmsa_id,STRING,Four-digit Metropolitan or Consolidated Metropolitan Statisti pmsa_id,STRING,Four-digit Primary Metropolitan Statistical Area code,,no,,,no,"Null where the source publishes 9999, meaning the area is not divided into PMSAs",PMSA Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block msa_name,STRING,"Name of the metropolitan area, as published",,no,,,no,,MA Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_place_annual.csv b/models/us_census_bps/code/architecture/permit_place_annual.csv index aa6294d0ea..f6ef7e6b2a 100644 --- a/models/us_census_bps/code/architecture/permit_place_annual.csv +++ b/models/us_census_bps/code/architecture/permit_place_annual.csv @@ -19,9 +19,9 @@ footnote_code,STRING,Whether the place carries an explanatory footnote in the so zip_code,STRING,ZIP code of the permit office or official,2000(1)2026,no,,,no,,Zip Code population,INT64,Population of the place as carried in the source file,2008(1)2026,no,,person,no,"The Census layout document labels this the 2000 population, but the values change from release to release, so it is a current population estimate. Published from 2008 only",Pop months_reported,INT64,Number of months of the year for which the permit office reported residential data,,no,,month,no,,Number of Months Rep -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_place_monthly.csv b/models/us_census_bps/code/architecture/permit_place_monthly.csv index 13103dc0a1..b79322f92f 100644 --- a/models/us_census_bps/code/architecture/permit_place_monthly.csv +++ b/models/us_census_bps/code/architecture/permit_place_monthly.csv @@ -20,6 +20,6 @@ footnote_code,STRING,Whether the place carries an explanatory footnote in the so zip_code,STRING,ZIP code of the permit office or official,2000(1)2026,no,,,no,,Zip Code population,INT64,Population of the place as carried in the source file,2008(1)2026,no,,person,no,"The Census layout document labels this the 2000 population, but the values change from release to release, so it is a current population estimate. Published from 2008 only",Pop source_code,STRING,How the data for the permit office was obtained for the month,,yes,,,no,"Codes 1 to 4 mark reported data, 5 marks imputed data and 9 marks a month with no report and no imputation. The monthly place file carries no separate reported-only block, so this column is how a reported subset is selected",Source Code -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value diff --git a/models/us_census_bps/code/architecture/permit_state_annual.csv b/models/us_census_bps/code/architecture/permit_state_annual.csv index f6bea8e677..d816e3b651 100644 --- a/models/us_census_bps/code/architecture/permit_state_annual.csv +++ b/models/us_census_bps/code/architecture/permit_state_annual.csv @@ -7,9 +7,9 @@ region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block geography_name,STRING,"Name of the geography, as published",,no,,,no,,State Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/architecture/permit_state_monthly.csv b/models/us_census_bps/code/architecture/permit_state_monthly.csv index 36163b3e16..794acaaa65 100644 --- a/models/us_census_bps/code/architecture/permit_state_monthly.csv +++ b/models/us_census_bps/code/architecture/permit_state_monthly.csv @@ -8,9 +8,9 @@ region_id,STRING,"Census region code, from 1 to 4",,yes,,,no,,Region Code division_id,STRING,"Census division code, from 1 to 9",,yes,,,no,,Division Code structure_type,STRING,"Type of residential structure, by the number of housing units in the building",,yes,,,no,"Census structure type codes: 101 single-family, 103 two-family, 104 three- and four-family, 105 five-or-more-family. Summing the four codes reproduces the published total for the geography",column block geography_name,STRING,"Name of the geography, as published",,no,,,no,,State Name -buildings,INT64,Number of buildings authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in buildings,Bldgs -units,INT64,Number of housing units authorized by building permits,,no,,,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default. Counted in housing units,Units +buildings,INT64,Number of buildings authorized by building permits,,no,,building,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Bldgs +units,INT64,Number of housing units authorized by building permits,,no,,housing_unit,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Units valuation,INT64,"Construction valuation of the buildings authorized, in US dollars",,no,,usd,no,Estimate with imputation: reported data for responding permit offices plus imputed data for non-respondents. This is the series to use by default,Value -buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in buildings,Bldgs rep -units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation. Counted in housing units,Units rep +buildings_reported,INT64,"Number of buildings authorized, counting reporting permit offices only",,no,,building,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Bldgs rep +units_reported,INT64,"Number of housing units authorized, counting reporting permit offices only",,no,,housing_unit,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Units rep valuation_reported,INT64,"Construction valuation of the buildings authorized, in US dollars, counting reporting permit offices only",,no,,usd,no,Reported only: counts data from responding permit offices and makes no imputation for non-respondents. Lower than the estimate with imputation,Value rep diff --git a/models/us_census_bps/code/build_architecture.py b/models/us_census_bps/code/build_architecture.py index 81def2ec33..98075e76a3 100644 --- a/models/us_census_bps/code/build_architecture.py +++ b/models/us_census_bps/code/build_architecture.py @@ -119,14 +119,16 @@ def measures(*, reported: bool) -> list[list[str]]: "buildings", "INT64", "Number of buildings authorized by building permits", - observations=IMPUTED_NOTE + ". Counted in buildings", + unit="building", + observations=IMPUTED_NOTE, original="Bldgs", ), col( "units", "INT64", "Number of housing units authorized by building permits", - observations=IMPUTED_NOTE + ". Counted in housing units", + unit="housing_unit", + observations=IMPUTED_NOTE, original="Units", ), col( @@ -146,7 +148,8 @@ def measures(*, reported: bool) -> list[list[str]]: "INT64", "Number of buildings authorized, counting reporting permit " "offices only", - observations=REPORTED_NOTE + ". Counted in buildings", + unit="building", + observations=REPORTED_NOTE, original="Bldgs rep", ), col( @@ -154,7 +157,8 @@ def measures(*, reported: bool) -> list[list[str]]: "INT64", "Number of housing units authorized, counting reporting permit " "offices only", - observations=REPORTED_NOTE + ". Counted in housing units", + unit="housing_unit", + observations=REPORTED_NOTE, original="Units rep", ), col( diff --git a/models/us_census_bps/code/metadata.py b/models/us_census_bps/code/metadata.py index c5c8b9a43f..74b5468710 100644 --- a/models/us_census_bps/code/metadata.py +++ b/models/us_census_bps/code/metadata.py @@ -26,7 +26,7 @@ from pipelines.datasets.us_census_bps.constants import constants ARCH = Path(__file__).resolve().parent / "architecture" -DATASET_SLUG = "building_permits_survey_bps" +DATASET_SLUG = "bps" GCP_DATASET = constants.DATASET_ID.value BASE = constants.BASE_URL.value @@ -66,7 +66,13 @@ "solamente reportadas." ) -TAG_SLUGS = ["housing", "construcao", "real_estate", "regulacao"] +# The two backends carry different tag vocabularies: staging still uses the +# older Portuguese slugs, production uses English ones. Same tags either way. +TAG_SLUGS = { + "staging": ["housing", "construcao", "real_estate", "regulacao"], + "prod": ["housing", "construction", "real-estate", "regulation"], + "dev": ["housing", "construcao", "real_estate", "regulacao"], +} NEW_TAGS = [ { "slug": "building-permit", @@ -299,13 +305,59 @@ def table_description(table: str) -> str: return DESCRIPTIONS[table] +def column_payload(table: str) -> list[dict]: + """Build the bulk_upsert_columns payload from a table's architecture CSV.""" + return [ + { + "name": c["name"], + "bigquery_type": c["bigquery_type"], + "description": c["description"], + "description_en": c["description"], + "temporal_coverage": c["temporal_coverage"], + "covered_by_dictionary": c["covered_by_dictionary"], + "directory_column": c["directory_column"], + "measurement_unit": c["measurement_unit"], + "has_sensitive_data": c["has_sensitive_data"], + "observations": c["observations"], + "observations_en": c["observations"], + } + for c in csv.DictReader((ARCH / f"{table}.csv").open()) + ] + + +def refresh_columns(env: str) -> int: + """Re-upsert every table's columns from the architecture CSVs.""" + tables = server.get_dataset(DATASET_SLUG, env=env)["tables"] + for table in TABLE_ORDER: + payload = column_payload(table) + server.bulk_upsert_columns( + table_id=tables[table]["id"], + columns_json=json.dumps(payload, ensure_ascii=False), + env=env, + ) + print(f"{table}: {len(payload)} columns refreshed") + return 0 + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--env", default="staging") parser.add_argument("--gcp-project", default="basedosdados-dev") + parser.add_argument( + "--columns-only", + action="store_true", + help=( + "Re-upsert columns from the architecture CSVs and nothing else. " + "Needed once a table has coverages, because create_update_table " + "then fails on a backend form error." + ), + ) args = parser.parse_args() env = args.env + if args.columns_only: + return refresh_columns(env) + ids = server.discover_ids( env=env, keys=["status", "theme", "entity", "license", "availability"] ) @@ -314,7 +366,7 @@ def main() -> int: org = server.lookup_id("organization", "census_bureau", env=env)["id"] tag_ids = [] - for slug in TAG_SLUGS: + for slug in TAG_SLUGS[env]: tag_ids.append(server.lookup_id("tag", slug, env=env)["id"]) for tag in NEW_TAGS: try: @@ -415,23 +467,7 @@ def main() -> int: env=env, ) - columns = list(csv.DictReader((ARCH / f"{table}.csv").open())) - payload = [ - { - "name": c["name"], - "bigquery_type": c["bigquery_type"], - "description": c["description"], - "description_en": c["description"], - "temporal_coverage": c["temporal_coverage"], - "covered_by_dictionary": c["covered_by_dictionary"], - "directory_column": c["directory_column"], - "measurement_unit": c["measurement_unit"], - "has_sensitive_data": c["has_sensitive_data"], - "observations": c["observations"], - "observations_en": c["observations"], - } - for c in columns - ] + payload = column_payload(table) server.bulk_upsert_columns( table_id=table_id, columns_json=json.dumps(payload, ensure_ascii=False), diff --git a/models/us_census_bps/code/publish.py b/models/us_census_bps/code/publish.py index cf26666d1f..7608016071 100644 --- a/models/us_census_bps/code/publish.py +++ b/models/us_census_bps/code/publish.py @@ -48,7 +48,7 @@ def main() -> int: org = server.lookup_id("organization", "census_bureau", env=env)["id"] tag_ids = [ server.lookup_id("tag", slug, env=env)["id"] - for slug in TAG_SLUGS + [t["slug"] for t in NEW_TAGS] + for slug in TAG_SLUGS[env] + [t["slug"] for t in NEW_TAGS] ] dataset = server.get_dataset(DATASET_SLUG, env=env) if not dataset.get("found"): From b69a4b052445923057b00eb57aade7e8ce68e2a6 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 9 Sep 2026 19:35:19 +1000 Subject: [PATCH 5/7] feat(us_census_bps): add the recurring monthly Prefect pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebuilds the whole series each run rather than appending the newest month. The survey revises prior periods and a revision can land in any earlier month, so a trailing window would leave stale figures wherever a correction fell outside it; a full rebuild also reuses the onboarding path exactly. The cost is about 700 MB and 25.7 million rows once a month. Scheduled 25 15 17-22 BRT: the release lands around the 17th, roughly four weeks after the reference month, and 15:25 was an unused slot — flows piled on the same instant compete for BigQuery slots and fail together. Three things the flow does deliberately: - `dump_mode="append"` with an explicit staging-prefix clear, never `overwrite`. `overwrite` calls tb.delete(mode="all"), which drops the materialized production table, and it fires from the dev half too because bd.Table resolves its projects from the pod config rather than from bucket_name. The prefix clear does the wanted half without touching a BigQuery table, and closes the orphaned-part-file gap that append alone leaves when a run produces fewer parts than the last. - Every table is built before any test runs. The dictionary-coverage and relationship tests read sibling models, so interleaving run and test per table fails against a sibling that does not exist yet — invisible in a re-run, fatal in a clean environment. - job_variables sets memory_limit, not only memory, which the work pool's job template drops silently while leaving the pod on the 4Gi default. The download moves into pipelines/datasets/us_census_bps/utils.py so the pipeline and the one-shot bootstrap share one implementation, including the firewall handling: www2.census.gov answers HTTP 200 with a "Request Rejected" page for some valid URLs, and only a real 404 is treated as a missing file. download_all raises rather than returning a partial set. Also registers the raw-data-source Update in both backends, which the flow would otherwise only write on a run with update_metadata=True. The metropolitan source gets 2023-12, its real last release, not today's period. --- models/us_census_bps/code/download.py | 142 +---------- models/us_census_bps/code/metadata_stage2.py | 55 ++++ pipelines/datasets/us_census_bps/flows.py | 253 +++++++++++++++++++ pipelines/datasets/us_census_bps/tasks.py | 94 +++++++ pipelines/datasets/us_census_bps/utils.py | 176 +++++++++++++ 5 files changed, 591 insertions(+), 129 deletions(-) create mode 100644 pipelines/datasets/us_census_bps/flows.py create mode 100644 pipelines/datasets/us_census_bps/tasks.py diff --git a/models/us_census_bps/code/download.py b/models/us_census_bps/code/download.py index 85d1093790..039e7b4904 100644 --- a/models/us_census_bps/code/download.py +++ b/models/us_census_bps/code/download.py @@ -1,160 +1,44 @@ """Download the Census Building Permits Survey ASCII files. -Writes to ``$BPS_DATA_DIR`` (default ``~/Downloads/us_census_bps_data/input``). +Thin CLI over ``pipelines.datasets.us_census_bps.utils.download_all``, so the +one-shot bootstrap and the recurring pipeline share one implementation. -Two Census server behaviours make a naive download lose data silently: - -* A file that does not exist is served as an HTML page. It does carry HTTP - 404, but the body must still be checked so an HTML page is never written to - disk as if it were data. -* A small number of perfectly valid URLs are rejected by the site firewall, - which answers **HTTP 200** with a "Request Rejected" HTML page. Treating - that as a missing file drops whole months without any error — four place - months and one metropolitan month were lost this way on the first run. - The rejection is cached against the exact URL, so a cache-busting query - string gets the real file. +Writes to ``$BPS_DATA_DIR/input`` (default +``~/Downloads/us_census_bps_data/input``). """ from __future__ import annotations import argparse -import concurrent.futures as cf import os import sys import time -import urllib.error -import urllib.parse -import urllib.request from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[3])) -from pipelines.datasets.us_census_bps.constants import constants - -BASE = constants.BASE_URL.value -HEADERS = constants.HEADERS.value -THIS_YEAR = time.gmtime().tm_year +from pipelines.datasets.us_census_bps.utils import download_all DATA_DIR = Path( os.environ.get( "BPS_DATA_DIR", Path.home() / "Downloads/us_census_bps_data" ) ) -INPUT_DIR = DATA_DIR / "input" - - -def targets() -> list[tuple[str, Path]]: - """Build the (url, destination) list for every published BPS file.""" - out: list[tuple[str, Path]] = [] - for level, directory in constants.GEO_DIRS.value.items(): - prefix = constants.GEO_PREFIXES.value[level] - for kind in ("monthly", "annual"): - first = constants.FIRST_YEAR.value[(level, kind)] - for year in range(first, THIS_YEAR + 1): - names = ( - [ - f"{prefix}{year % 100:02d}{m:02d}c.txt" - for m in range(1, 13) - ] - if kind == "monthly" - else [f"{prefix}{year}a.txt"] - ) - for name in names: - url = BASE + urllib.parse.quote(f"{directory}/{name}") - out.append((url, INPUT_DIR / level / name)) - for directory, prefix in constants.PLACE_REGIONS.value.items(): - for kind in ("monthly", "annual"): - first = constants.FIRST_YEAR.value[("place", kind)] - for year in range(first, THIS_YEAR + 1): - names = ( - [ - f"{prefix}{year % 100:02d}{m:02d}c.txt" - for m in range(1, 13) - ] - if kind == "monthly" - else [f"{prefix}{year}a.txt"] - ) - for name in names: - url = BASE + urllib.parse.quote( - f"Place/{directory}/{name}" - ) - out.append((url, INPUT_DIR / "place" / name)) - return out - - -def fetch(url: str, dest: Path, tries: int = 5) -> tuple[str, str]: - """Download one file. - - Args: - url: Absolute URL of the Census file. - dest: Destination path; parent directories are created. - tries: Attempts before giving up. - - Returns: - ``(status, detail)`` where status is "ok", "cached", "missing" or - "failed". A "missing" result means the server answered HTTP 404. - """ - if dest.exists() and dest.stat().st_size > 0: - return "cached", "" - detail = "" - for attempt in range(tries): - # The firewall caches its rejection against the exact URL, so every - # retry after the first carries a different cache-busting parameter. - target = url if attempt == 0 else f"{url}?attempt={attempt}" - req = urllib.request.Request(target, headers=HEADERS) - try: - with urllib.request.urlopen(req, timeout=300) as resp: - body = resp.read() - except urllib.error.HTTPError as exc: - if exc.code == 404: - return "missing", "HTTP 404" - detail = f"HTTP {exc.code}" - time.sleep(2 * (attempt + 1)) - continue - except Exception as exc: - detail = f"{type(exc).__name__}: {exc}" - time.sleep(2 * (attempt + 1)) - continue - head = body[:1000].lower() - if b" int: parser = argparse.ArgumentParser() parser.add_argument("--workers", type=int, default=6) + parser.add_argument( + "--through-year", type=int, default=time.gmtime().tm_year + ) args = parser.parse_args() - - jobs = targets() - print(f"{len(jobs)} candidate files -> {INPUT_DIR}") - counts = {"ok": 0, "cached": 0, "missing": 0, "failed": 0} - failures: list[str] = [] - with cf.ThreadPoolExecutor(max_workers=args.workers) as pool: - futures = {pool.submit(fetch, u, d): u for u, d in jobs} - for i, fut in enumerate(cf.as_completed(futures), start=1): - status, detail = fut.result() - counts[status] += 1 - if status == "failed": - failures.append(f"{futures[fut]} -- {detail}") - if i % 250 == 0: - print(f" {i}/{len(jobs)} {counts}", flush=True) + counts = download_all( + DATA_DIR / "input", + through_year=args.through_year, + workers=args.workers, + ) print(f"done: {counts}") - if failures: - print("FAILED (not a 404 -- do not treat as missing):") - for line in failures[:30]: - print(" ", line) - return 1 return 0 diff --git a/models/us_census_bps/code/metadata_stage2.py b/models/us_census_bps/code/metadata_stage2.py index ed59604a51..1f8f868592 100644 --- a/models/us_census_bps/code/metadata_stage2.py +++ b/models/us_census_bps/code/metadata_stage2.py @@ -84,12 +84,64 @@ def read_state(table_id: str, env: str) -> dict: } +# What each source has actually published, as a coverage date. The +# raw-data-source Update means "what the publisher last released", unlike the +# table Update and the Poll, which are wall clocks. The metropolitan files +# stopped at December 2023, when the series moved to the CBSA basis, so that +# source's last release is not today's period. +SOURCE_MAX_PERIOD = { + "place": "2026-07-01T00:00:00+00:00", + "county": "2026-07-01T00:00:00+00:00", + "cbsa": "2026-07-01T00:00:00+00:00", + "state": "2026-07-01T00:00:00+00:00", + "msa": "2023-12-01T00:00:00+00:00", +} + + +def register_source_update(env: str) -> int: + """Record what the source has published, against each raw data source. + + A recurring dataset needs three records and they mean different things: + the table Update is when Data Basis last refreshed, the raw-data-source + Update is the source's own max coverage date, and the Poll is when we last + looked. The flow writes the latter two, but only on a run with + update_metadata=True, so a dataset tested with metadata off ends up with a + Poll and no source Update. Registering it here means the record exists + from the start. + """ + entity = server.discover_ids(env=env, keys=["entity"])["entity"] + prior = server.get_raw_data_sources(DATASET_SLUG, env=env) + if isinstance(prior, dict): + prior = prior.get("raw_data_sources", []) + by_url = {BASE + path: level for level, (path, *_r) in SOURCES.items()} + for source in prior: + level = by_url[source["url"]] + latest = SOURCE_MAX_PERIOD[level] + server.create_update_update( + entity_id=entity["month"], + frequency=1, + latest=latest, + raw_data_source_id=source["id"], + env=env, + ) + print(f"source update {level:7s} -> {latest[:10]}") + return 0 + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--env", default="staging") + parser.add_argument( + "--source-update-only", + action="store_true", + help="Only (re)write the raw-data-source Update records.", + ) args = parser.parse_args() env = args.env + if args.source_update_only: + return register_source_update(env) + ids = server.discover_ids(env=env, keys=["entity", "status"]) entity = ids["entity"] published = ids["status"]["published"] @@ -248,6 +300,9 @@ def main() -> int: dataset_slug=DATASET_SLUG, table_slugs=TABLE_ORDER, env=env ) print("\n=== pass 3: table order set ===") + + print("\n=== pass 4: raw-data-source updates ===") + register_source_update(env) return 0 diff --git a/pipelines/datasets/us_census_bps/flows.py b/pipelines/datasets/us_census_bps/flows.py new file mode 100644 index 0000000000..5c4110f5a8 --- /dev/null +++ b/pipelines/datasets/us_census_bps/flows.py @@ -0,0 +1,253 @@ +"""Flows for us_census_bps — Prefect 3. + +Census Building Permits Survey. Each run rebuilds the whole series rather than +appending the newest month, for two reasons. The survey revises prior periods, +and a revision can land in any earlier month, not only the trailing few, so a +window-based refresh would leave stale figures behind wherever a correction +fell outside the window. And a full rebuild reuses the onboarding path exactly, +which avoids the class of bug where a trailing window and an upload mode +disagree and history is quietly erased. + +The cost is bounded: about 700 MB of downloads and 25.7 million rows once a +month. + +Deploy: `.github/scripts/deploy_flows.py` auto-discovers `us_census_bps_flow`; +the dev pool strips the schedule, the prod pool activates it. +""" + +import shutil +import tempfile + +from prefect import flow + +from pipelines.datasets.us_census_bps.constants import constants +from pipelines.datasets.us_census_bps.tasks import ( + clean_bps, + clear_staging_prefix, + download_bps, +) +from pipelines.utils.metadata.domain import ( + AllFree, + DateFormat, + FreeLag, + PartBdpro, + YearMonth, + YearOnly, +) +from pipelines.utils.metadata.tasks import ( + commit_source_update_task, + poll_source_for_update_task, + register_table_materialization_task, +) +from pipelines.utils.tasks import ( + rename_flow_run_dataset_table, + run_dbt, + upload_to_gcs, +) + +DATASET_ID = constants.DATASET_ID.value + +# The table whose coverage the source poll compares against. +POLL_TABLE = "permit_state_monthly" + +# Coverage spec per table. +# +# Data Basis paywalls the trailing window of any table that refreshes monthly +# or more often, so the four live monthly tables are part_bdpro with the +# standard six-month free lag: each run recomputes free_end = source_end - +# free_lag, rewrites both DateTimeRanges and re-issues the BigQuery row access +# policies, so the window slides on its own. +# +# part_bdpro requires a free (is_closed=False) *and* a pro (is_closed=True) +# Coverage to already exist on the table, or assert_coverage_topology raises +# before anything is written. Both were registered at onboarding. +# +# The MSA tables ended in 2003 and have nothing recent to gate, so they stay +# free even though one of them is monthly. The annual tables are +# lower-frequency and stay free. `dicionario` has no date column and takes no +# spec at all. +_MONTHLY = YearMonth(year="year", month="month") +_ANNUAL = YearOnly(col="year") +_COVERAGE = { + "permit_place_monthly": PartBdpro( + date_column=_MONTHLY, + date_format=DateFormat.YEAR_MONTH, + free_lag=FreeLag(unit="months", value=6), + ), + "permit_county_monthly": PartBdpro( + date_column=_MONTHLY, + date_format=DateFormat.YEAR_MONTH, + free_lag=FreeLag(unit="months", value=6), + ), + "permit_cbsa_monthly": PartBdpro( + date_column=_MONTHLY, + date_format=DateFormat.YEAR_MONTH, + free_lag=FreeLag(unit="months", value=6), + ), + "permit_state_monthly": PartBdpro( + date_column=_MONTHLY, + date_format=DateFormat.YEAR_MONTH, + free_lag=FreeLag(unit="months", value=6), + ), + "permit_msa_monthly": AllFree( + date_column=_MONTHLY, date_format=DateFormat.YEAR_MONTH + ), + "permit_place_annual": AllFree( + date_column=_ANNUAL, date_format=DateFormat.YEAR + ), + "permit_county_annual": AllFree( + date_column=_ANNUAL, date_format=DateFormat.YEAR + ), + "permit_cbsa_annual": AllFree( + date_column=_ANNUAL, date_format=DateFormat.YEAR + ), + "permit_msa_annual": AllFree( + date_column=_ANNUAL, date_format=DateFormat.YEAR + ), + "permit_state_annual": AllFree( + date_column=_ANNUAL, date_format=DateFormat.YEAR + ), +} + + +def _materialize(result: dict, bucket: str, target: str) -> None: + """Upload every table's Parquet and rebuild it, then test the whole set. + + Every table is built before any test runs. The dictionary-coverage and + relationship tests read sibling models, so interleaving run and test per + table fails the first table's tests against a sibling that does not exist + yet — invisible in a re-run where a stale sibling survives, and fatal in a + clean environment. + + Args: + result: Output of :func:`clean_bps`, mapping table slug to its path. + bucket: GCS bucket to stage into. + target: dbt target. + """ + tables = constants.TABLES.value + for table in tables: + # append, never overwrite: overwrite calls tb.delete(mode="all"), + # which drops the materialized production table even from a dev run. + # The prefix clear does the part of overwrite that is actually wanted. + clear_staging_prefix(table_id=table, bucket_name=bucket) + upload_to_gcs( + data_path=result[table], + dataset_id=DATASET_ID, + table_id=table, + bucket_name=bucket, + dump_mode="append", + source_format="parquet", + ) + run_dbt( + dataset_id=DATASET_ID, + table_id=table, + dbt_command="run", + target=target, + ) + for table in tables: + run_dbt( + dataset_id=DATASET_ID, + table_id=table, + dbt_command="test", + target=target, + ) + + +@flow(name="us_census_bps", log_prints=True) +def us_census_bps_flow( + materialize_to_prod: bool = True, + update_metadata: bool = True, + force_run: bool = False, +) -> None: + """Rebuild the Building Permits Survey tables from the published files. + + The source poll short-circuits the run when the Census Bureau has not + published a new month, which makes a scheduled run a cheap no-op between + releases. A completed run is therefore not evidence that anything was + ingested — read the logs or check whether coverage moved. + + Args: + materialize_to_prod: Continue past the dev materialization to write the + prod staging bucket and run dbt against ``target="prod"``. Set + False to exercise only the dev half; the default writes production. + update_metadata: After a successful prod materialization, register + table coverage and commit the source update. No effect when + ``materialize_to_prod`` is False. + force_run: Materialize even when the poll reports no new month. + """ + # pyrefly: ignore [unused-coroutine] + rename_flow_run_dataset_table( + prefix="Dump: ", dataset_id=DATASET_ID, table_id="bps" + ) + + work_dir = tempfile.mkdtemp(prefix="us_census_bps_") + try: + input_dir = download_bps(work_dir=work_dir) + result = clean_bps(work_dir=work_dir, input_dir=input_dir) + max_year_month = result["max_year_month"] + print( + f"source max period {max_year_month}; rows {result['row_counts']}" + ) + + has_new_data = poll_source_for_update_task( + dataset_id=DATASET_ID, + table_id=POLL_TABLE, + source_max_date=max_year_month, + env="prod", + date_format="%Y-%m", + compare_against="coverage", + ) + if not has_new_data and not force_run: + print("no new period published; nothing to do") + return + + if not materialize_to_prod: + _materialize(result, "basedosdados-dev", "dev") + return + + commit_source_update_task( + dataset_id=DATASET_ID, + table_id=POLL_TABLE, + source_max_date=max_year_month, + env="prod", + date_format="%Y-%m", + update_metadata=update_metadata, + materialize_after_dump=materialize_to_prod, + ) + + _materialize(result, "basedosdados", "prod") + + if update_metadata: + for table, coverage in _COVERAGE.items(): + register_table_materialization_task( + dataset_id=DATASET_ID, + table_id=table, + coverage=coverage, + env="prod", + bq_project="basedosdados", + ) + finally: + # Covers both early returns and any exception. The download is about + # 700 MB; a process worker reuses its filesystem between runs. + shutil.rmtree(work_dir, ignore_errors=True) + + +# The Census Bureau publishes a reference month about four weeks later, in the +# New Residential Construction release around the 17th. Poll across a few days +# of that window; the source-poll guard no-ops until a new period lands. +# 15:25 BRT is an unused slot — piling flows onto the same instant makes them +# compete for BigQuery slots and fail together. +# pyrefly: ignore [missing-attribute] +us_census_bps_flow.deploy_schedules = [ + {"cron": "25 15 17,18,19,20,21,22 * *", "timezone": "America/Sao_Paulo"} +] +# `memory` alone is silently dropped by the work pool's job template, which +# defaults to 4Gi; `memory_limit` is the key the pod actually gets. The clean +# step holds one 400k-row buffer at a time, but the downloaded files are about +# 700 MB on disk. +# pyrefly: ignore [missing-attribute] +us_census_bps_flow.job_variables = { + "memory": "8Gi", + "memory_limit": "8Gi", + "memory_request": "2Gi", +} diff --git a/pipelines/datasets/us_census_bps/tasks.py b/pipelines/datasets/us_census_bps/tasks.py new file mode 100644 index 0000000000..f26bddeba1 --- /dev/null +++ b/pipelines/datasets/us_census_bps/tasks.py @@ -0,0 +1,94 @@ +"""Prefect 3 tasks for us_census_bps — thin wrappers over utils.py.""" + +import time +from pathlib import Path + +import basedosdados as bd +from prefect import task + +from pipelines.datasets.us_census_bps.constants import constants +from pipelines.datasets.us_census_bps.utils import ( + clean_all, + download_all, + latest_monthly_period, +) + + +@task(retries=2, retry_delay_seconds=60) +def download_bps(work_dir: str) -> str: + """Download every published Building Permits Survey file. + + Retries: www2.census.gov drops connections under sustained concurrency, + and ``download_all`` raises rather than returning a partial set, so a + retry is the difference between a complete table and one with months + silently missing. + + Args: + work_dir: Directory to download into; files land in ``/input``. + + Returns: + The input directory path, as a string. + """ + input_dir = Path(work_dir) / "input" + download_all(input_dir, through_year=time.gmtime().tm_year) + return str(input_dir) + + +@task +def clean_bps(work_dir: str, input_dir: str) -> dict: + """Rebuild every table from the downloaded files. + + Args: + work_dir: Directory to write into; tables land under ``/output``. + input_dir: Directory holding the downloaded files. + + Returns: + A mapping of table slug to its partitioned output directory, plus + ``"max_year_month"``, the latest ``YYYY-MM`` the source has published, + which drives the source poll. + """ + output = Path(work_dir) / "output" + counts = clean_all(Path(input_dir), output) + result: dict = { + table: str(output / table) for table in constants.TABLES.value + } + result["row_counts"] = counts + result["max_year_month"] = latest_monthly_period(Path(input_dir)) + return result + + +@task(retries=2, retry_delay_seconds=30) +def clear_staging_prefix(table_id: str, bucket_name: str) -> None: + """Delete a table's staging objects before a full rebuild uploads new ones. + + Every run rebuilds the whole series, so the previous run's objects have to + go. ``dump_mode="overwrite"`` would do it, but it also calls + ``tb.delete(mode="all")``, which drops the **materialized production + table** — and it fires from the dev half too, because ``bd.Table`` resolves + its BigQuery projects from the pod config rather than from ``bucket_name``. + A dev-only smoke run has silently deleted a production table this way + before. + + ``dump_mode="append"`` avoids that, but it uploads object by object and + replaces only the names it writes. A run that produces fewer part files + than the last one would leave the extra ones behind, and the external table + would read them as real rows. Clearing the prefix first closes that gap + without touching any BigQuery table. + + Args: + table_id: Table slug. + bucket_name: Bucket holding the staging prefix. + """ + storage = bd.Storage( + dataset_id=constants.DATASET_ID.value, + table_id=table_id, + bucket_name=bucket_name, + billing_project_id=bucket_name, + ) + storage.delete_table( + mode="staging", bucket_name=bucket_name, not_found_ok=True + ) + print( + f"cleared gs://{bucket_name}/staging/" + f"{constants.DATASET_ID.value}/{table_id}" + ) diff --git a/pipelines/datasets/us_census_bps/utils.py b/pipelines/datasets/us_census_bps/utils.py index 317f2cec23..a266d43872 100644 --- a/pipelines/datasets/us_census_bps/utils.py +++ b/pipelines/datasets/us_census_bps/utils.py @@ -13,10 +13,15 @@ from __future__ import annotations +import concurrent.futures as cf import csv import logging import re import shutil +import time +import urllib.error +import urllib.parse +import urllib.request from collections.abc import Iterator from pathlib import Path @@ -807,3 +812,174 @@ def clean_all( for table, count in sorted(dropped.items()): log.info(f"{table}: dropped {count:,} rows with no geography code") return totals + + +# ── download ──────────────────────────────────────────────────────────────── + +BASE_URL = constants.BASE_URL.value +HEADERS = constants.HEADERS.value + + +def download_targets(through_year: int) -> list[tuple[str, Path]]: + """Build the (url, destination) list for every published BPS file. + + Args: + through_year: Last survey year to ask for. Years the Census Bureau has + not published yet simply answer 404 and are skipped. + + Returns: + One (url, destination) pair per candidate file. + """ + out: list[tuple[str, Path]] = [] + + def add(directory: str, prefix: str, level: str, kind: str, first: int): + for year in range(first, through_year + 1): + names = ( + [f"{prefix}{year % 100:02d}{m:02d}c.txt" for m in range(1, 13)] + if kind == "monthly" + else [f"{prefix}{year}a.txt"] + ) + for name in names: + url = BASE_URL + urllib.parse.quote(f"{directory}/{name}") + out.append((url, Path(level) / name)) + + for level, directory in constants.GEO_DIRS.value.items(): + prefix = constants.GEO_PREFIXES.value[level] + for kind in ("monthly", "annual"): + add( + directory, + prefix, + level, + kind, + constants.FIRST_YEAR.value[(level, kind)], + ) + for directory, prefix in constants.PLACE_REGIONS.value.items(): + for kind in ("monthly", "annual"): + add( + f"Place/{directory}", + prefix, + "place", + kind, + constants.FIRST_YEAR.value[("place", kind)], + ) + return out + + +def fetch_file(url: str, dest: Path, tries: int = 5) -> tuple[str, str]: + """Download one BPS file, distinguishing a missing file from a rejection. + + Two Census server behaviours would otherwise lose data silently. A file + that does not exist answers HTTP 404 with an HTML page, so the body is + checked before anything is written. Separately, the site firewall rejects a + few perfectly valid URLs with **HTTP 200** and a "Request Rejected" HTML + page; treating that as a missing file dropped four place months and one + metropolitan month on the first full run. The rejection is cached against + the exact URL, so every retry carries a cache-busting parameter. + + Args: + url: Absolute URL of the file. + dest: Destination path; parent directories are created. + tries: Attempts before giving up. + + Returns: + ``(status, detail)``; status is "ok", "cached", "missing" or "failed". + Only an HTTP 404 yields "missing". + """ + if dest.exists() and dest.stat().st_size > 0: + return "cached", "" + detail = "" + for attempt in range(tries): + target = url if attempt == 0 else f"{url}?attempt={attempt}" + request = urllib.request.Request(target, headers=HEADERS) + try: + with urllib.request.urlopen(request, timeout=300) as response: + body = response.read() + except urllib.error.HTTPError as exc: + if exc.code == 404: + return "missing", "HTTP 404" + detail = f"HTTP {exc.code}" + time.sleep(2 * (attempt + 1)) + continue + except Exception as exc: + detail = f"{type(exc).__name__}: {exc}" + time.sleep(2 * (attempt + 1)) + continue + head = body[:1000].lower() + if b" dict[str, int]: + """Download every published BPS file into ``input_dir``. + + Args: + input_dir: Directory to download into. + through_year: Last survey year to ask for. + workers: Concurrent downloads. + + Returns: + Counts by status. + + Raises: + RuntimeError: If any file failed for a reason other than a 404. A + partial download must stop the run rather than quietly produce a + table with months missing. + """ + jobs = [ + (url, input_dir / rel) for url, rel in download_targets(through_year) + ] + counts = {"ok": 0, "cached": 0, "missing": 0, "failed": 0} + failures: list[str] = [] + with cf.ThreadPoolExecutor(max_workers=workers) as pool: + futures = {pool.submit(fetch_file, u, d): u for u, d in jobs} + for future in cf.as_completed(futures): + status, detail = future.result() + counts[status] += 1 + if status == "failed": + failures.append(f"{futures[future]} -- {detail}") + log.info(f"download: {counts}") + if failures: + raise RuntimeError( + f"{len(failures)} files failed to download (not 404s): " + + "; ".join(failures[:10]) + ) + return counts + + +def latest_monthly_period(input_dir: Path) -> str: + """Return the most recent monthly survey period present, as ``YYYY-MM``. + + Read from the state files, the smallest monthly series and the one that + goes back furthest, so it is the cheapest reliable signal of what the + Census Bureau has published. + + Args: + input_dir: Directory holding the downloaded files. + + Returns: + The latest period, e.g. ``"2026-07"``. + + Raises: + ValueError: If no monthly state file was downloaded. + """ + periods = [] + for path in (input_dir / "state").glob("st*c.txt"): + _level, periodicity, year, month = parse_filename(path.name) + if periodicity == "monthly" and month is not None: + periods.append((year, month)) + if not periods: + raise ValueError(f"no monthly state files under {input_dir}") + year, month = max(periods) + return f"{year:04d}-{month:02d}" From 0fc2003cee271306af078b5f5c6e264f09791ae9 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 9 Sep 2026 20:13:40 +1000 Subject: [PATCH 6/7] fix(us_census_bps): back off properly when the Census server throttles The first dev run drew HTTP 429 on three metro files partway through the download. `download_all` raised rather than returning a partial set and the task retry recovered it, which is the behaviour I want, but the per-file backoff was sized for a dropped connection: 2 to 10 seconds across its attempts. Under heavier throttling both task retries would burn out. 408, 429 and 503 now back off 5, 10, 20, 40, 80, 120 seconds and honour a `Retry-After` header when the server sends one, capped at two minutes. Default concurrency drops from six to four, since six was enough to draw the throttling from a datacentre, where requests leave faster than they do from a laptop. --- pipelines/datasets/us_census_bps/utils.py | 43 +++++++++++++++++------ 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/pipelines/datasets/us_census_bps/utils.py b/pipelines/datasets/us_census_bps/utils.py index a266d43872..d930a192c0 100644 --- a/pipelines/datasets/us_census_bps/utils.py +++ b/pipelines/datasets/us_census_bps/utils.py @@ -865,16 +865,33 @@ def add(directory: str, prefix: str, level: str, kind: str, first: int): return out -def fetch_file(url: str, dest: Path, tries: int = 5) -> tuple[str, str]: +# Statuses that mean "slow down", not "broken". They need a far longer pause +# than a dropped connection: www2.census.gov returned 429 on three files +# partway through a full download from a datacentre, where requests leave +# faster than they do from a laptop. +THROTTLE_CODES = frozenset({408, 429, 503}) + + +def _throttle_delay(exc: urllib.error.HTTPError, attempt: int) -> float: + """Seconds to wait after a throttling response, honouring Retry-After.""" + header = exc.headers.get("Retry-After") if exc.headers else None + if header and header.strip().isdigit(): + return min(120.0, float(header.strip())) + return min(120.0, 5.0 * (2**attempt)) + + +def fetch_file(url: str, dest: Path, tries: int = 6) -> tuple[str, str]: """Download one BPS file, distinguishing a missing file from a rejection. - Two Census server behaviours would otherwise lose data silently. A file - that does not exist answers HTTP 404 with an HTML page, so the body is - checked before anything is written. Separately, the site firewall rejects a - few perfectly valid URLs with **HTTP 200** and a "Request Rejected" HTML - page; treating that as a missing file dropped four place months and one - metropolitan month on the first full run. The rejection is cached against - the exact URL, so every retry carries a cache-busting parameter. + Three Census server behaviours would otherwise lose data or fail the run. + A file that does not exist answers HTTP 404 with an HTML page, so the body + is checked before anything is written. The site firewall rejects a few + perfectly valid URLs with **HTTP 200** and a "Request Rejected" HTML page; + treating that as a missing file dropped four place months and one + metropolitan month on the first full run, and since the rejection is cached + against the exact URL, every retry carries a cache-busting parameter. And + the server throttles a sustained download with 429, which needs a much + longer pause than a network flake. Args: url: Absolute URL of the file. @@ -898,7 +915,10 @@ def fetch_file(url: str, dest: Path, tries: int = 5) -> tuple[str, str]: if exc.code == 404: return "missing", "HTTP 404" detail = f"HTTP {exc.code}" - time.sleep(2 * (attempt + 1)) + if exc.code in THROTTLE_CODES: + time.sleep(_throttle_delay(exc, attempt)) + else: + time.sleep(2 * (attempt + 1)) continue except Exception as exc: detail = f"{type(exc).__name__}: {exc}" @@ -920,14 +940,15 @@ def fetch_file(url: str, dest: Path, tries: int = 5) -> tuple[str, str]: def download_all( - input_dir: Path, through_year: int, workers: int = 6 + input_dir: Path, through_year: int, workers: int = 4 ) -> dict[str, int]: """Download every published BPS file into ``input_dir``. Args: input_dir: Directory to download into. through_year: Last survey year to ask for. - workers: Concurrent downloads. + workers: Concurrent downloads. Kept low: six was enough to draw + HTTP 429 from the server partway through a full run. Returns: Counts by status. From e507ba8e15a821dbbf2f72ae3d9d156fad5bed98 Mon Sep 17 00:00:00 2001 From: Ricardo Dahis Date: Wed, 9 Sep 2026 20:31:55 +1000 Subject: [PATCH 7/7] fix(us_census_bps): recover throttled downloads inside the task, not via retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second dev run consumed **both** Prefect task retries on HTTP 429 before the download completed on its third attempt — one more throttled file and the run would have failed. Leaning on the task retry was the wrong shape: it restarts the whole 3,400-file sweep to recover a handful of stragglers, and each restart applies the same concurrent pressure that caused the throttling. `download_all` now finishes its concurrent sweep, then retries whatever failed one file at a time, pausing 90 seconds before each pass and a second between files, for up to three passes. Only then does it raise. The cached-file check means the sweep itself is nearly free on a retry, so this costs nothing on a clean run. The task retry stays as a genuine last resort, raised to 3 attempts at 180 seconds since it now only fires when something is properly wrong. Verified with a stubbed fetch: two files failing the concurrent pass are recovered by the sequential pass, and the counts come back consistent. --- pipelines/datasets/us_census_bps/tasks.py | 2 +- pipelines/datasets/us_census_bps/utils.py | 47 ++++++++++++++++++++--- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/pipelines/datasets/us_census_bps/tasks.py b/pipelines/datasets/us_census_bps/tasks.py index f26bddeba1..bee3be8615 100644 --- a/pipelines/datasets/us_census_bps/tasks.py +++ b/pipelines/datasets/us_census_bps/tasks.py @@ -14,7 +14,7 @@ ) -@task(retries=2, retry_delay_seconds=60) +@task(retries=3, retry_delay_seconds=180) def download_bps(work_dir: str) -> str: """Download every published Building Permits Survey file. diff --git a/pipelines/datasets/us_census_bps/utils.py b/pipelines/datasets/us_census_bps/utils.py index d930a192c0..7ed5211100 100644 --- a/pipelines/datasets/us_census_bps/utils.py +++ b/pipelines/datasets/us_census_bps/utils.py @@ -871,6 +871,11 @@ def add(directory: str, prefix: str, level: str, kind: str, first: int): # faster than they do from a laptop. THROTTLE_CODES = frozenset({408, 429, 503}) +# After the concurrent sweep, stragglers get this many sequential passes, each +# preceded by a pause long enough for the server's throttle window to reset. +RETRY_PASSES = 3 +RETRY_PASS_PAUSE = 90.0 + def _throttle_delay(exc: urllib.error.HTTPError, attempt: int) -> float: """Seconds to wait after a throttling response, honouring Retry-After.""" @@ -962,19 +967,49 @@ def download_all( (url, input_dir / rel) for url, rel in download_targets(through_year) ] counts = {"ok": 0, "cached": 0, "missing": 0, "failed": 0} - failures: list[str] = [] + leftover: list[tuple[str, Path]] = [] + details: dict[str, str] = {} with cf.ThreadPoolExecutor(max_workers=workers) as pool: - futures = {pool.submit(fetch_file, u, d): u for u, d in jobs} + futures = {pool.submit(fetch_file, u, d): (u, d) for u, d in jobs} for future in cf.as_completed(futures): status, detail = future.result() counts[status] += 1 if status == "failed": - failures.append(f"{futures[future]} -- {detail}") + url, dest = futures[future] + leftover.append((url, dest)) + details[url] = detail log.info(f"download: {counts}") - if failures: + + # A concurrent sweep over 3,400 files reliably draws HTTP 429 near the + # end, and the handful of stragglers succeed once the pressure is off. + # Retrying them here, one at a time and slowly, keeps the Prefect task + # retry as a last resort rather than the mechanism the run depends on: on + # the first two dev runs the concurrent pass alone consumed both retries. + for attempt in range(1, RETRY_PASSES + 1): + if not leftover: + break + log.info( + f"retry pass {attempt}: {len(leftover)} files, one at a time, " + f"after a {RETRY_PASS_PAUSE:.0f}s pause" + ) + time.sleep(RETRY_PASS_PAUSE) + still: list[tuple[str, Path]] = [] + for url, dest in leftover: + status, detail = fetch_file(url, dest) + if status == "failed": + details[url] = detail + still.append((url, dest)) + else: + counts["failed"] -= 1 + counts[status] += 1 + time.sleep(1.0) + leftover = still + log.info(f"download after retries: {counts}") + + if leftover: raise RuntimeError( - f"{len(failures)} files failed to download (not 404s): " - + "; ".join(failures[:10]) + f"{len(leftover)} files failed to download (not 404s): " + + "; ".join(f"{u} -- {details[u]}" for u, _dest in leftover[:10]) ) return counts