From d9190aa7bbf8f2511ab97ba77144ee00ea5a78aa Mon Sep 17 00:00:00 2001 From: Pedro Castro Date: Fri, 7 Aug 2026 17:33:12 -0300 Subject: [PATCH] [update]: Saeb 2025 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Atualizando três tabelas de br_inep_saeb: - brasil - uf - municipio --- models/br_inep_saeb/br_inep_saeb__brasil.sql | 1 + .../br_inep_saeb/br_inep_saeb__municipio.sql | 1 + models/br_inep_saeb/br_inep_saeb__uf.sql | 1 - .../br_inep_saeb/code/br_inep_saeb_brasil.py | 64 +++++++++++-------- .../code/br_inep_saeb_municipio.py | 62 +++++++++++++----- models/br_inep_saeb/code/br_inep_saeb_uf.py | 61 +++++++++++++----- 6 files changed, 129 insertions(+), 61 deletions(-) diff --git a/models/br_inep_saeb/br_inep_saeb__brasil.sql b/models/br_inep_saeb/br_inep_saeb__brasil.sql index 9362c75b1c..eabaf04c8d 100644 --- a/models/br_inep_saeb/br_inep_saeb__brasil.sql +++ b/models/br_inep_saeb/br_inep_saeb__brasil.sql @@ -1,6 +1,7 @@ -- - Atualização 10/12/2025 {{ config(alias="brasil", schema="br_inep_saeb", materialized="table") }} + select safe_cast(ano as int64) ano, safe_cast(rede as string) rede, diff --git a/models/br_inep_saeb/br_inep_saeb__municipio.sql b/models/br_inep_saeb/br_inep_saeb__municipio.sql index ea64f42a54..0e27614542 100644 --- a/models/br_inep_saeb/br_inep_saeb__municipio.sql +++ b/models/br_inep_saeb/br_inep_saeb__municipio.sql @@ -7,6 +7,7 @@ ) }} + select safe_cast(ano as int64) ano, safe_cast(rede as string) rede, diff --git a/models/br_inep_saeb/br_inep_saeb__uf.sql b/models/br_inep_saeb/br_inep_saeb__uf.sql index 708624cfd9..5c7ac4e912 100644 --- a/models/br_inep_saeb/br_inep_saeb__uf.sql +++ b/models/br_inep_saeb/br_inep_saeb__uf.sql @@ -1,7 +1,6 @@ -- - Atualização 10/12/2025 {{ config(alias="uf", schema="br_inep_saeb", materialized="table") }} - select safe_cast(ano as int64) ano, safe_cast(rede as string) rede, diff --git a/models/br_inep_saeb/code/br_inep_saeb_brasil.py b/models/br_inep_saeb/code/br_inep_saeb_brasil.py index 4a526f86b3..a6670f37d2 100644 --- a/models/br_inep_saeb/code/br_inep_saeb_brasil.py +++ b/models/br_inep_saeb/code/br_inep_saeb_brasil.py @@ -1,50 +1,59 @@ import os -import zipfile from pathlib import Path import basedosdados as bd import pandas as pd import requests -# pyrefly: ignore [missing-import] -from utils import ( - # pyrefly: ignore [missing-module-attribute] +from models.br_inep_saeb.code.utils import ( convert_to_pd_dtype, - # pyrefly: ignore [missing-module-attribute] get_disciplina_serie, - # pyrefly: ignore [missing-module-attribute] get_nivel_serie_disciplina, ) -CWD = Path(os.getcwd()).parent +input = Path("input") / "br_inep_saeb" +output = Path("output") / "br_inep_saeb" -INPUT = CWD / "input" -OUTPUT = CWD / "output" +os.makedirs(input, exist_ok=True) +os.makedirs(output, exist_ok=True) -os.makedirs(INPUT, exist_ok=True) -os.makedirs(OUTPUT, exist_ok=True) +url = "https://download.inep.gov.br/saeb/resultados/saeb_2025_brasil_estados_municipios_censitario.xlsx" +xlsx_file = "saeb_2025.xlsx" -URL = "https://download.inep.gov.br/microdados/planilhas_de_resultados_20250507.zip" -r = requests.get( - URL, headers={"User-Agent": "Mozilla/5.0"}, verify=False, stream=True -) +def download(url: str, max_attempts: int = 3, timeout: float = 120): + for attempt in range(1, max_attempts + 1): + try: + r = requests.get( + url, + headers={"User-Agent": "Mozilla/5.0"}, + verify=False, + stream=True, + timeout=timeout, + ) + r.raise_for_status() + return r + except requests.exceptions.ConnectionError as exc: + print( + f"[attempt {attempt}/{max_attempts}] failed: {exc!r} - retrying.." + ) + raise Exception(f"All {max_attempts} attempts failed for {url}") + -with open(INPUT / "2023.zip", "wb") as fd: +r = download(url) + +with open(input / xlsx_file, "wb") as fd: for chunk in r.iter_content(chunk_size=128): fd.write(chunk) -with zipfile.ZipFile(INPUT / "2023.zip") as z: - z.extractall(INPUT) - br_saeb_latest = pd.read_excel( - INPUT / "PLANILHAS DE RESULTADOS_20250507" / "TS_BRASIL_20250507.xlsx", + input / xlsx_file, + sheet_name="Brasil", dtype=str, ) br_saeb_latest.head() - br_saeb_latest = ( br_saeb_latest.drop(0, axis="index") .pipe(lambda df: df.loc[df["CAPITAL"] == "Total"]) @@ -123,6 +132,7 @@ index=["DEPENDENCIA_ADM", "LOCALIZACAO", "disciplina", "serie"], columns="nivel", values="value", + aggfunc="first", ) .reset_index() .merge( @@ -156,7 +166,7 @@ ) ) -br_saeb_latest_output["ano"] = 2023 +br_saeb_latest_output["ano"] = 2025 br_saeb_latest_output.head() @@ -182,13 +192,15 @@ billing_project_id="basedosdados-dev", ) -pd.concat([br_saeb_latest_output, upstream_df]).to_csv( # type: ignore - os.path.join(OUTPUT, "brasil.csv"), index=False -) +br_saeb_updated = pd.concat([br_saeb_latest_output, upstream_df]) + +br_saeb_updated.to_csv(os.path.join(output, "brasil.csv"), index=False) + +print(br_saeb_updated) # Update table tb.create( - os.path.join(OUTPUT, "brasil.csv"), + output / "brasil.csv", if_table_exists="replace", if_storage_data_exists="replace", ) diff --git a/models/br_inep_saeb/code/br_inep_saeb_municipio.py b/models/br_inep_saeb/code/br_inep_saeb_municipio.py index 80ced2fcc1..98fb6b62a5 100644 --- a/models/br_inep_saeb/code/br_inep_saeb_municipio.py +++ b/models/br_inep_saeb/code/br_inep_saeb_municipio.py @@ -3,29 +3,54 @@ import basedosdados as bd import pandas as pd +import requests -# pyrefly: ignore [missing-import] -from utils import ( - # pyrefly: ignore [missing-module-attribute] +from models.br_inep_saeb.code.utils import ( convert_to_pd_dtype, - # pyrefly: ignore [missing-module-attribute] drop_empty_lines, - # pyrefly: ignore [missing-module-attribute] get_disciplina_serie, - # pyrefly: ignore [missing-module-attribute] get_nivel_serie_disciplina, ) -CWD = Path(os.getcwd()).parent +input = Path("input") / "br_inep_saeb" +output = Path("output") / "br_inep_saeb" -INPUT = CWD / "input" -OUTPUT = CWD / "output" +os.makedirs(input, exist_ok=True) +os.makedirs(output, exist_ok=True) + +url = "https://download.inep.gov.br/saeb/resultados/saeb_2025_brasil_estados_municipios_censitario.xlsx" +xlsx_file = "saeb_2025.xlsx" + + +def download(url: str, max_attempts: int = 3, timeout: float = 120): + for attempt in range(1, max_attempts + 1): + try: + r = requests.get( + url, + headers={"User-Agent": "Mozilla/5.0"}, + verify=False, + stream=True, + timeout=timeout, + ) + r.raise_for_status() + return r + except requests.exceptions.ConnectionError as exc: + print( + f"[attempt {attempt}/{max_attempts}] failed: {exc!r} - retrying.." + ) + raise Exception(f"All {max_attempts} attempts failed for {url}") + + +r = download(url) + +with open(input / xlsx_file, "wb") as fd: + for chunk in r.iter_content(chunk_size=128): + fd.write(chunk) -os.makedirs(INPUT, exist_ok=True) -os.makedirs(OUTPUT, exist_ok=True) mun_saeb_latest = pd.read_excel( - INPUT / "PLANILHAS DE RESULTADOS_20250507" / "TS_MUNICIPIO_20250507.xlsx", + input / xlsx_file, + sheet_name="Municípios", dtype=str, ) @@ -123,6 +148,7 @@ ], columns="nivel", values="value", + aggfunc="first", ) .reset_index() .merge( @@ -189,7 +215,7 @@ mun_saeb_latest_output = drop_empty_lines(mun_saeb_latest_output) -mun_saeb_latest_output["ano"] = 2023 +mun_saeb_latest_output["ano"] = 2025 mun_saeb_latest_output.head() @@ -215,13 +241,15 @@ billing_project_id="basedosdados-dev", ) -pd.concat([mun_saeb_latest_output, upstream_df]).to_csv( # type: ignore - os.path.join(OUTPUT, "municipio.csv"), index=False -) +mun_saeb_updated = pd.concat([mun_saeb_latest_output, upstream_df]) + +mun_saeb_updated.to_csv(os.path.join(output, "municipio.csv"), index=False) + +print(mun_saeb_updated) # Update table tb.create( - os.path.join(OUTPUT, "municipio.csv"), + output / "municipio.csv", if_table_exists="replace", if_storage_data_exists="replace", ) diff --git a/models/br_inep_saeb/code/br_inep_saeb_uf.py b/models/br_inep_saeb/code/br_inep_saeb_uf.py index 5a1eb847fc..d277272d90 100644 --- a/models/br_inep_saeb/code/br_inep_saeb_uf.py +++ b/models/br_inep_saeb/code/br_inep_saeb_uf.py @@ -3,29 +3,54 @@ import basedosdados as bd import pandas as pd +import requests -# pyrefly: ignore [missing-import] -from utils import ( - # pyrefly: ignore [missing-module-attribute] +from models.br_inep_saeb.code.utils import ( convert_to_pd_dtype, - # pyrefly: ignore [missing-module-attribute] drop_empty_lines, - # pyrefly: ignore [missing-module-attribute] get_disciplina_serie, - # pyrefly: ignore [missing-module-attribute] get_nivel_serie_disciplina, ) -CWD = Path(os.getcwd()).parent +input = Path("input") / "br_inep_saeb" +output = Path("output") / "br_inep_saeb" -INPUT = CWD / "input" -OUTPUT = CWD / "output" +os.makedirs(input, exist_ok=True) +os.makedirs(output, exist_ok=True) + +url = "https://download.inep.gov.br/saeb/resultados/saeb_2025_brasil_estados_municipios_censitario.xlsx" +xlsx_file = "saeb_2025.xlsx" + + +def download(url: str, max_attempts: int = 3, timeout: float = 120): + for attempt in range(1, max_attempts + 1): + try: + r = requests.get( + url, + headers={"User-Agent": "Mozilla/5.0"}, + verify=False, + stream=True, + timeout=timeout, + ) + r.raise_for_status() + return r + except requests.exceptions.ConnectionError as exc: + print( + f"[attempt {attempt}/{max_attempts}] failed: {exc!r} - retrying.." + ) + raise Exception(f"All {max_attempts} attempts failed for {url}") + + +r = download(url) + +with open(input / xlsx_file, "wb") as fd: + for chunk in r.iter_content(chunk_size=128): + fd.write(chunk) -os.makedirs(INPUT, exist_ok=True) -os.makedirs(OUTPUT, exist_ok=True) ufs_saeb_latest = pd.read_excel( - INPUT / "PLANILHAS DE RESULTADOS_20250507" / "TS_UF_20250507.xlsx", + input / xlsx_file, + sheet_name="Estados", dtype=str, ) @@ -113,6 +138,7 @@ ], columns="nivel", values="value", + aggfunc="first", ) .reset_index() .merge( @@ -171,7 +197,7 @@ ) # Add column ano -ufs_saeb_latest_output["ano"] = 2023 +ufs_saeb_latest_output["ano"] = 2025 ufs_saeb_latest_output.head() @@ -201,13 +227,14 @@ upstream_df["serie"].unique() -pd.concat([ufs_saeb_latest_output, upstream_df]).to_csv( # type: ignore - os.path.join(OUTPUT, "uf.csv"), index=False -) +ufs_saeb_updated = pd.concat([ufs_saeb_latest_output, upstream_df]) +ufs_saeb_updated.to_csv(os.path.join(output, "uf.csv"), index=False) + +print(ufs_saeb_updated) # Update table tb.create( - os.path.join(OUTPUT, "uf.csv"), + output / "uf.csv", if_table_exists="replace", if_storage_data_exists="replace", )