[update]: Saeb 2025

models/br_inep_saeb/br_inep_saeb__brasil.sql

@@ -1,6 +1,7 @@
 -- - Atualização 10/12/2025
 {{ config(alias="brasil", schema="br_inep_saeb", materialized="table") }}
 
+
 select
     safe_cast(ano as int64) ano,
     safe_cast(rede as string) rede,

models/br_inep_saeb/br_inep_saeb__municipio.sql

@@ -7,6 +7,7 @@
     )
 }}
 
+
 select
     safe_cast(ano as int64) ano,
     safe_cast(rede as string) rede,

models/br_inep_saeb/br_inep_saeb__uf.sql

@@ -1,7 +1,6 @@
 -- - Atualização 10/12/2025
 {{ config(alias="uf", schema="br_inep_saeb", materialized="table") }}
 
-
 select
     safe_cast(ano as int64) ano,
     safe_cast(rede as string) rede,

models/br_inep_saeb/code/br_inep_saeb_brasil.py

@@ -1,50 +1,59 @@
 import os
-import zipfile
 from pathlib import Path
 
 import basedosdados as bd
 import pandas as pd
 import requests
 
-# pyrefly: ignore [missing-import]
-from utils import (
-    # pyrefly: ignore [missing-module-attribute]
+from models.br_inep_saeb.code.utils import (
     convert_to_pd_dtype,
-    # pyrefly: ignore [missing-module-attribute]
     get_disciplina_serie,
-    # pyrefly: ignore [missing-module-attribute]
     get_nivel_serie_disciplina,
 )
 
-CWD = Path(os.getcwd()).parent
+input = Path("input") / "br_inep_saeb"
+output = Path("output") / "br_inep_saeb"
 
-INPUT = CWD / "input"
-OUTPUT = CWD / "output"
+os.makedirs(input, exist_ok=True)
+os.makedirs(output, exist_ok=True)
 
-os.makedirs(INPUT, exist_ok=True)
-os.makedirs(OUTPUT, exist_ok=True)
+url = "https://download.inep.gov.br/saeb/resultados/saeb_2025_brasil_estados_municipios_censitario.xlsx"
+xlsx_file = "saeb_2025.xlsx"
 
-URL = "https://download.inep.gov.br/microdados/planilhas_de_resultados_20250507.zip"
 
-r = requests.get(
-    URL, headers={"User-Agent": "Mozilla/5.0"}, verify=False, stream=True
-)
+def download(url: str, max_attempts: int = 3, timeout: float = 120):
+    for attempt in range(1, max_attempts + 1):
+        try:
+            r = requests.get(
+                url,
+                headers={"User-Agent": "Mozilla/5.0"},
+                verify=False,
+                stream=True,
+                timeout=timeout,
+            )
+            r.raise_for_status()
+            return r
+        except requests.exceptions.ConnectionError as exc:
+            print(
+                f"[attempt {attempt}/{max_attempts}] failed: {exc!r} - retrying.."
+            )
+    raise Exception(f"All {max_attempts} attempts failed for {url}")
+
 
-with open(INPUT / "2023.zip", "wb") as fd:
+r = download(url)
+
+with open(input / xlsx_file, "wb") as fd:
     for chunk in r.iter_content(chunk_size=128):
         fd.write(chunk)
 
-with zipfile.ZipFile(INPUT / "2023.zip") as z:
-    z.extractall(INPUT)
-
 br_saeb_latest = pd.read_excel(
-    INPUT / "PLANILHAS DE RESULTADOS_20250507" / "TS_BRASIL_20250507.xlsx",
+    input / xlsx_file,
+    sheet_name="Brasil",
     dtype=str,
 )
 
 br_saeb_latest.head()
 
-
 br_saeb_latest = (
     br_saeb_latest.drop(0, axis="index")
     .pipe(lambda df: df.loc[df["CAPITAL"] == "Total"])
@@ -123,6 +132,7 @@
             index=["DEPENDENCIA_ADM", "LOCALIZACAO", "disciplina", "serie"],
             columns="nivel",
             values="value",
+            aggfunc="first",
         )
         .reset_index()
         .merge(
@@ -156,7 +166,7 @@
     )
 )
 
-br_saeb_latest_output["ano"] = 2023
+br_saeb_latest_output["ano"] = 2025
 
 br_saeb_latest_output.head()
 
@@ -182,13 +192,15 @@
     billing_project_id="basedosdados-dev",
 )
 
-pd.concat([br_saeb_latest_output, upstream_df]).to_csv(  # type: ignore
-    os.path.join(OUTPUT, "brasil.csv"), index=False
-)
+br_saeb_updated = pd.concat([br_saeb_latest_output, upstream_df])
+
+br_saeb_updated.to_csv(os.path.join(output, "brasil.csv"), index=False)
+
+print(br_saeb_updated)
 
 # Update table
 tb.create(
-    os.path.join(OUTPUT, "brasil.csv"),
+    output / "brasil.csv",
     if_table_exists="replace",
     if_storage_data_exists="replace",
 )

models/br_inep_saeb/code/br_inep_saeb_municipio.py

@@ -3,29 +3,54 @@
 
 import basedosdados as bd
 import pandas as pd
+import requests
 
-# pyrefly: ignore [missing-import]
-from utils import (
-    # pyrefly: ignore [missing-module-attribute]
+from models.br_inep_saeb.code.utils import (
     convert_to_pd_dtype,
-    # pyrefly: ignore [missing-module-attribute]
     drop_empty_lines,
-    # pyrefly: ignore [missing-module-attribute]
     get_disciplina_serie,
-    # pyrefly: ignore [missing-module-attribute]
     get_nivel_serie_disciplina,
 )
 
-CWD = Path(os.getcwd()).parent
+input = Path("input") / "br_inep_saeb"
+output = Path("output") / "br_inep_saeb"
 
-INPUT = CWD / "input"
-OUTPUT = CWD / "output"
+os.makedirs(input, exist_ok=True)
+os.makedirs(output, exist_ok=True)
+
+url = "https://download.inep.gov.br/saeb/resultados/saeb_2025_brasil_estados_municipios_censitario.xlsx"
+xlsx_file = "saeb_2025.xlsx"
+
+
+def download(url: str, max_attempts: int = 3, timeout: float = 120):
+    for attempt in range(1, max_attempts + 1):
+        try:
+            r = requests.get(
+                url,
+                headers={"User-Agent": "Mozilla/5.0"},
+                verify=False,
+                stream=True,
+                timeout=timeout,
+            )
+            r.raise_for_status()
+            return r
+        except requests.exceptions.ConnectionError as exc:
+            print(
+                f"[attempt {attempt}/{max_attempts}] failed: {exc!r} - retrying.."
+            )
+    raise Exception(f"All {max_attempts} attempts failed for {url}")
+
+
+r = download(url)
+
+with open(input / xlsx_file, "wb") as fd:
+    for chunk in r.iter_content(chunk_size=128):
+        fd.write(chunk)
 
-os.makedirs(INPUT, exist_ok=True)
-os.makedirs(OUTPUT, exist_ok=True)
 
 mun_saeb_latest = pd.read_excel(
-    INPUT / "PLANILHAS DE RESULTADOS_20250507" / "TS_MUNICIPIO_20250507.xlsx",
+    input / xlsx_file,
+    sheet_name="Municípios",
     dtype=str,
 )
 
@@ -123,6 +148,7 @@
             ],
             columns="nivel",
             values="value",
+            aggfunc="first",
         )
         .reset_index()
         .merge(
@@ -189,7 +215,7 @@
 
 mun_saeb_latest_output = drop_empty_lines(mun_saeb_latest_output)
 
-mun_saeb_latest_output["ano"] = 2023
+mun_saeb_latest_output["ano"] = 2025
 
 mun_saeb_latest_output.head()
 
@@ -215,13 +241,15 @@
     billing_project_id="basedosdados-dev",
 )
 
-pd.concat([mun_saeb_latest_output, upstream_df]).to_csv(  # type: ignore
-    os.path.join(OUTPUT, "municipio.csv"), index=False
-)
+mun_saeb_updated = pd.concat([mun_saeb_latest_output, upstream_df])
+
+mun_saeb_updated.to_csv(os.path.join(output, "municipio.csv"), index=False)
+
+print(mun_saeb_updated)
 
 # Update table
 tb.create(
-    os.path.join(OUTPUT, "municipio.csv"),
+    output / "municipio.csv",
     if_table_exists="replace",
     if_storage_data_exists="replace",
 )

models/br_inep_saeb/code/br_inep_saeb_uf.py

@@ -3,29 +3,54 @@
 
 import basedosdados as bd
 import pandas as pd
+import requests
 
-# pyrefly: ignore [missing-import]
-from utils import (
-    # pyrefly: ignore [missing-module-attribute]
+from models.br_inep_saeb.code.utils import (
     convert_to_pd_dtype,
-    # pyrefly: ignore [missing-module-attribute]
     drop_empty_lines,
-    # pyrefly: ignore [missing-module-attribute]
     get_disciplina_serie,
-    # pyrefly: ignore [missing-module-attribute]
     get_nivel_serie_disciplina,
 )
 
-CWD = Path(os.getcwd()).parent
+input = Path("input") / "br_inep_saeb"
+output = Path("output") / "br_inep_saeb"
 
-INPUT = CWD / "input"
-OUTPUT = CWD / "output"
+os.makedirs(input, exist_ok=True)
+os.makedirs(output, exist_ok=True)
+
+url = "https://download.inep.gov.br/saeb/resultados/saeb_2025_brasil_estados_municipios_censitario.xlsx"
+xlsx_file = "saeb_2025.xlsx"
+
+
+def download(url: str, max_attempts: int = 3, timeout: float = 120):
+    for attempt in range(1, max_attempts + 1):
+        try:
+            r = requests.get(
+                url,
+                headers={"User-Agent": "Mozilla/5.0"},
+                verify=False,
+                stream=True,
+                timeout=timeout,
+            )
+            r.raise_for_status()
+            return r
+        except requests.exceptions.ConnectionError as exc:
+            print(
+                f"[attempt {attempt}/{max_attempts}] failed: {exc!r} - retrying.."
+            )
+    raise Exception(f"All {max_attempts} attempts failed for {url}")
+
+
+r = download(url)
+
+with open(input / xlsx_file, "wb") as fd:
+    for chunk in r.iter_content(chunk_size=128):
+        fd.write(chunk)
 
-os.makedirs(INPUT, exist_ok=True)
-os.makedirs(OUTPUT, exist_ok=True)
 
 ufs_saeb_latest = pd.read_excel(
-    INPUT / "PLANILHAS DE RESULTADOS_20250507" / "TS_UF_20250507.xlsx",
+    input / xlsx_file,
+    sheet_name="Estados",
     dtype=str,
 )
 
@@ -113,6 +138,7 @@
             ],
             columns="nivel",
             values="value",
+            aggfunc="first",
         )
         .reset_index()
         .merge(
@@ -171,7 +197,7 @@
 )
 
 # Add column ano
-ufs_saeb_latest_output["ano"] = 2023
+ufs_saeb_latest_output["ano"] = 2025
 
 ufs_saeb_latest_output.head()
 
@@ -201,13 +227,14 @@
 
 upstream_df["serie"].unique()
 
-pd.concat([ufs_saeb_latest_output, upstream_df]).to_csv(  # type: ignore
-    os.path.join(OUTPUT, "uf.csv"), index=False
-)
+ufs_saeb_updated = pd.concat([ufs_saeb_latest_output, upstream_df])
+ufs_saeb_updated.to_csv(os.path.join(output, "uf.csv"), index=False)
+
+print(ufs_saeb_updated)
 
 # Update table
 tb.create(
-    os.path.join(OUTPUT, "uf.csv"),
+    output / "uf.csv",
     if_table_exists="replace",
     if_storage_data_exists="replace",
 )