[Chore] Ajustes CNPJ

dbt_project.yml

@@ -424,6 +424,9 @@ models:
     br_rf_cno:
       +materialized: table
       +schema: br_rf_cno
+    br_rf_cnpj:
+      +materialized: table
+      +schema: br_rf_cnpj
     br_rj_isp_estatisticas_seguranca:
       +materialized: table
       +schema: br_rj_isp_estatisticas_seguranca

macros/custom_get_where_subquery.sql

@@ -68,6 +68,26 @@
             {% endif %}
         {% endif %}
 
+        {# This block looks for __most_recent_date_cnpj__  placeholder #}
+        {% if "__most_recent_date_cnpj__" in where %}
+            {% set max_date_query = (
+                "select max(data_referencia) as max_date from " ~ relation
+            ) %}
+            {% set max_date_result = run_query(max_date_query) %}
+            {% if execute and max_date_result.rows[0][0] %}
+                {% set max_date = max_date_result.rows[0][0] %}
+                {% set where = where | replace(
+                    "__most_recent_date_cnpj__",
+                    "data_referencia = '" ~ max_date ~ "'",
+                ) %}
+                {% do log(
+                    "The test will filter by the most recent date: "
+                    ~ max_date,
+                    info=True,
+                ) %}
+            {% endif %}
+        {% endif %}
+
         {% if "__most_recent_date_cno__" in where %}
             {% set max_date_query = (
                 "select max(data_extracao) as max_date from " ~ relation

models/br_rf_cnpj/br_rf_cnpj__dicionario.sql

@@ -0,0 +1,62 @@
+{{ config(alias="dicionario", schema="br_rf_cnpj", materialized="table") }}
+
+with
+    tmp_dict as (
+        select
+            safe_cast(id_tabela as string) id_tabela,
+            safe_cast(nome_coluna as string) nome_coluna,
+            safe_cast(chave as string) chave,
+            safe_cast(cobertura_temporal as string) cobertura_temporal,
+            regexp_replace(
+                {{ validate_null_cols("valor") }}, r'(^0+)(?:[^0]+|0{1})', ''
+            ) as valor,
+        from {{ set_datalake_project("br_rf_cnpj_staging.dicionario") }} as t
+    )
+select
+    id_tabela,
+    nome_coluna,
+    chave,
+    cobertura_temporal,
+    case when nome_coluna = "id_pais" then valor else initcap(valor) end as valor
+from tmp_dict
+where valor is not null
+union all
+{{
+    dicionario_not_found(
+        id_tabela="empresas",
+        nome_coluna="qualificacao_responsavel",
+        chave="36",
+    )
+}}
+union all
+{{
+    dicionario_not_found(
+        id_tabela="socios",
+        nome_coluna="id_pais",
+        chave=["994", "393"],
+    )
+}}
+union all
+{{
+    dicionario_not_found(
+        id_tabela="estabelecimentos",
+        nome_coluna="id_pais",
+        chave=["8", "9", "393"],
+    )
+}}
+union all
+{{
+    dicionario_not_found(
+        id_tabela="estabelecimentos",
+        nome_coluna="motivo_situacao_cadastral",
+        chave="32",
+    )
+}}
+union all
+{{
+    dicionario_not_found(
+        id_tabela="estabelecimentos",
+        nome_coluna="",
+        chave=["6202100", "4761000"],
+    )
+}}

models/br_rf_cnpj/br_rf_cnpj__empresas.sql

@@ -0,0 +1,55 @@
+{{
+    config(
+        schema="br_rf_cnpj",
+        alias="empresas",
+        materialized="incremental",
+        partition_by={
+            "field": "data_referencia",
+            "data_type": "date",
+            "granularity": "month",
+        },
+    )
+}}
+
+with
+    cnpj_empresas as (
+        select
+            safe.parse_date('%Y-%m', data_referencia) data_referencia,
+            safe_cast(lpad(cnpj_basico, 8, '0') as string) cnpj_basico,
+            safe_cast(razao_social as string) razao_social,
+            safe_cast(natureza_juridica as string) natureza_juridica,
+            safe_cast(
+                regexp_replace(qualificacao_responsavel, '^0', '') as string
+            ) qualificacao_responsavel,
+            safe_cast(capital_social as float64) capital_social,
+            safe_cast(regexp_replace(porte, '^0', '') as string) porte,
+            safe_cast(ente_federativo as string) ente_federativo,
+            safe_cast(data_modificacao as date) data_modificacao
+        from {{ set_datalake_project("br_rf_cnpj_staging.empresas") }} as t
+        where
+            porte != "porte"
+            {% if is_incremental() %}
+                and safe.parse_date('%Y-%m', data_referencia)
+                > (select max(data_referencia) from {{ this }})
+        {% else %}
+            -- Dados históricos até 2023-04-30 foram migrados do modelo
+            -- br_me_cnpj.estabelecimentos
+            union all
+            select
+                safe_cast(data as date) data_referencia,
+                safe_cast(lpad(cnpj_basico, 8, '0') as string) cnpj_basico,
+                safe_cast(razao_social as string) razao_social,
+                safe_cast(natureza_juridica as string) natureza_juridica,
+                safe_cast(
+                    regexp_replace(qualificacao_responsavel, '^0', '') as string
+                ) qualificacao_responsavel,
+                safe_cast(capital_social as float64) capital_social,
+                safe_cast(regexp_replace(porte, '^0', '') as string) porte,
+                safe_cast(ente_federativo as string) ente_federativo,
+                safe_cast(null as date) data_modificacao
+            from {{ set_datalake_project("br_rf_cnpj_staging.empresas_legado") }}
+            where porte != "porte" and safe_cast(data as date) <= date("2023-04-30")
+        {% endif %}
+    )
+select *
+from cnpj_empresas

models/br_rf_cnpj/br_rf_cnpj__estabelecimentos.sql

@@ -0,0 +1,148 @@
+{{
+    config(
+        schema="br_rf_cnpj",
+        alias="estabelecimentos",
+        materialized="incremental",
+        partition_by={
+            "field": "data_referencia",
+            "data_type": "date",
+            "granularity": "month",
+        },
+        cluster_by=["sigla_uf"],
+    )
+}}
+with
+    cnpj_estabelecimentos as (
+        select
+            safe.parse_date('%Y-%m', data_referencia) data_referencia,
+            safe_cast(lpad(cnpj, 14, "0") as string) cnpj,
+            safe_cast(lpad(cnpj_basico, 8, '0') as string) cnpj_basico,
+            safe_cast(lpad(cnpj_ordem, 4, '0') as string) cnpj_ordem,
+            safe_cast(lpad(cnpj_dv, 2, '0') as string) cnpj_dv,
+            safe_cast(
+                identificador_matriz_filial as string
+            ) identificador_matriz_filial,
+            safe_cast(nome_fantasia as string) nome_fantasia,
+            safe_cast(cast(situacao_cadastral as int64) as string) situacao_cadastral,
+            safe_cast(data_situacao_cadastral as date) data_situacao_cadastral,
+            safe_cast(
+                regexp_replace(motivo_situacao_cadastral, '^0', '') as string
+            ) motivo_situacao_cadastral,
+            safe_cast(nome_cidade_exterior as string) nome_cidade_exterior,
+            safe_cast(cast(id_pais as int64) as string) id_pais,
+            safe_cast(data_inicio_atividade as date) data_inicio_atividade,
+            safe_cast(cnae_fiscal_principal as string) cnae_fiscal_principal,
+            safe_cast(cnae_fiscal_secundaria as string) cnae_fiscal_secundaria,
+            safe_cast(sigla_uf as string) sigla_uf,
+            safe_cast(safe_cast(id_municipio_rf as numeric) as string) id_municipio_rf,
+            safe_cast(tipo_logradouro as string) tipo_logradouro,
+            safe_cast(logradouro as string) logradouro,
+            safe_cast(numero as string) numero,
+            safe_cast(complemento as string) complemento,
+            safe_cast(bairro as string) bairro,
+            safe_cast(replace (cep, ".0", "") as string) cep,
+            safe_cast(ddd_1 as string) ddd_1,
+            safe_cast(telefone_1 as string) telefone_1,
+            safe_cast(ddd_2 as string) ddd_2,
+            safe_cast(telefone_2 as string) telefone_2,
+            safe_cast(ddd_fax as string) ddd_fax,
+            safe_cast(fax as string) fax,
+            safe_cast(lower(email) as string) email,
+            safe_cast(situacao_especial as string) situacao_especial,
+            safe_cast(data_modificacao as date) data_modificacao,
+            safe_cast(data_situacao_especial as date) data_situacao_especial
+        from {{ set_datalake_project("br_rf_cnpj_staging.estabelecimentos") }}
+        {% if is_incremental() %}
+            where
+                safe.parse_date('%Y-%m', data_referencia)
+                > (select max(data_referencia) from {{ this }})
+        -- Dados históricos até 2023-04-30 foram migrados do modelo
+        -- br_me_cnpj.estabelecimentos
+        {% else %}
+            union all
+            select
+                safe_cast(data as date) data_referencia,
+                safe_cast(lpad(cnpj, 14, "0") as string) cnpj,
+                safe_cast(lpad(cnpj_basico, 8, '0') as string) cnpj_basico,
+                safe_cast(lpad(cnpj_ordem, 4, '0') as string) cnpj_ordem,
+                safe_cast(lpad(cnpj_dv, 2, '0') as string) cnpj_dv,
+                safe_cast(
+                    identificador_matriz_filial as string
+                ) identificador_matriz_filial,
+                safe_cast(nome_fantasia as string) nome_fantasia,
+                safe_cast(
+                    cast(situacao_cadastral as int64) as string
+                ) situacao_cadastral,
+                safe_cast(data_situacao_cadastral as date) data_situacao_cadastral,
+                safe_cast(
+                    regexp_replace(motivo_situacao_cadastral, '^0', '') as string
+                ) motivo_situacao_cadastral,
+                safe_cast(nome_cidade_exterior as string) nome_cidade_exterior,
+                safe_cast(cast(id_pais as int64) as string) id_pais,
+                safe_cast(data_inicio_atividade as date) data_inicio_atividade,
+                safe_cast(cnae_fiscal_principal as string) cnae_fiscal_principal,
+                safe_cast(cnae_fiscal_secundaria as string) cnae_fiscal_secundaria,
+                safe_cast(sigla_uf as string) sigla_uf,
+                safe_cast(
+                    safe_cast(id_municipio_rf as numeric) as string
+                ) id_municipio_rf,
+                safe_cast(tipo_logradouro as string) tipo_logradouro,
+                safe_cast(logradouro as string) logradouro,
+                safe_cast(numero as string) numero,
+                safe_cast(complemento as string) complemento,
+                safe_cast(bairro as string) bairro,
+                safe_cast(replace (cep, ".0", "") as string) cep,
+                safe_cast(ddd_1 as string) ddd_1,
+                safe_cast(telefone_1 as string) telefone_1,
+                safe_cast(ddd_2 as string) ddd_2,
+                safe_cast(telefone_2 as string) telefone_2,
+                safe_cast(ddd_fax as string) ddd_fax,
+                safe_cast(fax as string) fax,
+                safe_cast(lower(email) as string) email,
+                safe_cast(situacao_especial as string) situacao_especial,
+                safe_cast(null as date) data_modificacao,
+                safe_cast(data_situacao_especial as date) data_situacao_especial
+            from
+                {{ set_datalake_project("br_rf_cnpj_staging.estabelecimentos_legado") }}
+            where safe_cast(data as date) <= date("2023-04-30")
+        {% endif %}
+    )
+select
+    a.data_referencia,
+    a.cnpj,
+    a.cnpj_basico,
+    a.cnpj_ordem,
+    a.cnpj_dv,
+    a.identificador_matriz_filial,
+    a.nome_fantasia,
+    a.situacao_cadastral,
+    a.data_situacao_cadastral,
+    a.motivo_situacao_cadastral,
+    a.nome_cidade_exterior,
+    a.id_pais,
+    a.data_inicio_atividade,
+    a.cnae_fiscal_principal,
+    a.cnae_fiscal_secundaria,
+    a.sigla_uf,
+    safe_cast(b.id_municipio as string) id_municipio,
+    a.id_municipio_rf,
+    a.tipo_logradouro,
+    a.logradouro,
+    a.numero,
+    a.complemento,
+    a.bairro,
+    a.cep,
+    a.ddd_1,
+    a.telefone_1,
+    a.ddd_2,
+    a.telefone_2,
+    a.ddd_fax,
+    a.fax,
+    a.email,
+    a.situacao_especial,
+    a.data_modificacao,
+    a.data_situacao_especial
+from cnpj_estabelecimentos a
+left join
+    basedosdados.br_bd_diretorios_brasil.municipio b
+    on safe_cast(safe_cast(a.id_municipio_rf as numeric) as string) = b.id_municipio_rf

models/br_rf_cnpj/br_rf_cnpj__simples.sql

@@ -0,0 +1,20 @@
+{{
+    config(
+        schema="br_rf_cnpj",
+        alias="simples",
+        materialized="table",
+    )
+}}
+
+select
+    -- safe.parse_date('%Y-%m', data_referencia) data_referencia,  
+    lpad(safe_cast(cnpj_basico as string), 8, '0') cnpj_basico,
+    safe_cast(opcao_simples as int64) opcao_simples,
+    safe_cast(data_opcao_simples as date) data_opcao_simples,
+    safe_cast(data_exclusao_simples as date) data_exclusao_simples,
+    safe_cast(opcao_mei as int64) opcao_mei,
+    safe_cast(data_opcao_mei as date) data_opcao_mei,
+    safe_cast(data_exclusao_mei as date) data_exclusao_mei,
+-- safe_cast(data_modificacao as date) data_modificacao
+from {{ set_datalake_project("br_rf_cnpj_staging.simples") }} as t
+where safe_cast(opcao_mei as string) != "opcao_mei"

models/br_rf_cnpj/br_rf_cnpj__socios.sql

@@ -0,0 +1,64 @@
+{{
+    config(
+        schema="br_rf_cnpj",
+        alias="socios",
+        materialized="incremental",
+        partition_by={
+            "field": "data_referencia",
+            "data_type": "date",
+            "granularity": "month",
+        },
+    )
+}}
+with
+    cnpj_socios as (
+        select
+            safe.parse_date('%Y-%m', data_referencia) data_referencia,
+            lpad(safe_cast(cnpj_basico as string), 8, '0') cnpj_basico,
+            safe_cast(tipo as string) tipo,
+            safe_cast(nome as string) nome,
+            safe_cast(documento as string) documento,
+            safe_cast(cast(qualificacao as int64) as string) qualificacao,
+            safe_cast(data_entrada_sociedade as date) data_entrada_sociedade,
+            safe_cast(cast(id_pais as int64) as string) id_pais,
+            safe_cast(cpf_representante_legal as string) cpf_representante_legal,
+            safe_cast(nome_representante_legal as string) nome_representante_legal,
+            safe_cast(
+                cast(qualificacao_representante_legal as int64) as string
+            ) qualificacao_representante_legal,
+            safe_cast(faixa_etaria as string) faixa_etaria,
+            safe_cast(data_modificacao as date) data_modificacao
+        from {{ set_datalake_project("br_rf_cnpj_staging.socios") }} as t
+        where
+            safe_cast(qualificacao as string) != "qualificacao"
+            {% if is_incremental() %}
+                and safe.parse_date('%Y-%m', data_referencia)
+                > (select max(data_referencia) from {{ this }})
+        {% else %}
+            -- Dados históricos até 2023-04-30 foram migrados do modelo
+            -- br_me_cnpj.socios
+            union all
+            select
+                safe_cast(data as date) data_referencia,
+                lpad(safe_cast(cnpj_basico as string), 8, '0') cnpj_basico,
+                safe_cast(tipo as string) tipo,
+                safe_cast(nome as string) nome,
+                safe_cast(documento as string) documento,
+                safe_cast(cast(qualificacao as int64) as string) qualificacao,
+                safe_cast(data_entrada_sociedade as date) data_entrada_sociedade,
+                safe_cast(cast(id_pais as int64) as string) id_pais,
+                safe_cast(cpf_representante_legal as string) cpf_representante_legal,
+                safe_cast(nome_representante_legal as string) nome_representante_legal,
+                safe_cast(
+                    cast(qualificacao_representante_legal as int64) as string
+                ) qualificacao_representante_legal,
+                safe_cast(faixa_etaria as string) faixa_etaria,
+                safe_cast(null as date) data_modificacao
+            from {{ set_datalake_project("br_rf_cnpj_staging.socios_legado") }}
+            where
+                safe_cast(qualificacao as string) != "qualificacao"
+                and safe_cast(data as date) <= date("2023-04-30")
+        {% endif %}
+    )
+select *
+from cnpj_socios