test(qa): add comprehensive test suites for DAG, UI, generator, and in-memory execution

Author: Medalcode

test123.txt

@@ -0,0 +1,61 @@
+"""
+tests/test_airflow_dag.py
+--------------------------
+Unit and integration tests for Apache Airflow DAG in dags/data_pipeline_dag.py.
+"""
+
+import sys
+from pathlib import Path
+
+import pytest
+
+# Skip module if apache-airflow is not installed in the current environment
+pytest.importorskip("airflow")
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from dags.data_pipeline_dag import _get_safe_run_id, dag, default_args
+
+
+class TestAirflowDagDefinition:
+    """Test suite for Airflow DAG structure and helper functions."""
+
+    def test_dag_structure_and_metadata(self):
+        assert dag is not None
+        assert dag.dag_id == "dataprep_pipeline"
+        assert dag.schedule_interval == "@daily"
+        assert len(dag.tasks) == 5
+
+    def test_dag_task_dependencies(self):
+        task_dict = {t.task_id: t for t in dag.tasks}
+        assert set(task_dict.keys()) == {
+            "ingest_data",
+            "validate_data",
+            "clean_data",
+            "transform_data",
+            "load_data",
+        }
+
+        # Check linear dependency flow: ingest -> validate -> clean -> transform -> load
+        ingest_task = task_dict["ingest_data"]
+        validate_task = task_dict["validate_data"]
+        clean_task = task_dict["clean_data"]
+        transform_task = task_dict["transform_data"]
+        load_task = task_dict["load_data"]
+
+        assert validate_task in ingest_task.downstream_list
+        assert clean_task in validate_task.downstream_list
+        assert transform_task in clean_task.downstream_list
+        assert load_task in transform_task.downstream_list
+
+    def test_get_safe_run_id_sanitizes_illegal_chars(self):
+        context_with_colon = {"run_id": "scheduled__2026-08-01T12:00:00+00:00"}
+        safe_id = _get_safe_run_id(context_with_colon)
+        assert ":" not in safe_id
+        assert "+" not in safe_id
+        assert safe_id == "scheduled__2026-08-01T12_00_00_00_00"
+
+    def test_get_safe_run_id_default_fallback(self):
+        context_empty = {}
+        safe_id = _get_safe_run_id(context_empty)
+        assert safe_id == "default_run"

tests/test_airflow_dag.py

@@ -0,0 +1,52 @@
+"""
+tests/test_app_ui.py
+---------------------
+Unit tests for Streamlit App UI helper functions in app.py.
+"""
+
+import sys
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from app import run_pipeline_cached, to_csv_bytes
+
+
+class TestAppUiHelpers:
+    """Test suite for app.py helper functions."""
+
+    def test_to_csv_bytes_returns_utf8_bytes(self):
+        df = pd.DataFrame({"id": [1, 2], "nombre": ["Ana", "Carlos"]})
+        result_bytes = to_csv_bytes(df)
+
+        assert isinstance(result_bytes, bytes)
+        csv_str = result_bytes.decode("utf-8")
+        assert "id,nombre" in csv_str
+        assert "Ana" in csv_str
+        assert "Carlos" in csv_str
+
+    def test_run_pipeline_cached_execution(self):
+        df_raw = pd.DataFrame(
+            {
+                "fecha": ["2024-01-01", "2024-01-02"],
+                "precio": [100.0, 200.0],
+                "cantidad": [1, 2],
+            }
+        )
+
+        before_report, df_clean, df_final, after_report = run_pipeline_cached(
+            df_raw=df_raw,
+            normalize=False,
+            null_thresh=20.0,
+            dup_thresh=5.0,
+        )
+
+        assert before_report is not None
+        assert after_report is not None
+        assert df_clean is not None
+        assert df_final is not None
+        assert "total" in df_final.columns
+        assert len(df_final) == 2

tests/test_app_ui.py

@@ -0,0 +1,42 @@
+"""
+tests/test_dataset_generator.py
+--------------------------------
+Unit tests for synthetic dataset generator in generate_dataset.py.
+"""
+
+import sys
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from generate_dataset import generate_dataset, introduce_errors, make_clean_row, random_date
+
+
+class TestDatasetGenerator:
+    """Test suite for synthetic dataset generation utility."""
+
+    def test_random_date_returns_valid_iso_string(self):
+        d_str = random_date()
+        assert isinstance(d_str, str)
+        parsed_dt = pd.to_datetime(d_str)
+        assert parsed_dt.year in [2022, 2023, 2024]
+
+    def test_make_clean_row_structure(self):
+        row = make_clean_row(i=42)
+        assert isinstance(row, dict)
+        assert row["id_venta"] == 42
+        assert "precio" in row
+        assert "cantidad" in row
+        assert "fecha" in row
+
+    def test_generate_dataset_introduces_expected_dirty_data(self):
+        df = generate_dataset(n=200)
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) > 200  # Should include duplicate rows added by introduce_errors
+        assert "codigo_venta" in df.columns
+
+        # Verify intentional dirty characteristics
+        assert df.isnull().sum().sum() > 0

tests/test_dataset_generator.py

@@ -0,0 +1,48 @@
+"""
+tests/test_pipeline_inmemory.py
+--------------------------------
+Unit tests for execute_pipeline in-memory df_raw execution path.
+"""
+
+import sys
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from src.pipeline import execute_pipeline
+
+
+class TestPipelineInMemoryExecution:
+    """Test suite for execute_pipeline in-memory execution mode."""
+
+    def test_execute_pipeline_with_df_raw(self, tmp_path):
+        out_csv = tmp_path / "inmemory_out.csv"
+        out_html = tmp_path / "inmemory_report.html"
+
+        df_raw = pd.DataFrame(
+            {
+                "id_venta": [1, 2, 2],
+                "fecha": ["2024-01-01", "2024-01-02", "2024-01-02"],
+                "precio": [10.0, 20.0, 20.0],
+                "cantidad": [2, 1, 1],
+            }
+        )
+
+        result = execute_pipeline(
+            output_path=out_csv,
+            report_path=out_html,
+            df_raw=df_raw,
+            normalize=True,
+        )
+
+        assert result.success is True
+        assert result.df_raw is not None
+        assert result.df_final is not None
+        assert len(result.df_final) == 2  # Duplicate removed
+        assert out_csv.exists()
+        assert out_html.exists()
+        assert "total" in result.df_final.columns
+        assert "precio_norm" in result.df_final.columns