feat(pipeline): add in-memory DataFrame support and refactor app.py SSOT

Author: Medalcode

app.py

@@ -19,10 +19,8 @@
 sys.path.insert(0, str(Path(__file__).parent))
 
 from config.settings import CLEANING_CONFIG, PIPELINE_VERSION, VALIDATION_CONFIG
-from src.cleaning import clean_data
+from src.pipeline import execute_pipeline
 from src.report import generate_quality_report
-from src.transformation import transform_data
-from src.validation import validate_data
 
 # ── Page config ───────────────────────────────────────────────────────────────
 st.set_page_config(
@@ -143,11 +141,14 @@ def run_pipeline_cached(df_raw: pd.DataFrame, normalize: bool, null_thresh: floa
         "duplicate_threshold_pct": dup_thresh,
         "iqr_factor": 1.5,
     }
-    before_report = validate_data(df_raw, **v_config)
-    df_clean = clean_data(df_raw, config=CLEANING_CONFIG)
-    df_final = transform_data(df_clean, normalize=normalize)
-    after_report = validate_data(df_final)
-    return before_report, df_clean, df_final, after_report
+    result = execute_pipeline(
+        df_raw=df_raw,
+        validation_config=v_config,
+        normalize=normalize,
+    )
+    if not result.success:
+        raise RuntimeError(result.error_message or "Error executing pipeline")
+    return result.before_report, result.df_clean, result.df_final, result.after_report
 
 
 with st.spinner("Ejecutando pipeline..."):

src/pipeline.py

@@ -59,10 +59,11 @@ def execute_pipeline(
     cleaning_config: dict[str, Any] | None = None,
     validation_config: dict[str, Any] | None = None,
     normalize: bool = False,
+    df_raw: pd.DataFrame | None = None,
 ) -> PipelineResult:
     """
     Execute the unified 5-step DataPrep Pipeline:
-    1. Ingestion
+    1. Ingestion (skipped if df_raw is provided)
     2. Validation (Before)
     3. Cleaning
     4. Transformation
@@ -77,6 +78,7 @@ def execute_pipeline(
         cleaning_config:   Optional cleaning settings dict override.
         validation_config: Optional validation threshold dict override.
         normalize:         Whether to apply min-max normalization to numeric columns.
+        df_raw:            Optional in-memory raw DataFrame to bypass file ingestion.
 
     Returns:
         PipelineResult with execution status, DataFrames, reports, and execution time.
@@ -95,26 +97,29 @@ def execute_pipeline(
 
     # ── STEP 1: INGEST ──────────────────────────────────────────────────────
     logger.info("[1/5] INGESTION")
-    try:
-        if file_format == "excel":
-            df_raw = load_excel(input_path)
-        elif file_format == "api":
-            if not api_url:
-                err_msg = "API URL required when format is 'api'"
-                logger.error(err_msg)
-                return PipelineResult(success=False, error_message=err_msg)
-            df_raw = load_json_api(api_url)
-        else:
-            df_raw = load_csv(input_path)
-    except FileNotFoundError as e:
-        err_msg = f"Input file not found: {e}"
-        logger.error(err_msg)
-        logger.error("Run 'python generate_dataset.py' first to create test data.")
-        return PipelineResult(success=False, error_message=err_msg)
-    except Exception as e:
-        err_msg = f"Ingestion error: {e}"
-        logger.error(err_msg, exc_info=True)
-        return PipelineResult(success=False, error_message=err_msg)
+    if df_raw is None:
+        try:
+            if file_format == "excel":
+                df_raw = load_excel(input_path)
+            elif file_format == "api":
+                if not api_url:
+                    err_msg = "API URL required when format is 'api'"
+                    logger.error(err_msg)
+                    return PipelineResult(success=False, error_message=err_msg)
+                df_raw = load_json_api(api_url)
+            else:
+                df_raw = load_csv(input_path)
+        except FileNotFoundError as e:
+            err_msg = f"Input file not found: {e}"
+            logger.error(err_msg)
+            logger.error("Run 'python generate_dataset.py' first to create test data.")
+            return PipelineResult(success=False, error_message=err_msg)
+        except Exception as e:
+            err_msg = f"Ingestion error: {e}"
+            logger.error(err_msg, exc_info=True)
+            return PipelineResult(success=False, error_message=err_msg)
+    else:
+        logger.info(f"Using in-memory DataFrame ({len(df_raw):,} rows)")
 
     try:
         # ── STEP 2: VALIDATE (BEFORE) ─────────────────────────────────────────