feat(sql): add debug logging for dropped tables and views in SQLAlchemy and Teradata sources

metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py

@@ -739,7 +739,9 @@ def loop_tables(
                         continue
 
                     self.report.report_entity_scanned(dataset_name, ent_type="table")
+                    logger.debug(f"Scanning table: {dataset_name}")
                     if not sql_config.table_pattern.allowed(dataset_name):
+                        logger.debug(f"Dropped table by table_pattern: {dataset_name}")
                         self.report.report_dropped(dataset_name)
                         continue
 
@@ -1142,8 +1144,10 @@ def loop_views(
                     schema=schema, entity=view, inspector=inspector
                 )
                 self.report.report_entity_scanned(dataset_name, ent_type="view")
+                logger.debug(f"Scanning view: {dataset_name}")
 
                 if not sql_config.view_pattern.allowed(dataset_name):
+                    logger.debug(f"Dropped view by view_pattern: {dataset_name}")
                     self.report.report_dropped(dataset_name)
                     continue
 

metadata-ingestion/src/datahub/ingestion/source/sql/teradata.py

@@ -2404,8 +2404,12 @@ def process_single_view(
                             self.report.report_entity_scanned(
                                 dataset_name, ent_type="view"
                             )
+                        logger.debug(f"Scanning view: {dataset_name}")
 
                         if not sql_config.view_pattern.allowed(dataset_name):
+                            logger.debug(
+                                f"Dropped view by view_pattern: {dataset_name}"
+                            )
                             with self.report.atomic():
                                 self.report.report_dropped(dataset_name)
                             return results
@@ -2654,8 +2658,12 @@ def _process_views_single_threaded(
                         )
 
                         self.report.report_entity_scanned(dataset_name, ent_type="view")
+                        logger.debug(f"Scanning view: {dataset_name}")
 
                         if not sql_config.view_pattern.allowed(dataset_name):
+                            logger.debug(
+                                f"Dropped view by view_pattern: {dataset_name}"
+                            )
                             self.report.report_dropped(dataset_name)
                             continue
 

metadata-ingestion/tests/unit/test_teradata_source.py

@@ -4956,6 +4956,160 @@ def test_warn_view_error_emits_correct_category_message(
         assert expected_fragment in source.report.warnings[0].message
 
 
+class TestFilterDropDebugLogging:
+    """Pattern-filtered tables and views are dropped (not processed) and recorded.
+
+    A fully-filtered Teradata run emits only database containers with everything else
+    in ``report.filtered``. These tests assert the behavior at each drop point: the
+    denied object is recorded in ``report.filtered`` and its ``_process_*`` handler is
+    never invoked. Covers both view processing paths (single- and multi-threaded) and
+    the base ``loop_tables`` path that Teradata tables use.
+    """
+
+    def test_single_threaded_view_drop_reports_and_skips_processing(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A view denied by view_pattern is recorded in report.filtered and never
+        handed to _process_view (single-threaded path).
+
+        This is also the one site where we pin the PR's debug logging: assert the
+        drop is emitted at DEBUG level. The other drop-site tests only assert the
+        drop *behavior* (report.filtered + skipped processing) to avoid brittle,
+        duplicated log-string assertions across every path.
+        """
+        source = _create_source_patched(
+            {"max_workers": 1, "view_pattern": {"deny": [".*"]}}
+        )
+        process_view = MagicMock()
+
+        with (
+            patch.object(source, "get_metadata_engine", return_value=MagicMock()),
+            patch.object(
+                source,
+                "_retry_connect",
+                side_effect=_mock_retry_connect(_make_mock_conn()),
+            ),
+            patch.object(source, "_retry_execute"),
+            patch(
+                "datahub.ingestion.source.sql.teradata.inspect",
+                return_value=_make_mock_inspector("testdb"),
+            ),
+            patch.object(source, "_process_view", process_view),
+            caplog.at_level(
+                logging.DEBUG, logger="datahub.ingestion.source.sql.teradata"
+            ),
+        ):
+            wus = list(
+                source._process_views_single_threaded(
+                    ["denied_view"], "testdb", source.config
+                )
+            )
+
+        assert wus == []
+        process_view.assert_not_called()
+        assert "testdb.denied_view" in list(source.report.filtered)
+        assert "Dropped view by view_pattern: testdb.denied_view" in caplog.text
+
+    def test_multi_threaded_view_drop_reports_and_skips_processing(self) -> None:
+        """A view denied by view_pattern is recorded in report.filtered and never
+        handed to _process_view (multi-threaded path)."""
+        source = _create_source_patched(
+            {"max_workers": 2, "view_pattern": {"deny": [".*"]}}
+        )
+        process_view = MagicMock()
+
+        with (
+            patch.object(
+                source, "_get_or_create_pooled_engine", return_value=MagicMock()
+            ),
+            patch.object(
+                source,
+                "_retry_connect",
+                side_effect=_mock_retry_connect(_make_mock_conn()),
+            ),
+            patch.object(source, "_retry_execute"),
+            patch(
+                "datahub.ingestion.source.sql.teradata.inspect",
+                return_value=_make_mock_inspector("testdb"),
+            ),
+            patch.object(source, "_process_view", process_view),
+        ):
+            wus = list(
+                source._loop_views_with_connection_pool(
+                    ["denied_view"], "testdb", source.config
+                )
+            )
+
+        assert wus == []
+        process_view.assert_not_called()
+        assert "testdb.denied_view" in list(source.report.filtered)
+
+    def test_mixed_allow_deny_drops_denied_and_processes_allowed(self) -> None:
+        """With a denied and an allowed view in the same call, the denied one is
+        dropped and the allowed one is still processed.
+
+        Passing both in a single call guards against an early-return regression:
+        the drop must ``continue`` to the next view, not abort the whole loop. This
+        also serves as the positive control (an allowed view reaches _process_view
+        and is not recorded in report.filtered)."""
+        source = _create_source_patched(
+            {"max_workers": 1, "view_pattern": {"deny": ["testdb.denied_view"]}}
+        )
+        process_view = MagicMock(return_value=iter([]))
+
+        with (
+            patch.object(source, "get_metadata_engine", return_value=MagicMock()),
+            patch.object(
+                source,
+                "_retry_connect",
+                side_effect=_mock_retry_connect(_make_mock_conn()),
+            ),
+            patch.object(source, "_retry_execute"),
+            patch(
+                "datahub.ingestion.source.sql.teradata.inspect",
+                return_value=_make_mock_inspector("testdb"),
+            ),
+            patch.object(source, "_process_view", process_view),
+        ):
+            list(
+                source._process_views_single_threaded(
+                    ["denied_view", "allowed_view"], "testdb", source.config
+                )
+            )
+
+        # Denied view is dropped; the loop continues and processes the allowed view.
+        process_view.assert_called_once()
+        assert process_view.call_args.kwargs["view"] == "allowed_view"
+        assert "testdb.denied_view" in list(source.report.filtered)
+        assert "testdb.allowed_view" not in list(source.report.filtered)
+
+    def test_table_drop_reports_and_skips_processing(self) -> None:
+        """A table denied by table_pattern is recorded in report.filtered and never
+        handed to _process_table. Teradata tables flow through the base loop_tables
+        via cached_loop_tables, which sources table names from _tables_cache."""
+        source = _create_source_patched({"table_pattern": {"deny": [".*"]}})
+        source._tables_cache["testdb"] = [
+            TeradataTable(
+                database="testdb",
+                name="denied_table",
+                description=None,
+                object_type="Table",
+                create_timestamp=datetime(2024, 1, 1),
+                last_alter_name=None,
+                last_alter_timestamp=None,
+                request_text=None,
+            )
+        ]
+        process_table = MagicMock(return_value=iter([]))
+
+        with patch.object(source, "_process_table", process_table):
+            wus = list(source.cached_loop_tables(MagicMock(), "testdb", source.config))
+
+        assert wus == []
+        process_table.assert_not_called()
+        assert "testdb.denied_table" in list(source.report.filtered)
+
+
 # ---------------------------------------------------------------------------
 # Helpers for _fetch_lineage_entries_chunked tests
 # ---------------------------------------------------------------------------