test_generic_api_extractor.py

import unittest

from generic_api_extractor import detect_next_page, find_records, json_to_markdown, remove_noise


class ExtractorTests(unittest.TestCase):
    def test_find_records_priority_and_fallback(self):
        self.assertEqual(find_records({"items": [{"id": 1}]}), [{"id": 1}])
        self.assertEqual(find_records([{"id": 2}]), [{"id": 2}])
        self.assertEqual(find_records({"answer": 42}), [{"answer": 42}])

    def test_link_header_has_priority(self):
        url, method = detect_next_page(
            {"next": "https://body.example/2"},
            {"link": '<https://header.example/2>; rel="next", <https://header.example/9>; rel="last"'},
            "https://example.test/1",
        )
        self.assertEqual(url, "https://header.example/2")
        self.assertEqual(method, "link_header")

    def test_body_pagination_variants(self):
        url, method = detect_next_page({"links": {"next": "/page/2"}}, {}, "https://example.test/page/1")
        self.assertEqual(url, "https://example.test/page/2")
        self.assertEqual(method, "body_field:links.next")
        url, method = detect_next_page({"hasMore": True}, {}, "https://example.test/data?page=4")
        self.assertEqual(url, "https://example.test/data?page=5")
        self.assertEqual(method, "body_field:hasMore")

    def test_noise_removal_preserves_counts(self):
        records = [
            {
                "id": index,
                "same": "boilerplate",
                "stars": index,
                "meta": {"@type": f"x{index}"},
                "empty": None if index % 2 else "",
            }
            for index in range(10)
        ]
        cleaned, log, removed = remove_noise(records)
        self.assertTrue(all("id" not in record for record in cleaned))
        self.assertTrue(all("same" not in record for record in cleaned))
        self.assertTrue(all(record["stars"] == index for index, record in enumerate(cleaned)))
        self.assertTrue(any("meta.@type | protocol_convention" in line for line in log))
        self.assertEqual(removed, 40)

    def test_markdown_depth_limit(self):
        markdown = json_to_markdown({"a": {"b": {"c": {"d": {"e": 1}}}}})
        self.assertIn("#### c", markdown)
        self.assertIn('{"d": {"e": 1}}', markdown)


if __name__ == "__main__":
    unittest.main()