|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +import json |
| 4 | +import os |
| 5 | +import sys |
| 6 | +import tempfile |
| 7 | +import threading |
| 8 | +import unittest |
| 9 | +from concurrent.futures import ThreadPoolExecutor |
| 10 | +from pathlib import Path |
| 11 | +from unittest.mock import patch |
| 12 | + |
| 13 | +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) |
| 14 | +import publish_incremental as publish |
| 15 | + |
| 16 | + |
| 17 | +class ParallelArchiveTest(unittest.TestCase): |
| 18 | + def setUp(self): |
| 19 | + self.items = [ |
| 20 | + {"zip_path": Path(f"{name}.zip"), "rel": f"plugins/{name}/1.0.0/plugin.zip"} |
| 21 | + for name in ("a", "b", "c") |
| 22 | + ] |
| 23 | + |
| 24 | + def test_uploads_overlap_without_exceeding_the_limit(self): |
| 25 | + release = threading.Event() |
| 26 | + two_started = threading.Event() |
| 27 | + lock = threading.Lock() |
| 28 | + active = maximum = 0 |
| 29 | + started = [] |
| 30 | + |
| 31 | + def upload(_path, _bucket, _prefix, rel, **kwargs): |
| 32 | + nonlocal active, maximum |
| 33 | + self.assertEqual(kwargs["timeout_scale"], 2) |
| 34 | + with lock: |
| 35 | + active += 1 |
| 36 | + maximum = max(maximum, active) |
| 37 | + started.append(rel) |
| 38 | + if active == 2: |
| 39 | + two_started.set() |
| 40 | + try: |
| 41 | + if not release.wait(5): |
| 42 | + raise AssertionError("test did not release the uploads") |
| 43 | + finally: |
| 44 | + with lock: |
| 45 | + active -= 1 |
| 46 | + |
| 47 | + with patch.object(publish, "DRY_RUN", False), \ |
| 48 | + patch.object(publish, "oss_upload", side_effect=upload), \ |
| 49 | + patch.object(publish, "log"), \ |
| 50 | + ThreadPoolExecutor(max_workers=1) as caller: |
| 51 | + result = caller.submit(publish.upload_archives, self.items, "bucket", "prefix", 2) |
| 52 | + try: |
| 53 | + self.assertTrue(two_started.wait(5), "archive uploads did not overlap") |
| 54 | + self.assertFalse(result.done(), "returned before uploads completed") |
| 55 | + with lock: |
| 56 | + self.assertEqual(len(started), 2) |
| 57 | + finally: |
| 58 | + release.set() |
| 59 | + result.result(timeout=5) |
| 60 | + self.assertEqual(maximum, 2) |
| 61 | + self.assertEqual(active, 0) |
| 62 | + self.assertCountEqual(started, [item["rel"] for item in self.items]) |
| 63 | + |
| 64 | + def test_dry_run_and_single_worker_keep_deterministic_order(self): |
| 65 | + for dry_run, workers in ((True, 4), (False, 1)): |
| 66 | + with self.subTest(dry_run=dry_run, workers=workers), \ |
| 67 | + patch.object(publish, "DRY_RUN", dry_run), \ |
| 68 | + patch.object(publish, "ThreadPoolExecutor", side_effect=AssertionError("unexpected worker")), \ |
| 69 | + patch.object(publish, "oss_upload") as upload: |
| 70 | + publish.upload_archives(self.items, "bucket", "prefix", workers) |
| 71 | + self.assertEqual([c.args[3] for c in upload.call_args_list], |
| 72 | + [item["rel"] for item in self.items]) |
| 73 | + |
| 74 | + def test_default_and_override(self): |
| 75 | + with patch.dict(os.environ, {}, clear=True): |
| 76 | + self.assertEqual(publish.archive_upload_concurrency(), 4) |
| 77 | + with patch.dict(os.environ, {"OSS_UPLOAD_CONCURRENCY": "1"}): |
| 78 | + self.assertEqual(publish.archive_upload_concurrency(), 1) |
| 79 | + |
| 80 | + def test_parallel_timeout_allows_for_shared_uplink(self): |
| 81 | + size = 8 * 1024 * 1024 |
| 82 | + serial = publish.upload_timeout(size) - publish.OSS_UPLOAD_BASE_TIMEOUT |
| 83 | + parallel = publish.upload_timeout(size, 4) - publish.OSS_UPLOAD_BASE_TIMEOUT |
| 84 | + self.assertEqual(parallel, serial * 4) |
| 85 | + |
| 86 | + def test_failure_waits_for_in_flight_uploads_before_returning(self): |
| 87 | + second_started = threading.Event() |
| 88 | + failure_raised = threading.Event() |
| 89 | + release = threading.Event() |
| 90 | + second_finished = threading.Event() |
| 91 | + |
| 92 | + def upload(_path, _bucket, _prefix, rel, **_kwargs): |
| 93 | + if rel.startswith("plugins/a/"): |
| 94 | + if not second_started.wait(5): |
| 95 | + raise AssertionError("second upload did not start") |
| 96 | + failure_raised.set() |
| 97 | + raise RuntimeError("upload failed") |
| 98 | + second_started.set() |
| 99 | + if not release.wait(5): |
| 100 | + raise AssertionError("test did not release in-flight upload") |
| 101 | + second_finished.set() |
| 102 | + |
| 103 | + with patch.object(publish, "DRY_RUN", False), \ |
| 104 | + patch.object(publish, "oss_upload", side_effect=upload), \ |
| 105 | + patch.object(publish, "log"), \ |
| 106 | + ThreadPoolExecutor(max_workers=1) as caller: |
| 107 | + result = caller.submit(publish.upload_archives, self.items[:2], "bucket", "prefix", 2) |
| 108 | + try: |
| 109 | + self.assertTrue(failure_raised.wait(5)) |
| 110 | + self.assertFalse(result.done()) |
| 111 | + finally: |
| 112 | + release.set() |
| 113 | + with self.assertRaisesRegex(RuntimeError, "upload failed"): |
| 114 | + result.result(timeout=5) |
| 115 | + self.assertTrue(second_finished.is_set()) |
| 116 | + |
| 117 | + |
| 118 | +class CatalogBarrierTest(unittest.TestCase): |
| 119 | + def setUp(self): |
| 120 | + temporary = tempfile.TemporaryDirectory() |
| 121 | + self.addCleanup(temporary.cleanup) |
| 122 | + self.root = Path(temporary.name) |
| 123 | + self.output = self.root / "output" |
| 124 | + self.output.mkdir() |
| 125 | + entries = [] |
| 126 | + for name in ("a", "b"): |
| 127 | + (self.root / "plugins" / name).mkdir(parents=True) |
| 128 | + entries.append({"name": name, "version": "1.0.0", "source": f"./plugins/{name}"}) |
| 129 | + (self.root / "marketplace.json").write_text(json.dumps({"name": "test", "plugins": entries})) |
| 130 | + |
| 131 | + def build(_directory, path): |
| 132 | + path.parent.mkdir(parents=True, exist_ok=True) |
| 133 | + path.write_bytes(b"archive") |
| 134 | + |
| 135 | + patches = [ |
| 136 | + patch.dict(os.environ, {"OSS_BUCKET": "bucket", "OSS_PATH_PREFIX": "prefix", |
| 137 | + "CDN_BASE_URL": "https://cdn.example.test", |
| 138 | + "OSS_UPLOAD_CONCURRENCY": "2"}), |
| 139 | + patch.multiple(publish, ROOT=self.root, DRY_RUN=False, |
| 140 | + FAILURE_FILE=self.root / "failure.txt"), |
| 141 | + patch.object(publish.shutil, "which", return_value="/test/ossutil"), |
| 142 | + patch.object(publish, "preflight_cache_control_style"), |
| 143 | + patch.object(publish.tempfile, "mkdtemp", return_value=str(self.output)), |
| 144 | + patch.object(publish, "oss_download", return_value=False), |
| 145 | + patch.object(publish, "oss_object_exists", return_value=False), |
| 146 | + patch.object(publish, "build_zip", side_effect=build), |
| 147 | + patch.object(publish, "build_index_entry", side_effect=lambda entry, *_: dict(entry)), |
| 148 | + patch.object(publish, "plan_assets", return_value=([], [], 0)), |
| 149 | + patch.object(publish, "oss_upload_assets"), |
| 150 | + patch.object(publish, "log"), |
| 151 | + ] |
| 152 | + for patcher in patches: |
| 153 | + patcher.start() |
| 154 | + self.addCleanup(patcher.stop) |
| 155 | + |
| 156 | + def test_catalog_is_uploaded_only_after_every_archive_finishes(self): |
| 157 | + completed = set() |
| 158 | + lock = threading.Lock() |
| 159 | + |
| 160 | + def upload(_path, _bucket, _prefix, rel, **_kwargs): |
| 161 | + with lock: |
| 162 | + if rel == "marketplace.json": |
| 163 | + self.assertEqual(completed, {"a", "b"}) |
| 164 | + else: |
| 165 | + completed.add(rel.split("/")[1]) |
| 166 | + |
| 167 | + with patch.object(publish, "oss_upload", side_effect=upload) as uploads, \ |
| 168 | + patch.object(publish, "preload_cdn"), patch.object(publish, "refresh_cdn"): |
| 169 | + self.assertEqual(publish.main(), 0) |
| 170 | + self.assertEqual(uploads.call_args_list[-1].args[3], "marketplace.json") |
| 171 | + |
| 172 | + def test_failed_archive_prevents_catalog_and_cdn_publication(self): |
| 173 | + def upload(_path, _bucket, _prefix, rel, **_kwargs): |
| 174 | + if rel.startswith("plugins/a/"): |
| 175 | + raise SystemExit(1) |
| 176 | + |
| 177 | + with patch.object(publish, "oss_upload", side_effect=upload) as uploads, \ |
| 178 | + patch.object(publish, "preload_cdn") as preload, \ |
| 179 | + patch.object(publish, "refresh_cdn") as refresh: |
| 180 | + with self.assertRaises(SystemExit): |
| 181 | + publish.main() |
| 182 | + self.assertNotIn("marketplace.json", [c.args[3] for c in uploads.call_args_list]) |
| 183 | + preload.assert_not_called() |
| 184 | + refresh.assert_not_called() |
| 185 | + |
| 186 | + def test_invalid_limit_fails_before_any_remote_operation(self): |
| 187 | + for value in ("0", "-1", "17", "many"): |
| 188 | + with self.subTest(value=value), \ |
| 189 | + patch.dict(os.environ, {"OSS_UPLOAD_CONCURRENCY": value}), \ |
| 190 | + patch.object(publish, "preflight_cache_control_style") as preflight: |
| 191 | + with self.assertRaises(SystemExit): |
| 192 | + publish.main() |
| 193 | + preflight.assert_not_called() |
| 194 | + |
| 195 | + |
| 196 | +if __name__ == "__main__": |
| 197 | + unittest.main() |
0 commit comments