Skip to content

Commit c4be2c2

Browse files
author
ZCode Marketplace Bot
committed
fix(ci): parallelize plugin archive uploads
Source-Commit: b874535fd71b4197781d0e6227bdd546c173f2fc
1 parent 8ecf859 commit c4be2c2

1 file changed

Lines changed: 197 additions & 0 deletions

File tree

‎tests/test_parallel_publish.py‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
from __future__ import annotations
2+
3+
import json
4+
import os
5+
import sys
6+
import tempfile
7+
import threading
8+
import unittest
9+
from concurrent.futures import ThreadPoolExecutor
10+
from pathlib import Path
11+
from unittest.mock import patch
12+
13+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
14+
import publish_incremental as publish
15+
16+
17+
class ParallelArchiveTest(unittest.TestCase):
18+
def setUp(self):
19+
self.items = [
20+
{"zip_path": Path(f"{name}.zip"), "rel": f"plugins/{name}/1.0.0/plugin.zip"}
21+
for name in ("a", "b", "c")
22+
]
23+
24+
def test_uploads_overlap_without_exceeding_the_limit(self):
25+
release = threading.Event()
26+
two_started = threading.Event()
27+
lock = threading.Lock()
28+
active = maximum = 0
29+
started = []
30+
31+
def upload(_path, _bucket, _prefix, rel, **kwargs):
32+
nonlocal active, maximum
33+
self.assertEqual(kwargs["timeout_scale"], 2)
34+
with lock:
35+
active += 1
36+
maximum = max(maximum, active)
37+
started.append(rel)
38+
if active == 2:
39+
two_started.set()
40+
try:
41+
if not release.wait(5):
42+
raise AssertionError("test did not release the uploads")
43+
finally:
44+
with lock:
45+
active -= 1
46+
47+
with patch.object(publish, "DRY_RUN", False), \
48+
patch.object(publish, "oss_upload", side_effect=upload), \
49+
patch.object(publish, "log"), \
50+
ThreadPoolExecutor(max_workers=1) as caller:
51+
result = caller.submit(publish.upload_archives, self.items, "bucket", "prefix", 2)
52+
try:
53+
self.assertTrue(two_started.wait(5), "archive uploads did not overlap")
54+
self.assertFalse(result.done(), "returned before uploads completed")
55+
with lock:
56+
self.assertEqual(len(started), 2)
57+
finally:
58+
release.set()
59+
result.result(timeout=5)
60+
self.assertEqual(maximum, 2)
61+
self.assertEqual(active, 0)
62+
self.assertCountEqual(started, [item["rel"] for item in self.items])
63+
64+
def test_dry_run_and_single_worker_keep_deterministic_order(self):
65+
for dry_run, workers in ((True, 4), (False, 1)):
66+
with self.subTest(dry_run=dry_run, workers=workers), \
67+
patch.object(publish, "DRY_RUN", dry_run), \
68+
patch.object(publish, "ThreadPoolExecutor", side_effect=AssertionError("unexpected worker")), \
69+
patch.object(publish, "oss_upload") as upload:
70+
publish.upload_archives(self.items, "bucket", "prefix", workers)
71+
self.assertEqual([c.args[3] for c in upload.call_args_list],
72+
[item["rel"] for item in self.items])
73+
74+
def test_default_and_override(self):
75+
with patch.dict(os.environ, {}, clear=True):
76+
self.assertEqual(publish.archive_upload_concurrency(), 4)
77+
with patch.dict(os.environ, {"OSS_UPLOAD_CONCURRENCY": "1"}):
78+
self.assertEqual(publish.archive_upload_concurrency(), 1)
79+
80+
def test_parallel_timeout_allows_for_shared_uplink(self):
81+
size = 8 * 1024 * 1024
82+
serial = publish.upload_timeout(size) - publish.OSS_UPLOAD_BASE_TIMEOUT
83+
parallel = publish.upload_timeout(size, 4) - publish.OSS_UPLOAD_BASE_TIMEOUT
84+
self.assertEqual(parallel, serial * 4)
85+
86+
def test_failure_waits_for_in_flight_uploads_before_returning(self):
87+
second_started = threading.Event()
88+
failure_raised = threading.Event()
89+
release = threading.Event()
90+
second_finished = threading.Event()
91+
92+
def upload(_path, _bucket, _prefix, rel, **_kwargs):
93+
if rel.startswith("plugins/a/"):
94+
if not second_started.wait(5):
95+
raise AssertionError("second upload did not start")
96+
failure_raised.set()
97+
raise RuntimeError("upload failed")
98+
second_started.set()
99+
if not release.wait(5):
100+
raise AssertionError("test did not release in-flight upload")
101+
second_finished.set()
102+
103+
with patch.object(publish, "DRY_RUN", False), \
104+
patch.object(publish, "oss_upload", side_effect=upload), \
105+
patch.object(publish, "log"), \
106+
ThreadPoolExecutor(max_workers=1) as caller:
107+
result = caller.submit(publish.upload_archives, self.items[:2], "bucket", "prefix", 2)
108+
try:
109+
self.assertTrue(failure_raised.wait(5))
110+
self.assertFalse(result.done())
111+
finally:
112+
release.set()
113+
with self.assertRaisesRegex(RuntimeError, "upload failed"):
114+
result.result(timeout=5)
115+
self.assertTrue(second_finished.is_set())
116+
117+
118+
class CatalogBarrierTest(unittest.TestCase):
119+
def setUp(self):
120+
temporary = tempfile.TemporaryDirectory()
121+
self.addCleanup(temporary.cleanup)
122+
self.root = Path(temporary.name)
123+
self.output = self.root / "output"
124+
self.output.mkdir()
125+
entries = []
126+
for name in ("a", "b"):
127+
(self.root / "plugins" / name).mkdir(parents=True)
128+
entries.append({"name": name, "version": "1.0.0", "source": f"./plugins/{name}"})
129+
(self.root / "marketplace.json").write_text(json.dumps({"name": "test", "plugins": entries}))
130+
131+
def build(_directory, path):
132+
path.parent.mkdir(parents=True, exist_ok=True)
133+
path.write_bytes(b"archive")
134+
135+
patches = [
136+
patch.dict(os.environ, {"OSS_BUCKET": "bucket", "OSS_PATH_PREFIX": "prefix",
137+
"CDN_BASE_URL": "https://cdn.example.test",
138+
"OSS_UPLOAD_CONCURRENCY": "2"}),
139+
patch.multiple(publish, ROOT=self.root, DRY_RUN=False,
140+
FAILURE_FILE=self.root / "failure.txt"),
141+
patch.object(publish.shutil, "which", return_value="/test/ossutil"),
142+
patch.object(publish, "preflight_cache_control_style"),
143+
patch.object(publish.tempfile, "mkdtemp", return_value=str(self.output)),
144+
patch.object(publish, "oss_download", return_value=False),
145+
patch.object(publish, "oss_object_exists", return_value=False),
146+
patch.object(publish, "build_zip", side_effect=build),
147+
patch.object(publish, "build_index_entry", side_effect=lambda entry, *_: dict(entry)),
148+
patch.object(publish, "plan_assets", return_value=([], [], 0)),
149+
patch.object(publish, "oss_upload_assets"),
150+
patch.object(publish, "log"),
151+
]
152+
for patcher in patches:
153+
patcher.start()
154+
self.addCleanup(patcher.stop)
155+
156+
def test_catalog_is_uploaded_only_after_every_archive_finishes(self):
157+
completed = set()
158+
lock = threading.Lock()
159+
160+
def upload(_path, _bucket, _prefix, rel, **_kwargs):
161+
with lock:
162+
if rel == "marketplace.json":
163+
self.assertEqual(completed, {"a", "b"})
164+
else:
165+
completed.add(rel.split("/")[1])
166+
167+
with patch.object(publish, "oss_upload", side_effect=upload) as uploads, \
168+
patch.object(publish, "preload_cdn"), patch.object(publish, "refresh_cdn"):
169+
self.assertEqual(publish.main(), 0)
170+
self.assertEqual(uploads.call_args_list[-1].args[3], "marketplace.json")
171+
172+
def test_failed_archive_prevents_catalog_and_cdn_publication(self):
173+
def upload(_path, _bucket, _prefix, rel, **_kwargs):
174+
if rel.startswith("plugins/a/"):
175+
raise SystemExit(1)
176+
177+
with patch.object(publish, "oss_upload", side_effect=upload) as uploads, \
178+
patch.object(publish, "preload_cdn") as preload, \
179+
patch.object(publish, "refresh_cdn") as refresh:
180+
with self.assertRaises(SystemExit):
181+
publish.main()
182+
self.assertNotIn("marketplace.json", [c.args[3] for c in uploads.call_args_list])
183+
preload.assert_not_called()
184+
refresh.assert_not_called()
185+
186+
def test_invalid_limit_fails_before_any_remote_operation(self):
187+
for value in ("0", "-1", "17", "many"):
188+
with self.subTest(value=value), \
189+
patch.dict(os.environ, {"OSS_UPLOAD_CONCURRENCY": value}), \
190+
patch.object(publish, "preflight_cache_control_style") as preflight:
191+
with self.assertRaises(SystemExit):
192+
publish.main()
193+
preflight.assert_not_called()
194+
195+
196+
if __name__ == "__main__":
197+
unittest.main()

0 commit comments

Comments
 (0)