Skip to content

Commit 6c502bd

Browse files
msaroufimmsaroufim
committed
Record fresh T4 PCH benchmarks on current main
1 parent 38b9892 commit 6c502bd

2 files changed

Lines changed: 330 additions & 1 deletion

File tree

‎docs/benchmarks/modal-pch-current-main-2026-09-07.json‎

Lines changed: 302 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,302 @@
1+
{
2+
"date": "2026-09-07",
3+
"base_commit": "727212cdcf1b9b4d587c12f6d1484b3fd54549d0",
4+
"implementation_commit": "38b98928",
5+
"cuda_image_version": "13.3.0",
6+
"torch": "2.12.0+cu130",
7+
"cpu": 4,
8+
"memory_mib": 16384,
9+
"normal_ninja_parallelism": true,
10+
"volume": "kernelbot-pch-test-20260907",
11+
"warmed_profiles": [
12+
"cuda-default"
13+
],
14+
"run_links": {
15+
"minimal": "https://modal.com/apps/coreauto/main/ap-iNSFXRa4TIP80OqKP6Evxs",
16+
"implicit": "https://modal.com/apps/coreauto/main/ap-A5cGWJRnQYzTttK1ZeEbTV"
17+
},
18+
"summary": {
19+
"minimal": {
20+
"cpp_seconds": {
21+
"cold": 19.36,
22+
"warm": 8.906500000000001,
23+
"speedup": 2.173693370010666
24+
},
25+
"load_inline_seconds": {
26+
"cold": 20.447000000000003,
27+
"warm": 9.91,
28+
"speedup": 2.063269424823411
29+
}
30+
},
31+
"implicit": {
32+
"cpp_seconds": {
33+
"cold": 18.441,
34+
"warm": 8.7895,
35+
"speedup": 2.098071562659992
36+
},
37+
"load_inline_seconds": {
38+
"cold": 38.096000000000004,
39+
"warm": 43.245999999999995,
40+
"speedup": 0.8809138417425891
41+
}
42+
}
43+
},
44+
"rows": [
45+
{
46+
"api_write_denied": true,
47+
"enabled": false,
48+
"gpu": "Tesla T4",
49+
"torch": "2.12.0+cu130",
50+
"container_id": "ta-01M1YS0N16K76Q8V4C3VF51N3R",
51+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
52+
"load_inline_seconds": 19.445,
53+
"cpp_seconds": [
54+
18.61
55+
],
56+
"pch_consumed": false,
57+
"keys": [],
58+
"tests": {
59+
"test-count": "5",
60+
"test.0.spec": "size: 127; seed: 4242",
61+
"test.0.status": "pass",
62+
"test.1.spec": "size: 128; seed: 5236",
63+
"test.1.status": "pass",
64+
"test.2.spec": "size: 129; seed: 1001",
65+
"test.2.status": "pass",
66+
"test.3.spec": "size: 256; seed: 5531",
67+
"test.3.status": "pass",
68+
"test.4.spec": "size: 512; seed: 9173",
69+
"test.4.status": "pass",
70+
"check": "pass"
71+
},
72+
"remote_seconds": 31.217458912999998,
73+
"headers": "minimal",
74+
"variant": 0
75+
},
76+
{
77+
"api_write_denied": true,
78+
"enabled": true,
79+
"gpu": "Tesla T4",
80+
"torch": "2.12.0+cu130",
81+
"container_id": "ta-01M1YS1TYTV9BKMDNRHENGDD9R",
82+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
83+
"load_inline_seconds": 9.23,
84+
"cpp_seconds": [
85+
8.278
86+
],
87+
"pch_consumed": true,
88+
"keys": [
89+
"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38"
90+
],
91+
"tests": {
92+
"test-count": "5",
93+
"test.0.spec": "size: 127; seed: 4242",
94+
"test.0.status": "pass",
95+
"test.1.spec": "size: 128; seed: 5236",
96+
"test.1.status": "pass",
97+
"test.2.spec": "size: 129; seed: 1001",
98+
"test.2.status": "pass",
99+
"test.3.spec": "size: 256; seed: 5531",
100+
"test.3.status": "pass",
101+
"test.4.spec": "size: 512; seed: 9173",
102+
"test.4.status": "pass",
103+
"check": "pass"
104+
},
105+
"remote_seconds": 21.60613694,
106+
"headers": "minimal",
107+
"variant": 0
108+
},
109+
{
110+
"api_write_denied": true,
111+
"enabled": false,
112+
"gpu": "Tesla T4",
113+
"torch": "2.12.0+cu130",
114+
"container_id": "ta-01M1YS2KZ7R00G1ANY0ZQMS1TR",
115+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
116+
"load_inline_seconds": 21.449,
117+
"cpp_seconds": [
118+
20.11
119+
],
120+
"pch_consumed": false,
121+
"keys": [],
122+
"tests": {
123+
"test-count": "5",
124+
"test.0.spec": "size: 127; seed: 4242",
125+
"test.0.status": "pass",
126+
"test.1.spec": "size: 128; seed: 5236",
127+
"test.1.status": "pass",
128+
"test.2.spec": "size: 129; seed: 1001",
129+
"test.2.status": "pass",
130+
"test.3.spec": "size: 256; seed: 5531",
131+
"test.3.status": "pass",
132+
"test.4.spec": "size: 512; seed: 9173",
133+
"test.4.status": "pass",
134+
"check": "pass"
135+
},
136+
"remote_seconds": 33.334485347,
137+
"headers": "minimal",
138+
"variant": 1
139+
},
140+
{
141+
"api_write_denied": true,
142+
"enabled": true,
143+
"gpu": "Tesla T4",
144+
"torch": "2.12.0+cu130",
145+
"container_id": "ta-01M1YS3T71BGNZHNW70VDM0ZTR",
146+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
147+
"load_inline_seconds": 10.59,
148+
"cpp_seconds": [
149+
9.535
150+
],
151+
"pch_consumed": true,
152+
"keys": [
153+
"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38"
154+
],
155+
"tests": {
156+
"test-count": "5",
157+
"test.0.spec": "size: 127; seed: 4242",
158+
"test.0.status": "pass",
159+
"test.1.spec": "size: 128; seed: 5236",
160+
"test.1.status": "pass",
161+
"test.2.spec": "size: 129; seed: 1001",
162+
"test.2.status": "pass",
163+
"test.3.spec": "size: 256; seed: 5531",
164+
"test.3.status": "pass",
165+
"test.4.spec": "size: 512; seed: 9173",
166+
"test.4.status": "pass",
167+
"check": "pass"
168+
},
169+
"remote_seconds": 26.458702556,
170+
"headers": "minimal",
171+
"variant": 1
172+
},
173+
{
174+
"api_write_denied": true,
175+
"enabled": false,
176+
"gpu": "Tesla T4",
177+
"torch": "2.12.0+cu130",
178+
"container_id": "ta-01M1YS2KSKB7VPRNKQXTVN7S2R",
179+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
180+
"load_inline_seconds": 38.816,
181+
"cpp_seconds": [
182+
19.127
183+
],
184+
"pch_consumed": false,
185+
"keys": [],
186+
"tests": {
187+
"test-count": "5",
188+
"test.0.spec": "size: 127; seed: 4242",
189+
"test.0.status": "pass",
190+
"test.1.spec": "size: 128; seed: 5236",
191+
"test.1.status": "pass",
192+
"test.2.spec": "size: 129; seed: 1001",
193+
"test.2.status": "pass",
194+
"test.3.spec": "size: 256; seed: 5531",
195+
"test.3.status": "pass",
196+
"test.4.spec": "size: 512; seed: 9173",
197+
"test.4.status": "pass",
198+
"check": "pass"
199+
},
200+
"remote_seconds": 51.25590044,
201+
"headers": "implicit",
202+
"variant": 0
203+
},
204+
{
205+
"api_write_denied": true,
206+
"enabled": true,
207+
"gpu": "Tesla T4",
208+
"torch": "2.12.0+cu130",
209+
"container_id": "ta-01M1YS4C0R2T6HGV7RYZYNKZVR",
210+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
211+
"load_inline_seconds": 49.988,
212+
"cpp_seconds": [
213+
10.423
214+
],
215+
"pch_consumed": true,
216+
"keys": [
217+
"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38"
218+
],
219+
"tests": {
220+
"test-count": "5",
221+
"test.0.spec": "size: 127; seed: 4242",
222+
"test.0.status": "pass",
223+
"test.1.spec": "size: 128; seed: 5236",
224+
"test.1.status": "pass",
225+
"test.2.spec": "size: 129; seed: 1001",
226+
"test.2.status": "pass",
227+
"test.3.spec": "size: 256; seed: 5531",
228+
"test.3.status": "pass",
229+
"test.4.spec": "size: 512; seed: 9173",
230+
"test.4.status": "pass",
231+
"check": "pass"
232+
},
233+
"remote_seconds": 67.888768213,
234+
"headers": "implicit",
235+
"variant": 0
236+
},
237+
{
238+
"api_write_denied": true,
239+
"enabled": false,
240+
"gpu": "Tesla T4",
241+
"torch": "2.12.0+cu130",
242+
"container_id": "ta-01M1YS6W4ESSF84B2K39QJEMZR",
243+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
244+
"load_inline_seconds": 37.376,
245+
"cpp_seconds": [
246+
17.755
247+
],
248+
"pch_consumed": false,
249+
"keys": [],
250+
"tests": {
251+
"test-count": "5",
252+
"test.0.spec": "size: 127; seed: 4242",
253+
"test.0.status": "pass",
254+
"test.1.spec": "size: 128; seed: 5236",
255+
"test.1.status": "pass",
256+
"test.2.spec": "size: 129; seed: 1001",
257+
"test.2.status": "pass",
258+
"test.3.spec": "size: 256; seed: 5531",
259+
"test.3.status": "pass",
260+
"test.4.spec": "size: 512; seed: 9173",
261+
"test.4.status": "pass",
262+
"check": "pass"
263+
},
264+
"remote_seconds": 48.367798831,
265+
"headers": "implicit",
266+
"variant": 1
267+
},
268+
{
269+
"api_write_denied": true,
270+
"enabled": true,
271+
"gpu": "Tesla T4",
272+
"torch": "2.12.0+cu130",
273+
"container_id": "ta-01M1YS8GAY01Q5Y1XPHQ0E9DRR",
274+
"image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
275+
"load_inline_seconds": 36.504,
276+
"cpp_seconds": [
277+
7.156
278+
],
279+
"pch_consumed": true,
280+
"keys": [
281+
"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38"
282+
],
283+
"tests": {
284+
"test-count": "5",
285+
"test.0.spec": "size: 127; seed: 4242",
286+
"test.0.status": "pass",
287+
"test.1.spec": "size: 128; seed: 5236",
288+
"test.1.status": "pass",
289+
"test.2.spec": "size: 129; seed: 1001",
290+
"test.2.status": "pass",
291+
"test.3.spec": "size: 256; seed: 5531",
292+
"test.3.status": "pass",
293+
"test.4.spec": "size: 512; seed: 9173",
294+
"test.4.status": "pass",
295+
"check": "pass"
296+
},
297+
"remote_seconds": 47.203748833999995,
298+
"headers": "implicit",
299+
"variant": 1
300+
}
301+
]
302+
}

‎docs/modal-pch-cache.md‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,34 @@ To diagnose a submission, `KERNELBOT_PCH_TRACE=1` prints the host compile time a
113113
GCC's header trace on PCH-enabled compilations. `KERNELBOT_PCH_DISABLE=1` provides
114114
a baseline without the cache.
115115

116-
## Measured results (2026-09-07)
116+
## Current-main measurements (2026-09-07)
117+
118+
Revalidated on main `727212cd`, preserving CUDA 13.3.0, PyTorch 2.12.0+cu130,
119+
and MathDx. Both fixtures use the same image and freshly warmed `cuda-default`
120+
profile, normal Ninja parallelism, and eight distinct T4 containers.
121+
122+
| Submission | C++ compilation, cold → PCH | Total `load_inline`, cold → PCH |
123+
| --- | --- | --- |
124+
| Minimal CUDA headers | 19.36s → 8.91s (2.17×) | 20.45s → 9.91s (2.06×) |
125+
| Original implicit headers | 18.44s → 8.79s (2.10×) | 38.10s → 43.25s (0.88×) |
126+
127+
All 40 correctness checks passed. Every cache-enabled build consumed the same
128+
PCH, and every container rejected filesystem and Modal API writes. The
129+
Torch-heavy CUDA fixture did **not** improve end-to-end in this run: warm totals
130+
were 49.99s and 36.50s, despite faster host compilation. These two-sample medians
131+
show the host-stage benefit and the minimal-header fixture's total-time benefit;
132+
they do not establish an end-to-end speedup for NVCC-heavy submissions.
133+
134+
41 relevant local tests and Ruff passed on this base. GitHub unit, lint, Modal
135+
integration, and GitHub integration checks also passed on `38b98928`.
136+
Only `cuda-default` was rewarmed on the current image; the eight-profile warmup
137+
below applies to the earlier image. The deployment workflow warms all profiles
138+
for the actual image before deploying it. Production has not been deployed by
139+
this experiment.
140+
141+
[Current-main raw measurements and run links](benchmarks/modal-pch-current-main-2026-09-07.json).
142+
143+
## Earlier-image measurements (2026-09-07)
117144

118145
T4, CUDA 12.9.1, PyTorch 2.11.0+cu129, normal Ninja parallelism. Each row below
119146
is the median of two different kernel variants, each tested in separate cold

0 commit comments

Comments
 (0)