From 3875dd0aff3e27457bf375a373f544ee0bdb36e3 Mon Sep 17 00:00:00 2001 From: 2vibeornot2vibe Date: Sat, 30 May 2026 16:15:55 +0800 Subject: [PATCH 1/3] Use pre-built wheel for flash-attn on Linux --- setup_config.json | 122 +++++++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/setup_config.json b/setup_config.json index 402f77d18..c5cfe8aa3 100644 --- a/setup_config.json +++ b/setup_config.json @@ -4,26 +4,26 @@ "3.10": { "ver": "3.10.9", "label": "Python 3.10.9 (Legacy)" }, "3.11": { "ver": "3.11.14", "label": "Python 3.11.14 (Recommended)" } }, - "torch": { - "cu128": { - "label": "Torch 2.7.1 + CUDA 12.8", - "cmd": "torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu128" - }, - "cu130": { - "label": "Torch 2.10.0 + CUDA 13.0", - "cmd": "torch==2.10.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130" - }, - "rocm65": { - "label": "ROCm 6.5 (TheRock)", - "cmd": { - "win": "https://github.com/scottt/rocm-TheRock/releases/download/v6.5.0rc-pytorch-gfx110x/torch-2.7.0a0+rocm_git3f903c3-cp311-cp311-win_amd64.whl https://github.com/scottt/rocm-TheRock/releases/download/v6.5.0rc-pytorch-gfx110x/torchvision-0.22.0+9eb57cd-cp311-cp311-win_amd64.whl https://github.com/scottt/rocm-TheRock/releases/download/v6.5.0rc-pytorch-gfx110x/torchaudio-2.7.0a0+52638ef-cp311-cp311-win_amd64.whl", - "linux": "torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.2" - } - }, - "mps": { - "label": "Torch (MPS / Apple Silicon)", - "cmd": "torch torchvision torchaudio" - } + "torch": { + "cu128": { + "label": "Torch 2.7.1 + CUDA 12.8", + "cmd": "torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu128" + }, + "cu130": { + "label": "Torch 2.10.0 + CUDA 13.0", + "cmd": "torch==2.10.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130" + }, + "rocm65": { + "label": "ROCm 6.5 (TheRock)", + "cmd": { + "win": "https://github.com/scottt/rocm-TheRock/releases/download/v6.5.0rc-pytorch-gfx110x/torch-2.7.0a0+rocm_git3f903c3-cp311-cp311-win_amd64.whl https://github.com/scottt/rocm-TheRock/releases/download/v6.5.0rc-pytorch-gfx110x/torchvision-0.22.0+9eb57cd-cp311-cp311-win_amd64.whl https://github.com/scottt/rocm-TheRock/releases/download/v6.5.0rc-pytorch-gfx110x/torchaudio-2.7.0a0+52638ef-cp311-cp311-win_amd64.whl", + "linux": "torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.2" + } + }, + "mps": { + "label": "Torch (MPS / Apple Silicon)", + "cmd": "torch torchvision torchaudio" + } }, "triton": { "v33": { @@ -48,11 +48,11 @@ } } }, - "sage": { - "v1": { "label": "Sage Attention 1.0.6", "cmd": "sageattention==1.0.6" }, - "v211": { - "label": "Sage Attention 2.1.1", - "cmd": { + "sage": { + "v1": { "label": "Sage Attention 1.0.6", "cmd": "sageattention==1.0.6" }, + "v211": { + "label": "Sage Attention 2.1.1", + "cmd": { "win": "https://github.com/woct0rdho/SageAttention/releases/download/v2.1.1-windows/sageattention-2.1.1+cu126torch2.6.0-cp310-cp310-win_amd64.whl", "linux": "pip install \"setuptools<=75.8.2\" ninja wheel --force-reinstall && git clone https://github.com/thu-ml/SageAttention && pip install --no-build-isolation -e SageAttention" } @@ -69,31 +69,31 @@ "cmd": { "win": "https://github.com/woct0rdho/SageAttention/releases/download/v2.2.0-windows.post4/sageattention-2.2.0+cu130torch2.9.0andhigher.post4-cp39-abi3-win_amd64.whl", "linux": "pip install \"setuptools<=75.8.2\" ninja wheel --force-reinstall && git clone https://github.com/thu-ml/SageAttention && pip install --no-build-isolation -e SageAttention" - } - } - }, - "sparge": { - "v010_cu128": { - "label": "Sparge Attention 0.1.0 (CUDA 12.8 / Torch 2.7.1)", - "cmd": { - "win": "https://github.com/woct0rdho/SpargeAttn/releases/download/v0.1.0-windows.post3/spas_sage_attn-0.1.0%2Bcu128torch2.7.1.post3-cp39-abi3-win_amd64.whl", - "linux": "{pip} ninja wheel packaging && {pip} --no-build-isolation git+https://github.com/woct0rdho/SpargeAttn.git" - } - }, - "v010_cu13": { - "label": "Sparge Attention 0.1.0 (CUDA 13 / Torch 2.9+ ABI)", - "cmd": { - "win": "https://github.com/woct0rdho/SpargeAttn/releases/download/v0.1.0-windows.post4/spas_sage_attn-0.1.0%2Bcu130torch2.9.0andhigher.post4-cp39-abi3-win_amd64.whl", - "linux": "{pip} ninja wheel packaging && {pip} --no-build-isolation git+https://github.com/woct0rdho/SpargeAttn.git" - } - } - }, - "flash": { - "v210": { - "label": "Flash Attention 2.8.x", - "cmd": { + } + } + }, + "sparge": { + "v010_cu128": { + "label": "Sparge Attention 0.1.0 (CUDA 12.8 / Torch 2.7.1)", + "cmd": { + "win": "https://github.com/woct0rdho/SpargeAttn/releases/download/v0.1.0-windows.post3/spas_sage_attn-0.1.0%2Bcu128torch2.7.1.post3-cp39-abi3-win_amd64.whl", + "linux": "{pip} ninja wheel packaging && {pip} --no-build-isolation git+https://github.com/woct0rdho/SpargeAttn.git" + } + }, + "v010_cu13": { + "label": "Sparge Attention 0.1.0 (CUDA 13 / Torch 2.9+ ABI)", + "cmd": { + "win": "https://github.com/woct0rdho/SpargeAttn/releases/download/v0.1.0-windows.post4/spas_sage_attn-0.1.0%2Bcu130torch2.9.0andhigher.post4-cp39-abi3-win_amd64.whl", + "linux": "{pip} ninja wheel packaging && {pip} --no-build-isolation git+https://github.com/woct0rdho/SpargeAttn.git" + } + } + }, + "flash": { + "v210": { + "label": "Flash Attention 2.8.x", + "cmd": { "win": "https://github.com/deepbeepmeep/kernels/releases/download/Flash2/flash_attn-2.8.3-cp311-cp311-win_amd64.whl", - "linux": "flash-attn" + "linux": "https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.0/flash_attn-2.8.3+cu130torch2.10-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl" } } }, @@ -128,15 +128,15 @@ } } }, - "gpu_profiles": { - "GTX_10": { "python": "3.10", "torch": "cu128", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [] }, - "RTX_20": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v1", "sparge": null, "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, - "RTX_30": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, - "RTX_40": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, - "RTX_50": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13","light2xv", "gguf"] }, - "AMD_GFX110X": { "python": "3.11", "torch": "rocm65", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "env": {"HSA_OVERRIDE_GFX_VERSION": "11.0.0"} }, - "AMD_GFX1151": { "python": "3.11", "torch": "rocm65", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "env": {"HSA_OVERRIDE_GFX_VERSION": "11.5.1"} }, - "AMD_GFX1201": { "python": "3.11", "torch": "rocm65", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "env": {"HSA_OVERRIDE_GFX_VERSION": "12.0.1"} }, - "MPS": { "python": "3.11", "torch": "mps", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "attention": "sdpa" } - } -} + "gpu_profiles": { + "GTX_10": { "python": "3.10", "torch": "cu128", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [] }, + "RTX_20": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v1", "sparge": null, "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, + "RTX_30": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, + "RTX_40": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, + "RTX_50": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13","light2xv", "gguf"] }, + "AMD_GFX110X": { "python": "3.11", "torch": "rocm65", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "env": {"HSA_OVERRIDE_GFX_VERSION": "11.0.0"} }, + "AMD_GFX1151": { "python": "3.11", "torch": "rocm65", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "env": {"HSA_OVERRIDE_GFX_VERSION": "11.5.1"} }, + "AMD_GFX1201": { "python": "3.11", "torch": "rocm65", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "env": {"HSA_OVERRIDE_GFX_VERSION": "12.0.1"} }, + "MPS": { "python": "3.11", "torch": "mps", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [], "attention": "sdpa" } + } +} From 4bd7e33b4d0071b82ea03e33652855e0a1d5f272 Mon Sep 17 00:00:00 2001 From: 2vibeornot2vibe Date: Sat, 30 May 2026 16:41:41 +0800 Subject: [PATCH 2/3] Specify torchvision & torchaudio versions and update CUDA 12.8 config to torch 2.10.0 --- setup_config.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup_config.json b/setup_config.json index c5cfe8aa3..0f624a3a8 100644 --- a/setup_config.json +++ b/setup_config.json @@ -6,12 +6,12 @@ }, "torch": { "cu128": { - "label": "Torch 2.7.1 + CUDA 12.8", - "cmd": "torch==2.7.1 torchvision==0.22.1 torchaudio==2.7.1 --index-url https://download.pytorch.org/whl/cu128" + "label": "Torch 2.10.0 + CUDA 12.8", + "cmd": "torch==2.10.0+cu128 torchvision==0.25.0+cu128 torchaudio==2.10.0+cu128 --index-url https://download.pytorch.org/whl/cu128" }, "cu130": { "label": "Torch 2.10.0 + CUDA 13.0", - "cmd": "torch==2.10.0 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu130" + "cmd": "torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cu130" }, "rocm65": { "label": "ROCm 6.5 (TheRock)", From 57499eb61ed137f4549e0a8b34e507fb0ba965de Mon Sep 17 00:00:00 2001 From: 2vibeornot2vibe Date: Sat, 30 May 2026 17:32:59 +0800 Subject: [PATCH 3/3] Use Python 3.11 instead of 3.10 as default for pre-Turin cards (tested, works on Volta) --- setup_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup_config.json b/setup_config.json index 0f624a3a8..d63bad980 100644 --- a/setup_config.json +++ b/setup_config.json @@ -129,7 +129,7 @@ } }, "gpu_profiles": { - "GTX_10": { "python": "3.10", "torch": "cu128", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [] }, + "GTX_10": { "python": "3.11", "torch": "cu128", "triton": null, "sage": null, "sparge": null, "flash": null, "kernels": [] }, "RTX_20": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v1", "sparge": null, "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, "RTX_30": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] }, "RTX_40": { "python": "3.11", "torch": "cu130", "triton": "latest", "sage": "v220_cu13", "sparge": "v010_cu13", "flash": "v210", "kernels": ["nunchaku_cu13", "gguf"] },