Fix installer and ROCm environment setup
Browse files- .gitattributes +1 -0
- Ling-3.0-Flash-CIRU-int4-Strix-native.tar.gz +2 -2
- README.md +25 -7
- SHA256SUMS +9 -8
- docs/UBUNTU_BUILD.md +23 -0
- install.sh +236 -0
- scripts/build-vllm-gfx1151.sh +14 -1
- scripts/download-model.sh +33 -3
- scripts/install-host-deps.sh +0 -1
- scripts/run-common.sh +25 -3
.gitattributes
CHANGED
|
@@ -3,4 +3,5 @@
|
|
| 3 |
*.sh text eol=lf
|
| 4 |
*.md text eol=lf
|
| 5 |
*.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 6 |
Ling-3.0-Flash-CIRU-int4-Strix-native.tar.gz filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 3 |
*.sh text eol=lf
|
| 4 |
*.md text eol=lf
|
| 5 |
*.png filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 7 |
Ling-3.0-Flash-CIRU-int4-Strix-native.tar.gz filter=lfs diff=lfs merge=lfs -text
|
Ling-3.0-Flash-CIRU-int4-Strix-native.tar.gz
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5fbc813c26bb470d7a38c6fbfa08ca099cd4455b1af3f2db32117133a3fb5515
|
| 3 |
+
size 268772
|
README.md
CHANGED
|
@@ -30,7 +30,7 @@ The model weights were **not** changed, requantized, merged, fine-tuned, or rena
|
|
| 30 |
|
| 31 |
The runtime is based on vLLM commit `d35eb6c44071ea806018841c490f0d2f3219c485` plus a 15-commit CIRU branch ending at `388d82de2abe418e73380ba250270d3abc62ac12`. Its net source delta is six files, 365 insertions, and 26 deletions.
|
| 32 |
|
| 33 |
-
-
|
| 34 |
- Adds an opt-in safe PyTorch attention-state merge for a `gfx1151` Triton/HSA fault, including both supported MLA LSE layouts.
|
| 35 |
- Routes causal uniform multi-token MLA verification blocks through Triton decode with correct per-token sequence lengths.
|
| 36 |
- Adds narrowly guarded W4A16 MoE decode/verification fast paths for Ling's exact small-token geometry.
|
|
@@ -99,7 +99,8 @@ The native row predates the release's final SiLU kernel commit, so this table is
|
|
| 99 |
|
| 100 |
- `runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.bundle`: exact Git objects for the CIRU branch, based on the pinned upstream commit.
|
| 101 |
- `runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.patch`: reviewable net source diff.
|
| 102 |
-
- `
|
|
|
|
| 103 |
- `scripts/install-host-deps.sh`: distro-aware Linux/WSL host dependency installer.
|
| 104 |
- `scripts/build-vllm-gfx1151.sh`: reproducible build entry point for the pinned AMD wheel stack.
|
| 105 |
- `scripts/run-256k.sh`: native 256K, native-MTP K1 server profile with five active sequence slots.
|
|
@@ -112,16 +113,32 @@ The named release archive is `Ling-3.0-Flash-CIRU-int4-Strix-native.tar.gz`. It
|
|
| 112 |
|
| 113 |
The exact validated environment is Python 3.12, Torch `2.13.0+rocm7.15.0a20260728`, HIP `7.15.0`, Triton `3.8.0+git4cff872c.rocm7.15.0a20260728`, FlashAttention `2.8.3`, and the CIRU vLLM source at `388d82de2`.
|
| 114 |
|
|
|
|
|
|
|
| 115 |
```bash
|
| 116 |
-
bash
|
| 117 |
-
|
| 118 |
-
bash scripts/download-model.sh "$HOME/models"
|
| 119 |
|
| 120 |
-
|
| 121 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
bash scripts/run-256k.sh
|
| 123 |
```
|
| 124 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
The API advertises model ID `Ling-3.0-Flash-CIRU-int4-Strix-native` on `http://127.0.0.1:18081/v1`.
|
| 126 |
|
| 127 |
The default launcher provides the checkpoint's native 256K context (`262144` tokens), five active sequence slots, and `gpu_memory_utilization=0.72`. Requests beyond five wait in vLLM's scheduler.
|
|
@@ -190,6 +207,7 @@ Other W4A16 models will run only if already supported by this vLLM base; they sh
|
|
| 190 |
|
| 191 |
- Model and official INT4 weights: [InclusionAI/Ling-3.0-flash-int4](https://huggingface.co/inclusionAI/Ling-3.0-flash-int4), MIT license.
|
| 192 |
- Inference engine: [vLLM](https://github.com/vllm-project/vllm), Apache-2.0.
|
|
|
|
| 193 |
- ROCm, Torch, and Triton packages: AMD and their respective upstream projects.
|
| 194 |
- Strix Halo kernel, TTM/GTT, and WSL/ROCDXG guidance: [AMD ROCm documentation](https://rocm.docs.amd.com/en/docs-7.2.0/how-to/system-optimization/strixhalo.html) and [ROCDXG](https://github.com/ROCm/librocdxg).
|
| 195 |
- Runtime integration, profiling, kernels, and validation: CIRU.
|
|
|
|
| 30 |
|
| 31 |
The runtime is based on vLLM commit `d35eb6c44071ea806018841c490f0d2f3219c485` plus a 15-commit CIRU branch ending at `388d82de2abe418e73380ba250270d3abc62ac12`. Its net source delta is six files, 365 insertions, and 26 deletions.
|
| 32 |
|
| 33 |
+
- Includes [`liminfei-amd`'s Wave32 LDS fix](https://github.com/vllm-project/vllm/pull/46012) for the ROCm top-k merge launch, preserving the AMD author's commit credit.
|
| 34 |
- Adds an opt-in safe PyTorch attention-state merge for a `gfx1151` Triton/HSA fault, including both supported MLA LSE layouts.
|
| 35 |
- Routes causal uniform multi-token MLA verification blocks through Triton decode with correct per-token sequence lengths.
|
| 36 |
- Adds narrowly guarded W4A16 MoE decode/verification fast paths for Ling's exact small-token geometry.
|
|
|
|
| 99 |
|
| 100 |
- `runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.bundle`: exact Git objects for the CIRU branch, based on the pinned upstream commit.
|
| 101 |
- `runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.patch`: reviewable net source diff.
|
| 102 |
+
- `install.sh`: guided human installer plus a flag-driven interface for agents and automation.
|
| 103 |
+
- `scripts/download-model.sh`: downloads the unchanged official checkpoint to a parent directory or exact `--model-path`.
|
| 104 |
- `scripts/install-host-deps.sh`: distro-aware Linux/WSL host dependency installer.
|
| 105 |
- `scripts/build-vllm-gfx1151.sh`: reproducible build entry point for the pinned AMD wheel stack.
|
| 106 |
- `scripts/run-256k.sh`: native 256K, native-MTP K1 server profile with five active sequence slots.
|
|
|
|
| 113 |
|
| 114 |
The exact validated environment is Python 3.12, Torch `2.13.0+rocm7.15.0a20260728`, HIP `7.15.0`, Triton `3.8.0+git4cff872c.rocm7.15.0a20260728`, FlashAttention `2.8.3`, and the CIRU vLLM source at `388d82de2`.
|
| 115 |
|
| 116 |
+
For a guided install, run this from the repository root:
|
| 117 |
+
|
| 118 |
```bash
|
| 119 |
+
bash install.sh --interactive
|
| 120 |
+
```
|
|
|
|
| 121 |
|
| 122 |
+
The runtime path defaults to the current directory. The installer asks for the
|
| 123 |
+
exact model directory, makes the 77 GB checkpoint download opt-in, validates
|
| 124 |
+
the completed runtime, and writes a sourceable `ling3-runtime.env` containing
|
| 125 |
+
`VENV` and `MODEL_PATH`. It prints the launch command when finished.
|
| 126 |
+
|
| 127 |
+
Agents and automation can avoid prompts:
|
| 128 |
+
|
| 129 |
+
```bash
|
| 130 |
+
bash install.sh \
|
| 131 |
+
--install-root "$PWD" \
|
| 132 |
+
--model-path "/path/to/Ling-checkpoint"
|
| 133 |
+
|
| 134 |
+
source "$PWD/ling3-runtime.env"
|
| 135 |
bash scripts/run-256k.sh
|
| 136 |
```
|
| 137 |
|
| 138 |
+
Add `--install-host-deps` when the supported distro packages are not already
|
| 139 |
+
installed, or `--download-model` when the exact model path does not yet contain
|
| 140 |
+
the checkpoint. Use `bash install.sh --help` for the complete agent interface.
|
| 141 |
+
|
| 142 |
The API advertises model ID `Ling-3.0-Flash-CIRU-int4-Strix-native` on `http://127.0.0.1:18081/v1`.
|
| 143 |
|
| 144 |
The default launcher provides the checkpoint's native 256K context (`262144` tokens), five active sequence slots, and `gpu_memory_utilization=0.72`. Requests beyond five wait in vLLM's scheduler.
|
|
|
|
| 207 |
|
| 208 |
- Model and official INT4 weights: [InclusionAI/Ling-3.0-flash-int4](https://huggingface.co/inclusionAI/Ling-3.0-flash-int4), MIT license.
|
| 209 |
- Inference engine: [vLLM](https://github.com/vllm-project/vllm), Apache-2.0.
|
| 210 |
+
- Wave32 top-k merge fix: [`liminfei-amd`](https://github.com/liminfei-amd), [vLLM PR #46012](https://github.com/vllm-project/vllm/pull/46012). The release history retains the original AMD authorship.
|
| 211 |
- ROCm, Torch, and Triton packages: AMD and their respective upstream projects.
|
| 212 |
- Strix Halo kernel, TTM/GTT, and WSL/ROCDXG guidance: [AMD ROCm documentation](https://rocm.docs.amd.com/en/docs-7.2.0/how-to/system-optimization/strixhalo.html) and [ROCDXG](https://github.com/ROCm/librocdxg).
|
| 213 |
- Runtime integration, profiling, kernels, and validation: CIRU.
|
SHA256SUMS
CHANGED
|
@@ -1,21 +1,22 @@
|
|
| 1 |
-
|
| 2 |
411ac21b4849cb3bf65f84a55263550dcd658e0623c2b117233ba1fced1c17ff assets/ling30int4.png
|
| 3 |
65cf5b48f9df164c40fbac98e21c6f9c5ced98660d653630cb58f5ab4b28919f CHANGELOG.md
|
| 4 |
16213dfa6abe90226ab92f9407f3bb92f9ed729e30bb8c37ffd92c7dbc999c73 config/experimental-yarn-1m.env
|
| 5 |
c3cdeda6c1bdd47f2b947eac7c4c7182e27249220ddd6eb372c5d12c2c6c405f config/native-256k.env
|
| 6 |
0bb41680a6f0c8366e8f883ca7d76ce1a2ccd3e42541f6d0616df33fe867f55a docs/BENCHMARKS.md
|
| 7 |
-
|
| 8 |
23336099c40774a77a555165da7aee562d5e603017201668178f35f6a8e5cd0d docs/VALIDATED_ENVIRONMENT.md
|
| 9 |
3176eba46cccbd60d33589230190e530eaace8c96466c9621772a1f2c50c07e4 docs/WINDOWS_WSL2.md
|
| 10 |
-
|
| 11 |
-
|
|
|
|
| 12 |
15e48a8df3b65a73c81cf73c820ba12b4ab9474697d6b3f876c26c0b0cd3d455 runtime/COMMITS.txt
|
| 13 |
6b03af3a1685a1969ff96870ea776537f5f67fd3674fc353880bd088bcef9a9c runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.bundle
|
| 14 |
8f307e9f2f16207856ca0b064a765a73bbebc029ad0b0fed61e5975a17e07ab6 runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.patch
|
| 15 |
f4caeda80ed751a40fa976aa7853357b88075e6eb6d2d49e68b0bc8599754c34 runtime/SHA256SUMS.runtime
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
27d3cfba9f4178f83cbd2d7e44db098ab69e515f76e5c98c40ac219b7b0597ca scripts/run-1m-yarn-experimental.sh
|
| 20 |
296ee94a925db5b97e21afb6ffd4ae3c0cc1c2be67af81e21350f43efaa59ead scripts/run-256k.sh
|
| 21 |
-
|
|
|
|
| 1 |
+
7b77dda00cc6bd6d0ce15f7c055f9e997f56e1cf1709c61cca5a57283ce3f008 .gitattributes
|
| 2 |
411ac21b4849cb3bf65f84a55263550dcd658e0623c2b117233ba1fced1c17ff assets/ling30int4.png
|
| 3 |
65cf5b48f9df164c40fbac98e21c6f9c5ced98660d653630cb58f5ab4b28919f CHANGELOG.md
|
| 4 |
16213dfa6abe90226ab92f9407f3bb92f9ed729e30bb8c37ffd92c7dbc999c73 config/experimental-yarn-1m.env
|
| 5 |
c3cdeda6c1bdd47f2b947eac7c4c7182e27249220ddd6eb372c5d12c2c6c405f config/native-256k.env
|
| 6 |
0bb41680a6f0c8366e8f883ca7d76ce1a2ccd3e42541f6d0616df33fe867f55a docs/BENCHMARKS.md
|
| 7 |
+
482e0b0b86622890f4e5db69636ea6427d4d061c6939fc2456a5f7581bda3bd9 docs/UBUNTU_BUILD.md
|
| 8 |
23336099c40774a77a555165da7aee562d5e603017201668178f35f6a8e5cd0d docs/VALIDATED_ENVIRONMENT.md
|
| 9 |
3176eba46cccbd60d33589230190e530eaace8c96466c9621772a1f2c50c07e4 docs/WINDOWS_WSL2.md
|
| 10 |
+
b85d98b8c4f66ff3e69696a47470d11d010b36358d1b6e184ec3c8edb7ab81fb install.sh
|
| 11 |
+
852b2422c20355b533884dae00f330aea8e836efecf0b070d59b3e848540c169 MANIFEST.json
|
| 12 |
+
a4288054963d206dc4978658e12b4dc9e7a77d0f5e4717dfd4cd486ba4c7b993 README.md
|
| 13 |
15e48a8df3b65a73c81cf73c820ba12b4ab9474697d6b3f876c26c0b0cd3d455 runtime/COMMITS.txt
|
| 14 |
6b03af3a1685a1969ff96870ea776537f5f67fd3674fc353880bd088bcef9a9c runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.bundle
|
| 15 |
8f307e9f2f16207856ca0b064a765a73bbebc029ad0b0fed61e5975a17e07ab6 runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.patch
|
| 16 |
f4caeda80ed751a40fa976aa7853357b88075e6eb6d2d49e68b0bc8599754c34 runtime/SHA256SUMS.runtime
|
| 17 |
+
2c066e583d3b24d4c0caa7d04263228dff80e464c61a557736e9c5c5f79b9869 scripts/build-vllm-gfx1151.sh
|
| 18 |
+
ef474884f7d52d1643e0209da4fb9407f42ba4f7d1e19b91cf7d1112849f74cc scripts/download-model.sh
|
| 19 |
+
8142ef953e2c9618df885756eb897303e6443a4d139a09fe9a882c80d054f466 scripts/install-host-deps.sh
|
| 20 |
27d3cfba9f4178f83cbd2d7e44db098ab69e515f76e5c98c40ac219b7b0597ca scripts/run-1m-yarn-experimental.sh
|
| 21 |
296ee94a925db5b97e21afb6ffd4ae3c0cc1c2be67af81e21350f43efaa59ead scripts/run-256k.sh
|
| 22 |
+
718bd922b3bb0a07fb9e2bb0c5ca9a67b799582ac6f614c17af291793e2a69e0 scripts/run-common.sh
|
docs/UBUNTU_BUILD.md
CHANGED
|
@@ -39,6 +39,23 @@ ls -l /dev/kfd /dev/dri/renderD*
|
|
| 39 |
|
| 40 |
If `/dev/kfd` is absent, stop here. The Python wheels cannot replace a missing kernel driver.
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
## 2. Configure unified-memory access
|
| 43 |
|
| 44 |
Strix Halo maps system memory dynamically through GTT/TTM. AMD recommends keeping the fixed BIOS GPU reservation small and increasing the shared TTM limit instead. On CIRU's 128 GB host, the validated mapping limit was 124 GiB. That limit is not a permanent reservation, but the model can still leave very little free RAM during long-context requests.
|
|
@@ -87,8 +104,14 @@ A successful build ends with:
|
|
| 87 |
|
| 88 |
```text
|
| 89 |
CIRU vLLM gfx1151 build: PASS
|
|
|
|
| 90 |
```
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
## 5. Download the unchanged checkpoint and serve
|
| 93 |
|
| 94 |
```bash
|
|
|
|
| 39 |
|
| 40 |
If `/dev/kfd` is absent, stop here. The Python wheels cannot replace a missing kernel driver.
|
| 41 |
|
| 42 |
+
## Guided installer
|
| 43 |
+
|
| 44 |
+
After completing the host-driver and unified-memory setup, the recommended
|
| 45 |
+
human interface is:
|
| 46 |
+
|
| 47 |
+
```bash
|
| 48 |
+
bash install.sh --interactive
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
The runtime install directory defaults to the current directory. The installer
|
| 52 |
+
asks for the exact checkpoint directory, makes host-package installation and
|
| 53 |
+
the approximately 77 GB model download explicit choices, builds the runtime,
|
| 54 |
+
validates `vllm`, and writes `ling3-runtime.env` with the launch environment.
|
| 55 |
+
|
| 56 |
+
For non-interactive agents, see `bash install.sh --help`. The numbered steps
|
| 57 |
+
below remain available for manual diagnosis and custom automation.
|
| 58 |
+
|
| 59 |
## 2. Configure unified-memory access
|
| 60 |
|
| 61 |
Strix Halo maps system memory dynamically through GTT/TTM. AMD recommends keeping the fixed BIOS GPU reservation small and increasing the shared TTM limit instead. On CIRU's 128 GB host, the validated mapping limit was 124 GiB. That limit is not a permanent reservation, but the model can still leave very little free RAM during long-context requests.
|
|
|
|
| 104 |
|
| 105 |
```text
|
| 106 |
CIRU vLLM gfx1151 build: PASS
|
| 107 |
+
Runtime virtual environment ready: /home/USER/ciru-ling-runtime/.venv
|
| 108 |
```
|
| 109 |
|
| 110 |
+
The command argument is the install root. Use `.venv` beneath that exact same
|
| 111 |
+
path as `VENV` when launching. If the build exits before the two lines above,
|
| 112 |
+
the environment is partial even if ROCm tools such as `hipcc` already exist;
|
| 113 |
+
rerun the corrected build command against the same install root.
|
| 114 |
+
|
| 115 |
## 5. Download the unchanged checkpoint and serve
|
| 116 |
|
| 117 |
```bash
|
install.sh
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
set -Eeuo pipefail
|
| 3 |
+
|
| 4 |
+
script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
| 5 |
+
release_name=Ling-3.0-Flash-CIRU-int4-Strix-native
|
| 6 |
+
export PATH="$HOME/.local/bin:$PATH"
|
| 7 |
+
|
| 8 |
+
usage() {
|
| 9 |
+
cat <<EOF
|
| 10 |
+
Install the CIRU Ling 3.0 Flash gfx1151 runtime.
|
| 11 |
+
|
| 12 |
+
Human-guided install:
|
| 13 |
+
bash install.sh --interactive
|
| 14 |
+
|
| 15 |
+
Non-interactive install for agents and automation:
|
| 16 |
+
bash install.sh --install-root PATH --model-path PATH [OPTIONS]
|
| 17 |
+
|
| 18 |
+
Options:
|
| 19 |
+
--interactive Ask for paths and optional actions.
|
| 20 |
+
--install-root PATH Runtime root; defaults to the current directory.
|
| 21 |
+
The vLLM environment is created at PATH/.venv.
|
| 22 |
+
--model-path PATH Exact checkpoint directory; defaults to
|
| 23 |
+
./Ling-3.0-Flash-CIRU-int4-Strix-native.
|
| 24 |
+
--install-host-deps Run scripts/install-host-deps.sh (may use sudo).
|
| 25 |
+
--download-model Download the pinned official checkpoint to MODEL_PATH.
|
| 26 |
+
The checkpoint is approximately 77 GB.
|
| 27 |
+
--skip-build Validate/configure an already-built runtime.
|
| 28 |
+
--dry-run Print the commands and resolved paths without running them.
|
| 29 |
+
--print-env Print launch exports for the resolved paths and exit.
|
| 30 |
+
-h, --help Show this help.
|
| 31 |
+
|
| 32 |
+
The installer writes PATH/ling3-runtime.env after validation. Agents can use:
|
| 33 |
+
source PATH/ling3-runtime.env
|
| 34 |
+
bash scripts/run-256k.sh
|
| 35 |
+
|
| 36 |
+
Environment variables can always be supplied directly:
|
| 37 |
+
VENV=PATH/.venv MODEL_PATH=MODEL_PATH bash scripts/run-256k.sh
|
| 38 |
+
EOF
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
die() {
|
| 42 |
+
printf 'install.sh: %s\n' "$*" >&2
|
| 43 |
+
exit 2
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
absolute_path() {
|
| 47 |
+
local path="$1"
|
| 48 |
+
case "$path" in
|
| 49 |
+
'~') path="$HOME" ;;
|
| 50 |
+
'~/'*) path="$HOME/${path#\~/}" ;;
|
| 51 |
+
esac
|
| 52 |
+
if command -v realpath >/dev/null 2>&1; then
|
| 53 |
+
realpath -m -- "$path"
|
| 54 |
+
elif [[ "$path" = /* ]]; then
|
| 55 |
+
printf '%s\n' "$path"
|
| 56 |
+
else
|
| 57 |
+
printf '%s/%s\n' "$PWD" "${path#./}"
|
| 58 |
+
fi
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
prompt_value() {
|
| 62 |
+
local variable_name="$1"
|
| 63 |
+
local label="$2"
|
| 64 |
+
local default_value="$3"
|
| 65 |
+
local reply
|
| 66 |
+
read -r -p "$label [$default_value]: " reply
|
| 67 |
+
printf -v "$variable_name" '%s' "${reply:-$default_value}"
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
ask_yes_no() {
|
| 71 |
+
local label="$1"
|
| 72 |
+
local default_answer="$2"
|
| 73 |
+
local suffix reply
|
| 74 |
+
if [ "$default_answer" = yes ]; then
|
| 75 |
+
suffix='[Y/n]'
|
| 76 |
+
else
|
| 77 |
+
suffix='[y/N]'
|
| 78 |
+
fi
|
| 79 |
+
while true; do
|
| 80 |
+
read -r -p "$label $suffix: " reply
|
| 81 |
+
reply="${reply:-$default_answer}"
|
| 82 |
+
case "${reply,,}" in
|
| 83 |
+
y|yes) return 0 ;;
|
| 84 |
+
n|no) return 1 ;;
|
| 85 |
+
*) printf 'Please answer yes or no.\n' >&2 ;;
|
| 86 |
+
esac
|
| 87 |
+
done
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
interactive=0
|
| 91 |
+
install_root=
|
| 92 |
+
model_path=
|
| 93 |
+
install_host_deps=0
|
| 94 |
+
download_model=0
|
| 95 |
+
build_runtime=1
|
| 96 |
+
dry_run=0
|
| 97 |
+
print_env=0
|
| 98 |
+
|
| 99 |
+
if [ "$#" -eq 0 ]; then
|
| 100 |
+
usage
|
| 101 |
+
exit 0
|
| 102 |
+
fi
|
| 103 |
+
|
| 104 |
+
while [ "$#" -gt 0 ]; do
|
| 105 |
+
case "$1" in
|
| 106 |
+
--interactive)
|
| 107 |
+
interactive=1
|
| 108 |
+
shift
|
| 109 |
+
;;
|
| 110 |
+
--install-root)
|
| 111 |
+
[ "$#" -ge 2 ] || die '--install-root requires a path'
|
| 112 |
+
install_root="$2"
|
| 113 |
+
shift 2
|
| 114 |
+
;;
|
| 115 |
+
--model-path)
|
| 116 |
+
[ "$#" -ge 2 ] || die '--model-path requires a path'
|
| 117 |
+
model_path="$2"
|
| 118 |
+
shift 2
|
| 119 |
+
;;
|
| 120 |
+
--install-host-deps)
|
| 121 |
+
install_host_deps=1
|
| 122 |
+
shift
|
| 123 |
+
;;
|
| 124 |
+
--download-model)
|
| 125 |
+
download_model=1
|
| 126 |
+
shift
|
| 127 |
+
;;
|
| 128 |
+
--skip-build)
|
| 129 |
+
build_runtime=0
|
| 130 |
+
shift
|
| 131 |
+
;;
|
| 132 |
+
--dry-run)
|
| 133 |
+
dry_run=1
|
| 134 |
+
shift
|
| 135 |
+
;;
|
| 136 |
+
--print-env)
|
| 137 |
+
print_env=1
|
| 138 |
+
shift
|
| 139 |
+
;;
|
| 140 |
+
-h|--help)
|
| 141 |
+
usage
|
| 142 |
+
exit 0
|
| 143 |
+
;;
|
| 144 |
+
*)
|
| 145 |
+
die "unknown option: $1"
|
| 146 |
+
;;
|
| 147 |
+
esac
|
| 148 |
+
done
|
| 149 |
+
|
| 150 |
+
if [ "$interactive" = 1 ]; then
|
| 151 |
+
[ -t 0 ] || die '--interactive requires a terminal; use explicit flags for automation'
|
| 152 |
+
prompt_value install_root 'Runtime install directory' "${install_root:-$PWD}"
|
| 153 |
+
prompt_value model_path 'Ling checkpoint directory' \
|
| 154 |
+
"${model_path:-$PWD/$release_name}"
|
| 155 |
+
if ask_yes_no 'Run the distro host-dependency installer (may use sudo)?' no; then
|
| 156 |
+
install_host_deps=1
|
| 157 |
+
fi
|
| 158 |
+
if [ ! -f "$model_path/config.json" ]; then
|
| 159 |
+
if ask_yes_no 'Checkpoint not found there. Download the pinned model (~77 GB)?' no; then
|
| 160 |
+
download_model=1
|
| 161 |
+
fi
|
| 162 |
+
fi
|
| 163 |
+
fi
|
| 164 |
+
|
| 165 |
+
install_root="$(absolute_path "${install_root:-$PWD}")"
|
| 166 |
+
model_path="$(absolute_path "${model_path:-$PWD/$release_name}")"
|
| 167 |
+
venv="$install_root/.venv"
|
| 168 |
+
env_file="$install_root/ling3-runtime.env"
|
| 169 |
+
|
| 170 |
+
if [ "$print_env" = 1 ]; then
|
| 171 |
+
printf 'export VENV=%q\n' "$venv"
|
| 172 |
+
printf 'export MODEL_PATH=%q\n' "$model_path"
|
| 173 |
+
exit 0
|
| 174 |
+
fi
|
| 175 |
+
|
| 176 |
+
printf 'CIRU Ling runtime installation\n'
|
| 177 |
+
printf ' package: %s\n install root: %s\n venv: %s\n model: %s\n' \
|
| 178 |
+
"$script_dir" "$install_root" "$venv" "$model_path"
|
| 179 |
+
printf ' host dependencies: %s\n download model: %s\n build runtime: %s\n' \
|
| 180 |
+
"$install_host_deps" "$download_model" "$build_runtime"
|
| 181 |
+
|
| 182 |
+
if [ "$interactive" = 1 ] && ! ask_yes_no 'Continue with these settings?' yes; then
|
| 183 |
+
printf 'Installation cancelled.\n'
|
| 184 |
+
exit 0
|
| 185 |
+
fi
|
| 186 |
+
|
| 187 |
+
if [ "$dry_run" = 1 ]; then
|
| 188 |
+
if [ "$install_host_deps" = 1 ]; then
|
| 189 |
+
printf 'DRY RUN: bash %q\n' "$script_dir/scripts/install-host-deps.sh"
|
| 190 |
+
fi
|
| 191 |
+
if [ "$download_model" = 1 ]; then
|
| 192 |
+
printf 'DRY RUN: bash %q --model-path %q\n' \
|
| 193 |
+
"$script_dir/scripts/download-model.sh" "$model_path"
|
| 194 |
+
fi
|
| 195 |
+
if [ "$build_runtime" = 1 ]; then
|
| 196 |
+
printf 'DRY RUN: bash %q %q\n' \
|
| 197 |
+
"$script_dir/scripts/build-vllm-gfx1151.sh" "$install_root"
|
| 198 |
+
fi
|
| 199 |
+
printf 'DRY RUN: write %q\n' "$env_file"
|
| 200 |
+
exit 0
|
| 201 |
+
fi
|
| 202 |
+
|
| 203 |
+
if [ "$download_model" != 1 ] && [ ! -f "$model_path/config.json" ]; then
|
| 204 |
+
die "model config not found at $model_path/config.json; choose an existing checkpoint or pass --download-model"
|
| 205 |
+
fi
|
| 206 |
+
|
| 207 |
+
if [ "$install_host_deps" = 1 ]; then
|
| 208 |
+
bash "$script_dir/scripts/install-host-deps.sh"
|
| 209 |
+
fi
|
| 210 |
+
|
| 211 |
+
if [ "$download_model" = 1 ]; then
|
| 212 |
+
bash "$script_dir/scripts/download-model.sh" --model-path "$model_path"
|
| 213 |
+
fi
|
| 214 |
+
|
| 215 |
+
if [ "$build_runtime" = 1 ]; then
|
| 216 |
+
bash "$script_dir/scripts/build-vllm-gfx1151.sh" "$install_root"
|
| 217 |
+
fi
|
| 218 |
+
|
| 219 |
+
[ -x "$venv/bin/python" ] || die "runtime Python is missing after installation: $venv/bin/python"
|
| 220 |
+
[ -x "$venv/bin/vllm" ] || die "vLLM launcher is missing after installation: $venv/bin/vllm"
|
| 221 |
+
[ -f "$model_path/config.json" ] || die "model config is missing after installation: $model_path/config.json"
|
| 222 |
+
|
| 223 |
+
mkdir -p "$install_root"
|
| 224 |
+
{
|
| 225 |
+
printf '# Generated by %s/install.sh\n' "$script_dir"
|
| 226 |
+
printf 'export VENV=%q\n' "$venv"
|
| 227 |
+
printf 'export MODEL_PATH=%q\n' "$model_path"
|
| 228 |
+
} >"$env_file"
|
| 229 |
+
|
| 230 |
+
printf '\nInstallation complete.\n'
|
| 231 |
+
printf 'Environment file: %s\n\n' "$env_file"
|
| 232 |
+
printf 'Launch the native 256K profile:\n'
|
| 233 |
+
printf ' source %q\n bash %q\n\n' "$env_file" "$script_dir/scripts/run-256k.sh"
|
| 234 |
+
printf 'For agents or automation:\n'
|
| 235 |
+
printf ' export VENV=%q\n export MODEL_PATH=%q\n' "$venv" "$model_path"
|
| 236 |
+
printf ' bash %q\n' "$script_dir/scripts/run-256k.sh"
|
scripts/build-vllm-gfx1151.sh
CHANGED
|
@@ -12,6 +12,10 @@ head=388d82de2abe418e73380ba250270d3abc62ac12
|
|
| 12 |
bundle="$package_root/runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.bundle"
|
| 13 |
flash_source="$install_root/flash-attention"
|
| 14 |
flash_commit=0e60e39473e8df549a20fb5353760f7a65b30e2d
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
command -v git >/dev/null
|
| 17 |
command -v uv >/dev/null || {
|
|
@@ -56,7 +60,6 @@ UV_CACHE_DIR="${UV_CACHE_DIR:-$install_root/uv-cache}" uv pip install \
|
|
| 56 |
'rocm[libraries,devel,device-gfx1151]==7.15.0a20260728'
|
| 57 |
|
| 58 |
"$venv/bin/rocm-sdk" init
|
| 59 |
-
"$venv/bin/rocm-sdk" test
|
| 60 |
|
| 61 |
if [ ! -d "$source_dir/.git" ]; then
|
| 62 |
git clone https://github.com/vllm-project/vllm.git "$source_dir"
|
|
@@ -111,6 +114,10 @@ export MAX_JOBS="${MAX_JOBS:-16}"
|
|
| 111 |
export CMAKE_BUILD_PARALLEL_LEVEL="$MAX_JOBS"
|
| 112 |
export CMAKE_ARGS="-DROCM_PATH=$sdk -DCMAKE_HIP_ARCHITECTURES=gfx1151 ${CMAKE_ARGS:-}"
|
| 113 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 114 |
UV_CACHE_DIR="${UV_CACHE_DIR:-$install_root/uv-cache}" uv pip install \
|
| 115 |
--python "$venv/bin/python" \
|
| 116 |
'cmake>=3.26.1,<4' ninja 'packaging>=24.2' \
|
|
@@ -124,6 +131,11 @@ UV_CACHE_DIR="${UV_CACHE_DIR:-$install_root/uv-cache}" uv pip install \
|
|
| 124 |
cd "$source_dir"
|
| 125 |
UV_CACHE_DIR="${UV_CACHE_DIR:-$install_root/uv-cache}" uv pip install \
|
| 126 |
--python "$venv/bin/python" --no-build-isolation --no-deps -e .
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 127 |
|
| 128 |
if [ ! -d "$flash_source/.git" ]; then
|
| 129 |
git clone https://github.com/Dao-AILab/flash-attention.git "$flash_source"
|
|
@@ -153,3 +165,4 @@ for name in (
|
|
| 153 |
importlib.import_module(name)
|
| 154 |
print("CIRU vLLM gfx1151 build: PASS")
|
| 155 |
PY
|
|
|
|
|
|
| 12 |
bundle="$package_root/runtime/Ling-3.0-Flash-CIRU-int4-Strix-native-vllm.bundle"
|
| 13 |
flash_source="$install_root/flash-attention"
|
| 14 |
flash_commit=0e60e39473e8df549a20fb5353760f7a65b30e2d
|
| 15 |
+
export PATH="$HOME/.local/bin:$PATH"
|
| 16 |
+
|
| 17 |
+
printf 'CIRU build install root: %s\n' "$install_root"
|
| 18 |
+
printf 'CIRU runtime virtual environment: %s\n' "$venv"
|
| 19 |
|
| 20 |
command -v git >/dev/null
|
| 21 |
command -v uv >/dev/null || {
|
|
|
|
| 60 |
'rocm[libraries,devel,device-gfx1151]==7.15.0a20260728'
|
| 61 |
|
| 62 |
"$venv/bin/rocm-sdk" init
|
|
|
|
| 63 |
|
| 64 |
if [ ! -d "$source_dir/.git" ]; then
|
| 65 |
git clone https://github.com/vllm-project/vllm.git "$source_dir"
|
|
|
|
| 114 |
export CMAKE_BUILD_PARALLEL_LEVEL="$MAX_JOBS"
|
| 115 |
export CMAKE_ARGS="-DROCM_PATH=$sdk -DCMAKE_HIP_ARCHITECTURES=gfx1151 ${CMAKE_ARGS:-}"
|
| 116 |
|
| 117 |
+
# Run the SDK checks against this virtual environment's development root, not
|
| 118 |
+
# an inherited system ROCm installation.
|
| 119 |
+
"$venv/bin/rocm-sdk" test
|
| 120 |
+
|
| 121 |
UV_CACHE_DIR="${UV_CACHE_DIR:-$install_root/uv-cache}" uv pip install \
|
| 122 |
--python "$venv/bin/python" \
|
| 123 |
'cmake>=3.26.1,<4' ninja 'packaging>=24.2' \
|
|
|
|
| 131 |
cd "$source_dir"
|
| 132 |
UV_CACHE_DIR="${UV_CACHE_DIR:-$install_root/uv-cache}" uv pip install \
|
| 133 |
--python "$venv/bin/python" --no-build-isolation --no-deps -e .
|
| 134 |
+
if [ ! -x "$venv/bin/vllm" ]; then
|
| 135 |
+
printf 'vLLM installation did not create the expected launcher: %s\n' \
|
| 136 |
+
"$venv/bin/vllm" >&2
|
| 137 |
+
exit 2
|
| 138 |
+
fi
|
| 139 |
|
| 140 |
if [ ! -d "$flash_source/.git" ]; then
|
| 141 |
git clone https://github.com/Dao-AILab/flash-attention.git "$flash_source"
|
|
|
|
| 165 |
importlib.import_module(name)
|
| 166 |
print("CIRU vLLM gfx1151 build: PASS")
|
| 167 |
PY
|
| 168 |
+
printf 'Runtime virtual environment ready: %s\n' "$venv"
|
scripts/download-model.sh
CHANGED
|
@@ -1,10 +1,40 @@
|
|
| 1 |
#!/usr/bin/env bash
|
| 2 |
set -Eeuo pipefail
|
| 3 |
|
| 4 |
-
destination_root="${1:-$PWD/models}"
|
| 5 |
release_name=Ling-3.0-Flash-CIRU-int4-Strix-native
|
| 6 |
revision=ca3ea63b0255d212c4fe6020db9e0a51ce136006
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
if command -v hf >/dev/null 2>&1; then
|
| 10 |
hf_cmd=(hf)
|
|
@@ -15,7 +45,7 @@ else
|
|
| 15 |
exit 2
|
| 16 |
fi
|
| 17 |
|
| 18 |
-
mkdir -p "$
|
| 19 |
"${hf_cmd[@]}" download inclusionAI/Ling-3.0-flash-int4 \
|
| 20 |
--revision "$revision" \
|
| 21 |
--local-dir "$destination"
|
|
|
|
| 1 |
#!/usr/bin/env bash
|
| 2 |
set -Eeuo pipefail
|
| 3 |
|
|
|
|
| 4 |
release_name=Ling-3.0-Flash-CIRU-int4-Strix-native
|
| 5 |
revision=ca3ea63b0255d212c4fe6020db9e0a51ce136006
|
| 6 |
+
destination_root="$PWD/models"
|
| 7 |
+
destination=
|
| 8 |
+
export PATH="$HOME/.local/bin:$PATH"
|
| 9 |
+
|
| 10 |
+
usage() {
|
| 11 |
+
cat <<EOF
|
| 12 |
+
Download the pinned official Ling checkpoint.
|
| 13 |
+
|
| 14 |
+
Usage:
|
| 15 |
+
bash scripts/download-model.sh [DESTINATION_ROOT]
|
| 16 |
+
bash scripts/download-model.sh --model-path EXACT_DIRECTORY
|
| 17 |
+
|
| 18 |
+
The legacy positional form downloads to DESTINATION_ROOT/$release_name.
|
| 19 |
+
EOF
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
if [ "${1:-}" = --model-path ]; then
|
| 23 |
+
[ "$#" -eq 2 ] || {
|
| 24 |
+
echo '--model-path requires exactly one directory.' >&2
|
| 25 |
+
exit 2
|
| 26 |
+
}
|
| 27 |
+
destination="$2"
|
| 28 |
+
elif [ "${1:-}" = -h ] || [ "${1:-}" = --help ]; then
|
| 29 |
+
usage
|
| 30 |
+
exit 0
|
| 31 |
+
elif [ "$#" -le 1 ]; then
|
| 32 |
+
destination_root="${1:-$destination_root}"
|
| 33 |
+
destination="$destination_root/$release_name"
|
| 34 |
+
else
|
| 35 |
+
usage >&2
|
| 36 |
+
exit 2
|
| 37 |
+
fi
|
| 38 |
|
| 39 |
if command -v hf >/dev/null 2>&1; then
|
| 40 |
hf_cmd=(hf)
|
|
|
|
| 45 |
exit 2
|
| 46 |
fi
|
| 47 |
|
| 48 |
+
mkdir -p "$destination"
|
| 49 |
"${hf_cmd[@]}" download inclusionAI/Ling-3.0-flash-int4 \
|
| 50 |
--revision "$revision" \
|
| 51 |
--local-dir "$destination"
|
scripts/install-host-deps.sh
CHANGED
|
@@ -44,4 +44,3 @@ command -v uv >/dev/null || {
|
|
| 44 |
}
|
| 45 |
|
| 46 |
printf 'Host dependencies ready on %s.\n' "${PRETTY_NAME:-$distro}"
|
| 47 |
-
printf 'Next: bash scripts/build-vllm-gfx1151.sh "$HOME/ciru-ling-runtime"\n'
|
|
|
|
| 44 |
}
|
| 45 |
|
| 46 |
printf 'Host dependencies ready on %s.\n' "${PRETTY_NAME:-$distro}"
|
|
|
scripts/run-common.sh
CHANGED
|
@@ -8,8 +8,23 @@ host="${HOST:-127.0.0.1}"
|
|
| 8 |
port="${PORT:-18081}"
|
| 9 |
cache_root="${CACHE_ROOT:-$HOME/.cache/ciru-ling3-vllm}"
|
| 10 |
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
site="$($venv/bin/python - <<'PY'
|
| 14 |
import site
|
| 15 |
print(site.getsitepackages()[0])
|
|
@@ -17,7 +32,11 @@ PY
|
|
| 17 |
)"
|
| 18 |
sdk="$site/_rocm_sdk_devel"
|
| 19 |
core="$site/_rocm_sdk_core"
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
mkdir -p "$cache_root/vllm" "$cache_root/triton" "$cache_root/huggingface-modules"
|
| 22 |
|
| 23 |
export PATH="$sdk/bin:$sdk/llvm/bin:$venv/bin:$PATH"
|
|
@@ -74,4 +93,7 @@ if [ "${DRY_RUN:-0}" = 1 ]; then
|
|
| 74 |
exit 0
|
| 75 |
fi
|
| 76 |
|
|
|
|
|
|
|
|
|
|
| 77 |
exec "$venv/bin/vllm" "${args[@]}"
|
|
|
|
| 8 |
port="${PORT:-18081}"
|
| 9 |
cache_root="${CACHE_ROOT:-$HOME/.cache/ciru-ling3-vllm}"
|
| 10 |
|
| 11 |
+
if [ ! -x "$venv/bin/python" ]; then
|
| 12 |
+
printf 'Runtime virtual environment is missing Python: %s\n' \
|
| 13 |
+
"$venv/bin/python" >&2
|
| 14 |
+
printf 'Set VENV to the install root .venv created by scripts/build-vllm-gfx1151.sh.\n' >&2
|
| 15 |
+
exit 2
|
| 16 |
+
fi
|
| 17 |
+
if [ ! -x "$venv/bin/vllm" ]; then
|
| 18 |
+
printf 'Runtime virtual environment is missing the vLLM launcher: %s\n' \
|
| 19 |
+
"$venv/bin/vllm" >&2
|
| 20 |
+
printf 'Re-run scripts/build-vllm-gfx1151.sh for this VENV.\n' >&2
|
| 21 |
+
exit 2
|
| 22 |
+
fi
|
| 23 |
+
if [ ! -d "$model_path" ]; then
|
| 24 |
+
printf 'Model checkpoint directory does not exist: %s\n' "$model_path" >&2
|
| 25 |
+
printf 'Set MODEL_PATH to the directory containing the downloaded Ling checkpoint.\n' >&2
|
| 26 |
+
exit 2
|
| 27 |
+
fi
|
| 28 |
site="$($venv/bin/python - <<'PY'
|
| 29 |
import site
|
| 30 |
print(site.getsitepackages()[0])
|
|
|
|
| 32 |
)"
|
| 33 |
sdk="$site/_rocm_sdk_devel"
|
| 34 |
core="$site/_rocm_sdk_core"
|
| 35 |
+
if [ ! -x "$sdk/bin/hipcc" ]; then
|
| 36 |
+
printf 'Private ROCm SDK is missing hipcc: %s\n' "$sdk/bin/hipcc" >&2
|
| 37 |
+
printf 'Re-run scripts/build-vllm-gfx1151.sh to install and initialize the SDK.\n' >&2
|
| 38 |
+
exit 2
|
| 39 |
+
fi
|
| 40 |
mkdir -p "$cache_root/vllm" "$cache_root/triton" "$cache_root/huggingface-modules"
|
| 41 |
|
| 42 |
export PATH="$sdk/bin:$sdk/llvm/bin:$venv/bin:$PATH"
|
|
|
|
| 93 |
exit 0
|
| 94 |
fi
|
| 95 |
|
| 96 |
+
printf 'Launching %s\n' "$release_name"
|
| 97 |
+
printf ' vLLM: %s\n model: %s\n listen: %s:%s\n' \
|
| 98 |
+
"$venv/bin/vllm" "$model_path" "$host" "$port"
|
| 99 |
exec "$venv/bin/vllm" "${args[@]}"
|