fix: read pip output in chunks splitting on and
Browse files- training/space_runner.py +30 -17
training/space_runner.py
CHANGED
|
@@ -55,31 +55,44 @@ def _run_training():
|
|
| 55 |
_pip_base = [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--target", _PKGS]
|
| 56 |
|
| 57 |
install_cmds = [
|
| 58 |
-
_pip_base + ["-
|
| 59 |
-
_pip_base + ["-
|
| 60 |
-
|
| 61 |
-
_pip_base + ["--progress-bar", "on", "git+https://github.com/unslothai/unsloth.git"],
|
| 62 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
installed = False
|
| 64 |
for cmd in install_cmds:
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
text=True, bufsize=1,
|
| 68 |
-
)
|
| 69 |
-
lines = []
|
| 70 |
-
for line in proc.stdout:
|
| 71 |
-
line = line.rstrip()
|
| 72 |
-
if line:
|
| 73 |
-
_append_log(line)
|
| 74 |
-
lines.append(line)
|
| 75 |
-
proc.wait()
|
| 76 |
-
if proc.returncode == 0:
|
| 77 |
if _PKGS not in sys.path:
|
| 78 |
sys.path.insert(0, _PKGS)
|
| 79 |
_append_log("✅ unsloth installed successfully.")
|
| 80 |
installed = True
|
| 81 |
break
|
| 82 |
-
_append_log(f"⚠️ Install attempt failed (exit {
|
| 83 |
if not installed:
|
| 84 |
_training_status = "failed"
|
| 85 |
_append_log("❌ All unsloth install attempts failed. Check logs above.")
|
|
|
|
| 55 |
_pip_base = [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--target", _PKGS]
|
| 56 |
|
| 57 |
install_cmds = [
|
| 58 |
+
_pip_base + ["-v", "unsloth"],
|
| 59 |
+
_pip_base + ["-v", f"unsloth[cu121-ampere-{torch_tag}] @ git+https://github.com/unslothai/unsloth.git"],
|
| 60 |
+
_pip_base + ["-v", "git+https://github.com/unslothai/unsloth.git"],
|
|
|
|
| 61 |
]
|
| 62 |
+
|
| 63 |
+
def _stream_cmd(cmd):
|
| 64 |
+
"""Run cmd and stream output; handles \r and \n progress lines."""
|
| 65 |
+
env = {**os.environ, "PYTHONUNBUFFERED": "1", "PIP_NO_COLOR": "1"}
|
| 66 |
+
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
| 67 |
+
bufsize=0, env=env)
|
| 68 |
+
buf = b""
|
| 69 |
+
while True:
|
| 70 |
+
chunk = proc.stdout.read(512)
|
| 71 |
+
if not chunk:
|
| 72 |
+
break
|
| 73 |
+
buf += chunk
|
| 74 |
+
# split on \n and \r so progress lines aren't held
|
| 75 |
+
parts = buf.replace(b"\r", b"\n").split(b"\n")
|
| 76 |
+
buf = parts[-1]
|
| 77 |
+
for p in parts[:-1]:
|
| 78 |
+
line = p.decode("utf-8", errors="replace").strip()
|
| 79 |
+
if line:
|
| 80 |
+
_append_log(line)
|
| 81 |
+
if buf.strip():
|
| 82 |
+
_append_log(buf.decode("utf-8", errors="replace").strip())
|
| 83 |
+
proc.wait()
|
| 84 |
+
return proc.returncode
|
| 85 |
+
|
| 86 |
installed = False
|
| 87 |
for cmd in install_cmds:
|
| 88 |
+
rc = _stream_cmd(cmd)
|
| 89 |
+
if rc == 0:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
if _PKGS not in sys.path:
|
| 91 |
sys.path.insert(0, _PKGS)
|
| 92 |
_append_log("✅ unsloth installed successfully.")
|
| 93 |
installed = True
|
| 94 |
break
|
| 95 |
+
_append_log(f"⚠️ Install attempt failed (exit {rc}), trying next...")
|
| 96 |
if not installed:
|
| 97 |
_training_status = "failed"
|
| 98 |
_append_log("❌ All unsloth install attempts failed. Check logs above.")
|