fix: select writable cache root at startup
Browse files- training/space_runner.py +46 -8
training/space_runner.py
CHANGED
|
@@ -6,20 +6,46 @@ Unsloth is pre-installed in the Docker image.
|
|
| 6 |
import logging
|
| 7 |
import os
|
| 8 |
import sys
|
|
|
|
| 9 |
import threading
|
| 10 |
import time
|
| 11 |
import traceback
|
| 12 |
from pathlib import Path
|
| 13 |
|
| 14 |
-
import gradio as gr
|
| 15 |
-
from dotenv import load_dotenv
|
| 16 |
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
runtime_root = Path(os.getenv("SPACE_RUNTIME_DIR", "/tmp/context-corruption-training"))
|
| 23 |
cache_root = runtime_root / "cache"
|
| 24 |
env_dirs = {
|
| 25 |
"HOME": runtime_root,
|
|
@@ -32,12 +58,24 @@ def _configure_writable_runtime_dirs():
|
|
| 32 |
"MPLCONFIGDIR": cache_root / "matplotlib",
|
| 33 |
"OUTPUT_DIR": runtime_root / "checkpoints" / "grpo-qwen-1.5b",
|
| 34 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
for key, path in env_dirs.items():
|
| 36 |
os.environ[key] = str(path)
|
| 37 |
-
path.mkdir(parents=True, exist_ok=True)
|
| 38 |
|
| 39 |
|
| 40 |
-
_configure_writable_runtime_dirs()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
|
| 42 |
_log_lines: list[str] = []
|
| 43 |
_training_status = "idle" # idle | installing | running | complete | failed
|
|
|
|
| 6 |
import logging
|
| 7 |
import os
|
| 8 |
import sys
|
| 9 |
+
import tempfile
|
| 10 |
import threading
|
| 11 |
import time
|
| 12 |
import traceback
|
| 13 |
from pathlib import Path
|
| 14 |
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
def _configure_writable_runtime_dirs():
|
| 17 |
+
"""HF Spaces can run with HOME=/ and /app read-only; use a verified cache root."""
|
| 18 |
+
candidates = []
|
| 19 |
+
if os.getenv("SPACE_RUNTIME_DIR"):
|
| 20 |
+
candidates.append(Path(os.environ["SPACE_RUNTIME_DIR"]))
|
| 21 |
+
|
| 22 |
+
uid = getattr(os, "getuid", lambda: "user")()
|
| 23 |
+
candidates.extend(
|
| 24 |
+
[
|
| 25 |
+
Path(tempfile.gettempdir()) / f"context-corruption-training-{uid}",
|
| 26 |
+
Path("/dev/shm") / f"context-corruption-training-{uid}",
|
| 27 |
+
]
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
last_error = None
|
| 31 |
+
for runtime_root in candidates:
|
| 32 |
+
try:
|
| 33 |
+
_apply_runtime_dirs(runtime_root)
|
| 34 |
+
return runtime_root
|
| 35 |
+
except OSError as exc:
|
| 36 |
+
last_error = exc
|
| 37 |
|
| 38 |
+
runtime_root = Path(tempfile.mkdtemp(prefix="context-corruption-training-"))
|
| 39 |
+
try:
|
| 40 |
+
_apply_runtime_dirs(runtime_root)
|
| 41 |
+
except OSError as exc:
|
| 42 |
+
raise RuntimeError(
|
| 43 |
+
f"Could not create writable runtime directories; last error: {last_error}"
|
| 44 |
+
) from exc
|
| 45 |
+
return runtime_root
|
| 46 |
|
| 47 |
+
|
| 48 |
+
def _apply_runtime_dirs(runtime_root: Path):
|
|
|
|
| 49 |
cache_root = runtime_root / "cache"
|
| 50 |
env_dirs = {
|
| 51 |
"HOME": runtime_root,
|
|
|
|
| 58 |
"MPLCONFIGDIR": cache_root / "matplotlib",
|
| 59 |
"OUTPUT_DIR": runtime_root / "checkpoints" / "grpo-qwen-1.5b",
|
| 60 |
}
|
| 61 |
+
|
| 62 |
+
for path in env_dirs.values():
|
| 63 |
+
path.mkdir(parents=True, exist_ok=True)
|
| 64 |
+
|
| 65 |
+
probe = runtime_root / ".write-test"
|
| 66 |
+
probe.write_text("ok")
|
| 67 |
+
probe.unlink(missing_ok=True)
|
| 68 |
+
|
| 69 |
for key, path in env_dirs.items():
|
| 70 |
os.environ[key] = str(path)
|
|
|
|
| 71 |
|
| 72 |
|
| 73 |
+
_runtime_root = _configure_writable_runtime_dirs()
|
| 74 |
+
|
| 75 |
+
import gradio as gr
|
| 76 |
+
from dotenv import load_dotenv
|
| 77 |
+
|
| 78 |
+
load_dotenv()
|
| 79 |
|
| 80 |
_log_lines: list[str] = []
|
| 81 |
_training_status = "idle" # idle | installing | running | complete | failed
|