Siddh12334 commited on
Commit
bb169f7
·
verified ·
1 Parent(s): 1f75c91

fix: select writable cache root at startup

Browse files
Files changed (1) hide show
  1. training/space_runner.py +46 -8
training/space_runner.py CHANGED
@@ -6,20 +6,46 @@ Unsloth is pre-installed in the Docker image.
6
  import logging
7
  import os
8
  import sys
 
9
  import threading
10
  import time
11
  import traceback
12
  from pathlib import Path
13
 
14
- import gradio as gr
15
- from dotenv import load_dotenv
16
 
17
- load_dotenv()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
 
 
 
 
 
 
 
 
19
 
20
- def _configure_writable_runtime_dirs():
21
- """HF Spaces can run with HOME=/ and /app read-only; use /tmp for caches."""
22
- runtime_root = Path(os.getenv("SPACE_RUNTIME_DIR", "/tmp/context-corruption-training"))
23
  cache_root = runtime_root / "cache"
24
  env_dirs = {
25
  "HOME": runtime_root,
@@ -32,12 +58,24 @@ def _configure_writable_runtime_dirs():
32
  "MPLCONFIGDIR": cache_root / "matplotlib",
33
  "OUTPUT_DIR": runtime_root / "checkpoints" / "grpo-qwen-1.5b",
34
  }
 
 
 
 
 
 
 
 
35
  for key, path in env_dirs.items():
36
  os.environ[key] = str(path)
37
- path.mkdir(parents=True, exist_ok=True)
38
 
39
 
40
- _configure_writable_runtime_dirs()
 
 
 
 
 
41
 
42
  _log_lines: list[str] = []
43
  _training_status = "idle" # idle | installing | running | complete | failed
 
6
  import logging
7
  import os
8
  import sys
9
+ import tempfile
10
  import threading
11
  import time
12
  import traceback
13
  from pathlib import Path
14
 
 
 
15
 
16
+ def _configure_writable_runtime_dirs():
17
+ """HF Spaces can run with HOME=/ and /app read-only; use a verified cache root."""
18
+ candidates = []
19
+ if os.getenv("SPACE_RUNTIME_DIR"):
20
+ candidates.append(Path(os.environ["SPACE_RUNTIME_DIR"]))
21
+
22
+ uid = getattr(os, "getuid", lambda: "user")()
23
+ candidates.extend(
24
+ [
25
+ Path(tempfile.gettempdir()) / f"context-corruption-training-{uid}",
26
+ Path("/dev/shm") / f"context-corruption-training-{uid}",
27
+ ]
28
+ )
29
+
30
+ last_error = None
31
+ for runtime_root in candidates:
32
+ try:
33
+ _apply_runtime_dirs(runtime_root)
34
+ return runtime_root
35
+ except OSError as exc:
36
+ last_error = exc
37
 
38
+ runtime_root = Path(tempfile.mkdtemp(prefix="context-corruption-training-"))
39
+ try:
40
+ _apply_runtime_dirs(runtime_root)
41
+ except OSError as exc:
42
+ raise RuntimeError(
43
+ f"Could not create writable runtime directories; last error: {last_error}"
44
+ ) from exc
45
+ return runtime_root
46
 
47
+
48
+ def _apply_runtime_dirs(runtime_root: Path):
 
49
  cache_root = runtime_root / "cache"
50
  env_dirs = {
51
  "HOME": runtime_root,
 
58
  "MPLCONFIGDIR": cache_root / "matplotlib",
59
  "OUTPUT_DIR": runtime_root / "checkpoints" / "grpo-qwen-1.5b",
60
  }
61
+
62
+ for path in env_dirs.values():
63
+ path.mkdir(parents=True, exist_ok=True)
64
+
65
+ probe = runtime_root / ".write-test"
66
+ probe.write_text("ok")
67
+ probe.unlink(missing_ok=True)
68
+
69
  for key, path in env_dirs.items():
70
  os.environ[key] = str(path)
 
71
 
72
 
73
+ _runtime_root = _configure_writable_runtime_dirs()
74
+
75
+ import gradio as gr
76
+ from dotenv import load_dotenv
77
+
78
+ load_dotenv()
79
 
80
  _log_lines: list[str] = []
81
  _training_status = "idle" # idle | installing | running | complete | failed