Run SoVITS CPU training in a single process

app.py

@@ -68,7 +68,7 @@ MODEL_PATTERNS = [
     "*.safetensors",
     "*.model",
 ]
-UPSTREAM_PATCH_VERSION = "2026-05-25-worker0-v4"
+UPSTREAM_PATCH_VERSION = "2026-05-26-s2-cpu-singleproc-v5"
 
 logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
 log = logging.getLogger(__name__)
@@ -269,16 +269,45 @@ def now_iso():
     return datetime.now().isoformat(timespec="seconds")
 
 
+def child_pids(pid: int):
+    children_path = Path(f"/proc/{pid}/task/{pid}/children")
+    if not children_path.exists():
+        return []
+    try:
+        direct_children = [int(item) for item in children_path.read_text(encoding="utf-8").split() if item.strip()]
+    except Exception:
+        return []
+    descendants = []
+    seen = set()
+    queue = list(direct_children)
+    while queue:
+        current = queue.pop(0)
+        if current in seen:
+            continue
+        seen.add(current)
+        descendants.append(current)
+        queue.extend(child_pids(current))
+    return descendants
+
+
 def process_metrics(pid: int, previous=None):
-    stat_path = Path(f"/proc/{pid}/stat")
-    statm_path = Path(f"/proc/{pid}/statm")
-    if not stat_path.exists() or not statm_path.exists():
+    pids = [pid, *child_pids(pid)]
+    live_pids = []
+    cpu_ticks = 0
+    rss_pages = 0
+    for current_pid in pids:
+        stat_path = Path(f"/proc/{current_pid}/stat")
+        statm_path = Path(f"/proc/{current_pid}/statm")
+        if not stat_path.exists() or not statm_path.exists():
+            continue
+        stat_fields = stat_path.read_text(encoding="utf-8").split()
+        cpu_ticks += int(stat_fields[13]) + int(stat_fields[14])
+        rss_pages += int(statm_path.read_text(encoding="utf-8").split()[1])
+        live_pids.append(current_pid)
+    if not live_pids:
         return {"alive": False, "cpu_percent": 0.0, "rss_mb": 0.0, "cpu_time_seconds": 0.0}
-    stat_fields = stat_path.read_text(encoding="utf-8").split()
-    cpu_ticks = int(stat_fields[13]) + int(stat_fields[14])
     clock_ticks = os.sysconf(os.sysconf_names["SC_CLK_TCK"])
     cpu_time_seconds = cpu_ticks / float(clock_ticks)
-    rss_pages = int(statm_path.read_text(encoding="utf-8").split()[1])
     page_size = os.sysconf("SC_PAGE_SIZE")
     rss_mb = rss_pages * page_size / (1024 * 1024)
     cpu_percent = 0.0
@@ -291,9 +320,26 @@ def process_metrics(pid: int, previous=None):
         "cpu_percent": round(cpu_percent, 2),
         "rss_mb": round(rss_mb, 2),
         "cpu_time_seconds": round(cpu_time_seconds, 2),
+        "process_count": len(live_pids),
     }
 
 
+def refresh_task_status(task):
+    if not task:
+        return None
+    pid = task.get("pid")
+    if pid is None:
+        return task
+    metrics = process_metrics(int(pid))
+    refreshed = dict(task)
+    refreshed.update(metrics)
+    refreshed["refreshed_at"] = now_iso()
+    if not metrics.get("alive"):
+        if refreshed.get("state") in {"starting", "running"} and refreshed.get("exit_code") is None:
+            refreshed["state"] = "stale"
+    return refreshed
+
+
 def status_summary(status):
     if not status:
         return ""
@@ -303,6 +349,8 @@ def status_summary(status):
     ]
     if status.get("pid") is not None:
         parts.append(f"pid={status['pid']}")
+    if status.get("process_count") is not None:
+        parts.append(f"proc={status['process_count']}")
     if status.get("cpu_percent") is not None:
         parts.append(f"cpu={status['cpu_percent']}%")
     if status.get("rss_mb") is not None:
@@ -996,6 +1044,46 @@ class Text2SemanticDataModule(LightningDataModule):
         s1_train.write_text(s1_train_content, encoding="utf-8")
     s2_train = GPT_SOVITS_DIR / "GPT_SoVITS" / "s2_train.py"
     s2_train_content = s2_train.read_text(encoding="utf-8")
+    old_main = """def main():
+    if torch.cuda.is_available():
+        n_gpus = torch.cuda.device_count()
+    else:
+        n_gpus = 1
+    os.environ["MASTER_ADDR"] = "localhost"
+    os.environ["MASTER_PORT"] = str(randint(20000, 55555))
+
+    mp.spawn(
+        run,
+        nprocs=n_gpus,
+        args=(
+            n_gpus,
+            hps,
+        ),
+    )
+"""
+    new_main = """def main():
+    if torch.cuda.is_available():
+        n_gpus = torch.cuda.device_count()
+    else:
+        n_gpus = 1
+    os.environ["MASTER_ADDR"] = "localhost"
+    os.environ["MASTER_PORT"] = str(randint(20000, 55555))
+
+    if not torch.cuda.is_available() and n_gpus == 1:
+        run(0, n_gpus, hps)
+        return
+
+    mp.spawn(
+        run,
+        nprocs=n_gpus,
+        args=(
+            n_gpus,
+            hps,
+        ),
+    )
+"""
+    if old_main in s2_train_content:
+        s2_train_content = s2_train_content.replace(old_main, new_main, 1)
     old_loader = """    train_loader = DataLoader(
         train_dataset,
         num_workers=5,
@@ -1505,9 +1593,9 @@ def agent_status(version=DEFAULT_VERSION):
             "gpt": [file_record(path) for path in gpt_files[:10]],
         },
         "tasks": {
-            "prepare": read_status_file(ctx.prep_status_path),
-            "sovits": read_status_file(ctx.sovits_status_path),
-            "gpt": read_status_file(ctx.gpt_status_path),
+            "prepare": refresh_task_status(read_status_file(ctx.prep_status_path)),
+            "sovits": refresh_task_status(read_status_file(ctx.sovits_status_path)),
+            "gpt": refresh_task_status(read_status_file(ctx.gpt_status_path)),
         },
     }