Codex commited on
Commit
8711321
·
1 Parent(s): 444221d

Add safe web file management

Browse files
Files changed (1) hide show
  1. app.py +243 -2
app.py CHANGED
@@ -347,6 +347,198 @@ def artifacts_summary(version=DEFAULT_VERSION):
347
  )
348
 
349
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
350
  def dataset_prepared(ctx: VersionContext):
351
  prepared = (
352
  ctx.text_path.exists()
@@ -900,6 +1092,7 @@ def load_dashboard():
900
  return (
901
  version_markdown(version),
902
  *sync_live_state(version),
 
903
  )
904
 
905
 
@@ -975,7 +1168,8 @@ def create_ui():
975
  "`/check_environment(version)`、`/download_dataset(version)`、`/prepare_data(version)`、"
976
  "`/start_training(version, epochs, batch_size, save_every_epoch, lr)`、"
977
  "`/start_gpt_training(version, epochs, batch_size, save_every_epoch)`、"
978
- "`/refresh_outputs(version)`、`/agent_status(version)`"
 
979
  )
980
  agent_status_out = gr.JSON(label="Agent 状态", value=agent_status(DEFAULT_VERSION))
981
  agent_status_btn = gr.Button("刷新 Agent 状态", variant="secondary")
@@ -984,8 +1178,23 @@ def create_ui():
984
  live_logs_api_btn = gr.Button(visible=False)
985
  sync_state_api_btn = gr.Button(visible=False)
986
 
 
 
 
 
 
 
 
 
 
 
 
 
 
987
  refresh_outputs_targets = [refresh_text, output_dirs, refresh_sovits, refresh_gpt, all_sovits, all_gpt]
988
  sync_targets = [prep_out, sovits_log, gpt_log, *refresh_outputs_targets, agent_status_out]
 
 
989
  env_btn.click(check_environment, inputs=[version_select], outputs=env_out, api_name="check_environment")
990
  dataset_btn.click(download_dataset, inputs=[version_select], outputs=dataset_out, api_name="download_dataset")
991
  prep_btn.click(prepare_data, inputs=[version_select], outputs=prep_out, api_name="prepare_data")
@@ -1006,6 +1215,32 @@ def create_ui():
1006
  live_logs_api_btn.click(live_logs, inputs=[version_select], outputs=[prep_out, sovits_log, gpt_log], api_name="live_logs")
1007
  sync_state_api_btn.click(sync_live_state, inputs=[version_select], outputs=sync_targets, api_name="sync_live_state")
1008
  agent_status_btn.click(agent_status, inputs=[version_select], outputs=agent_status_out, api_name="agent_status")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1009
  version_select.change(version_markdown, inputs=[version_select], outputs=version_note, api_name=False)
1010
  version_select.change(
1011
  sync_live_state,
@@ -1013,6 +1248,12 @@ def create_ui():
1013
  outputs=sync_targets,
1014
  api_name=False,
1015
  )
 
 
 
 
 
 
1016
  sovits_event.then(
1017
  sync_live_state,
1018
  inputs=[version_select],
@@ -1035,7 +1276,7 @@ def create_ui():
1035
  )
1036
  demo.load(
1037
  load_dashboard,
1038
- outputs=[version_note, *sync_targets],
1039
  api_name=False,
1040
  )
1041
 
 
347
  )
348
 
349
 
350
+ def path_is_under(path: Path, root: Path):
351
+ try:
352
+ path.resolve().relative_to(root.resolve())
353
+ return True
354
+ except ValueError:
355
+ return False
356
+
357
+
358
+ def managed_roots(ctx: VersionContext):
359
+ return [
360
+ ("版本日志目录", ctx.exp_dir),
361
+ ("SoVITS 输出目录", ctx.sovits_output_dir),
362
+ ("GPT 输出目录", ctx.gpt_output_dir),
363
+ ("数据音频目录", AUDIO_DIR),
364
+ ("临时目录", TMP_ROOT),
365
+ ]
366
+
367
+
368
+ def managed_single_paths(ctx: VersionContext):
369
+ return [
370
+ ("metadata", DATASET_DIR / "metadata.csv"),
371
+ ("训练清单", ctx.input_list),
372
+ ("文本特征总表", ctx.text_path),
373
+ ("语义特征总表", ctx.semantic_path),
374
+ ("预处理实时日志", ctx.prep_live_log),
375
+ ("SoVITS 实时日志", ctx.sovits_live_log),
376
+ ("GPT 实时日志", ctx.gpt_live_log),
377
+ ]
378
+
379
+
380
+ def managed_path_choices(ctx: VersionContext):
381
+ items = set()
382
+ for _, root in managed_roots(ctx):
383
+ if not root.exists():
384
+ continue
385
+ for child in sorted(root.iterdir(), key=lambda item: item.name):
386
+ rel = child.relative_to(WORK_DIR).as_posix()
387
+ if child.is_dir():
388
+ rel = f"{rel}/"
389
+ items.add(rel)
390
+ for _, path in managed_single_paths(ctx):
391
+ if path.exists():
392
+ items.add(path.relative_to(WORK_DIR).as_posix())
393
+ return sorted(items)
394
+
395
+
396
+ def describe_path(path: Path):
397
+ if not path.exists():
398
+ return f"不存在: {path}"
399
+ lines = [
400
+ f"路径: {path}",
401
+ f"类型: {'目录' if path.is_dir() else '文件'}",
402
+ f"修改时间: {format_mtime(path)}",
403
+ ]
404
+ if path.is_file():
405
+ lines.append(f"大小: {format_size(path.stat().st_size)}")
406
+ else:
407
+ children = list(path.iterdir())
408
+ lines.append(f"直接子项: {len(children)}")
409
+ return "\n".join(lines)
410
+
411
+
412
+ def file_manager_summary(version=DEFAULT_VERSION):
413
+ ctx = get_version_context(version)
414
+ lines = [
415
+ "文件管理",
416
+ f"版本: {ctx.spec.version}",
417
+ "说明: 这里只暴露安全清理范围,不包含 GPT-SoVITS 仓库和预训练缓存。",
418
+ ]
419
+ for label, root in managed_roots(ctx):
420
+ if not root.exists():
421
+ lines.append(f"{label}: 不存在")
422
+ continue
423
+ children = sorted(root.iterdir(), key=lambda item: item.name)
424
+ lines.append(f"{label}: {root} | 直接子项 {len(children)}")
425
+ for child in children[:12]:
426
+ rel = child.relative_to(WORK_DIR).as_posix()
427
+ kind = "D" if child.is_dir() else "F"
428
+ suffix = "/" if child.is_dir() else ""
429
+ if child.is_dir():
430
+ extra = f"{len(list(child.iterdir()))} 项"
431
+ else:
432
+ extra = format_size(child.stat().st_size)
433
+ lines.append(f"[{kind}] {rel}{suffix} | {extra} | {format_mtime(child)}")
434
+ if len(children) > 12:
435
+ lines.append(f"... 其余 {len(children) - 12} 项省略")
436
+ for label, path in managed_single_paths(ctx):
437
+ if path.exists():
438
+ lines.append(f"{label}: {path.relative_to(WORK_DIR).as_posix()} | {format_size(path.stat().st_size)}")
439
+ return "\n".join(lines)
440
+
441
+
442
+ def file_manager_state(version=DEFAULT_VERSION, selected_path=None):
443
+ ctx = get_version_context(version)
444
+ choices = managed_path_choices(ctx)
445
+ if selected_path not in choices:
446
+ selected_path = choices[0] if choices else None
447
+ detail = describe_managed_path(version, selected_path)
448
+ return (
449
+ file_manager_summary(version),
450
+ gr.update(choices=choices, value=selected_path),
451
+ detail,
452
+ )
453
+
454
+
455
+ def describe_managed_path(version=DEFAULT_VERSION, selected_path=None):
456
+ if not selected_path:
457
+ return "未选择路径。"
458
+ target = (WORK_DIR / selected_path.rstrip("/")).resolve()
459
+ return describe_path(target)
460
+
461
+
462
+ def allowed_delete_roots(ctx: VersionContext):
463
+ return [
464
+ ctx.exp_dir,
465
+ ctx.sovits_output_dir,
466
+ ctx.gpt_output_dir,
467
+ AUDIO_DIR,
468
+ TMP_ROOT,
469
+ DATASET_DIR / "metadata.csv",
470
+ ctx.input_list,
471
+ ctx.text_path,
472
+ ctx.semantic_path,
473
+ ctx.prep_live_log,
474
+ ctx.sovits_live_log,
475
+ ctx.gpt_live_log,
476
+ ]
477
+
478
+
479
+ def is_allowed_delete_target(ctx: VersionContext, target: Path):
480
+ return any(path_is_under(target, root) for root in allowed_delete_roots(ctx))
481
+
482
+
483
+ def manager_action_result(version, status, selected_path=None):
484
+ return (
485
+ status,
486
+ *file_manager_state(version, selected_path),
487
+ *sync_live_state(version),
488
+ )
489
+
490
+
491
+ def delete_managed_path(version=DEFAULT_VERSION, selected_path=None):
492
+ ctx = get_version_context(version)
493
+ if not selected_path:
494
+ return manager_action_result(version, "未选择要删除的路径。", selected_path)
495
+ target = (WORK_DIR / selected_path.rstrip("/")).resolve()
496
+ if not target.exists():
497
+ return manager_action_result(version, f"路径不存在: {selected_path}", None)
498
+ if not is_allowed_delete_target(ctx, target):
499
+ return manager_action_result(version, f"拒绝删除: {selected_path} 不在安全范围内。", selected_path)
500
+ if target.is_dir():
501
+ shutil.rmtree(target)
502
+ status = f"已删除目录: {selected_path}"
503
+ else:
504
+ target.unlink()
505
+ status = f"已删除文件: {selected_path}"
506
+ return manager_action_result(version, status, None)
507
+
508
+
509
+ def cleanup_preprocess(version=DEFAULT_VERSION, selected_path=None):
510
+ ctx = get_version_context(version)
511
+ reset_preprocess_outputs(ctx)
512
+ clear_live_log(ctx.prep_live_log)
513
+ status = f"已清理 {ctx.spec.version} 的预处理产物。"
514
+ return manager_action_result(version, status, selected_path)
515
+
516
+
517
+ def cleanup_model_outputs(version=DEFAULT_VERSION, selected_path=None):
518
+ ctx = get_version_context(version)
519
+ for path in ctx.sovits_output_dir.glob("*.pth"):
520
+ path.unlink()
521
+ for path in ctx.gpt_output_dir.glob("*.ckpt"):
522
+ path.unlink()
523
+ if ctx.sovits_ckpt_dir.exists():
524
+ shutil.rmtree(ctx.sovits_ckpt_dir)
525
+ ctx.sovits_ckpt_dir.mkdir(parents=True, exist_ok=True)
526
+ if ctx.gpt_log_dir.exists():
527
+ shutil.rmtree(ctx.gpt_log_dir)
528
+ ctx.gpt_log_dir.mkdir(parents=True, exist_ok=True)
529
+ status = f"已清理 {ctx.spec.version} 的模型输出和训练断点。"
530
+ return manager_action_result(version, status, selected_path)
531
+
532
+
533
+ def cleanup_live_logs(version=DEFAULT_VERSION, selected_path=None):
534
+ ctx = get_version_context(version)
535
+ clear_live_log(ctx.prep_live_log)
536
+ clear_live_log(ctx.sovits_live_log)
537
+ clear_live_log(ctx.gpt_live_log)
538
+ status = f"已清理 {ctx.spec.version} 的实时日志。"
539
+ return manager_action_result(version, status, selected_path)
540
+
541
+
542
  def dataset_prepared(ctx: VersionContext):
543
  prepared = (
544
  ctx.text_path.exists()
 
1092
  return (
1093
  version_markdown(version),
1094
  *sync_live_state(version),
1095
+ *file_manager_state(version),
1096
  )
1097
 
1098
 
 
1168
  "`/check_environment(version)`、`/download_dataset(version)`、`/prepare_data(version)`、"
1169
  "`/start_training(version, epochs, batch_size, save_every_epoch, lr)`、"
1170
  "`/start_gpt_training(version, epochs, batch_size, save_every_epoch)`、"
1171
+ "`/refresh_outputs(version)`、`/agent_status(version)`"
1172
+ "`/live_logs(version)`、`/sync_live_state(version)`。"
1173
  )
1174
  agent_status_out = gr.JSON(label="Agent 状态", value=agent_status(DEFAULT_VERSION))
1175
  agent_status_btn = gr.Button("刷新 Agent 状态", variant="secondary")
 
1178
  live_logs_api_btn = gr.Button(visible=False)
1179
  sync_state_api_btn = gr.Button(visible=False)
1180
 
1181
+ gr.Markdown("### 10. 文件管理")
1182
+ gr.Markdown("支持安全清理当前版本的预处理、模型输出和实时日志,也可以删除选中的关键路径。")
1183
+ manager_status = gr.Textbox(label="文件管理操作结果", lines=3, interactive=False)
1184
+ manager_text = gr.Textbox(label="文件管理概览", lines=18, interactive=False, autoscroll=True)
1185
+ manager_select = gr.Dropdown(choices=[], value=None, label="可管理路径")
1186
+ manager_detail = gr.Textbox(label="所选路径详情", lines=5, interactive=False)
1187
+ with gr.Row():
1188
+ manager_refresh_btn = gr.Button("刷新文件列表", variant="secondary")
1189
+ manager_delete_btn = gr.Button("删除所选路径", variant="stop")
1190
+ cleanup_prep_btn = gr.Button("清理本版本预处理", variant="secondary")
1191
+ cleanup_models_btn = gr.Button("清理本版本模型输出", variant="secondary")
1192
+ cleanup_live_logs_btn = gr.Button("清理本版本实时日志", variant="secondary")
1193
+
1194
  refresh_outputs_targets = [refresh_text, output_dirs, refresh_sovits, refresh_gpt, all_sovits, all_gpt]
1195
  sync_targets = [prep_out, sovits_log, gpt_log, *refresh_outputs_targets, agent_status_out]
1196
+ manager_targets = [manager_text, manager_select, manager_detail]
1197
+ manager_action_targets = [manager_status, *manager_targets, *sync_targets]
1198
  env_btn.click(check_environment, inputs=[version_select], outputs=env_out, api_name="check_environment")
1199
  dataset_btn.click(download_dataset, inputs=[version_select], outputs=dataset_out, api_name="download_dataset")
1200
  prep_btn.click(prepare_data, inputs=[version_select], outputs=prep_out, api_name="prepare_data")
 
1215
  live_logs_api_btn.click(live_logs, inputs=[version_select], outputs=[prep_out, sovits_log, gpt_log], api_name="live_logs")
1216
  sync_state_api_btn.click(sync_live_state, inputs=[version_select], outputs=sync_targets, api_name="sync_live_state")
1217
  agent_status_btn.click(agent_status, inputs=[version_select], outputs=agent_status_out, api_name="agent_status")
1218
+ manager_refresh_btn.click(file_manager_state, inputs=[version_select, manager_select], outputs=manager_targets, api_name="file_manager_state")
1219
+ manager_select.change(describe_managed_path, inputs=[version_select, manager_select], outputs=manager_detail, api_name=False)
1220
+ manager_delete_btn.click(
1221
+ delete_managed_path,
1222
+ inputs=[version_select, manager_select],
1223
+ outputs=manager_action_targets,
1224
+ api_name="delete_managed_path",
1225
+ )
1226
+ cleanup_prep_btn.click(
1227
+ cleanup_preprocess,
1228
+ inputs=[version_select, manager_select],
1229
+ outputs=manager_action_targets,
1230
+ api_name="cleanup_preprocess",
1231
+ )
1232
+ cleanup_models_btn.click(
1233
+ cleanup_model_outputs,
1234
+ inputs=[version_select, manager_select],
1235
+ outputs=manager_action_targets,
1236
+ api_name="cleanup_model_outputs",
1237
+ )
1238
+ cleanup_live_logs_btn.click(
1239
+ cleanup_live_logs,
1240
+ inputs=[version_select, manager_select],
1241
+ outputs=manager_action_targets,
1242
+ api_name="cleanup_live_logs",
1243
+ )
1244
  version_select.change(version_markdown, inputs=[version_select], outputs=version_note, api_name=False)
1245
  version_select.change(
1246
  sync_live_state,
 
1248
  outputs=sync_targets,
1249
  api_name=False,
1250
  )
1251
+ version_select.change(
1252
+ file_manager_state,
1253
+ inputs=[version_select, manager_select],
1254
+ outputs=manager_targets,
1255
+ api_name=False,
1256
+ )
1257
  sovits_event.then(
1258
  sync_live_state,
1259
  inputs=[version_select],
 
1276
  )
1277
  demo.load(
1278
  load_dashboard,
1279
+ outputs=[version_note, *sync_targets, *manager_targets],
1280
  api_name=False,
1281
  )
1282