daniya_trainer/dataset/load_dataset.py

#!/usr/bin/env python3
"""
Hugging Face 数据集加载脚本
"""

import os
import pandas as pd
from datasets import Dataset, Audio

# 配置
DATASET_DIR = "/path/to/dataset"  # 修改为您的数据集路径
AUDIO_DIR = os.path.join(DATASET_DIR, "raw_audio")
METADATA_FILE = os.path.join(DATASET_DIR, "metadata.csv")

def load_dataset():
    """加载数据集"""
    # 读取元数据
    df = pd.read_csv(METADATA_FILE)
    
    # 添加音频文件路径
    df["audio"] = df["file"].apply(lambda x: os.path.join(AUDIO_DIR, x))
    
    # 创建Hugging Face数据集
    dataset = Dataset.from_pandas(df)
    
    # 转换为音频格式
    dataset = dataset.cast_column("audio", Audio())
    
    return dataset

def upload_to_hub(dataset, repo_name):
    """上传到Hugging Face Hub"""
    from huggingface_hub import HfApi
    
    api = HfApi()
    api.upload_folder(
        folder_path=DATASET_DIR,
        repo_id=repo_name,
        repo_type="dataset",
    )
    print(f"数据集已上传到: https://huggingface.co/datasets/{repo_name}")

if __name__ == "__main__":
    # 加载数据集
    dataset = load_dataset()
    print(f"数据集大小: {len(dataset)}")
    print(f"示例: {dataset[0]}")
    
    # 上传到Hugging Face (取消注释以启用)
    # upload_to_hub(dataset, "your-username/daniya-voice-gptsovits")