distill_fm sg1: .hydra/config.yaml

runs/vrs_flux_asymflow_distill_fm_globalflow_sg1_shift03_plainhead_ema_scratch_qkvattnfull_gridrange_poolmax_gridsubnorm_s001_01_a001_1/.hydra/config.yaml

@@ -0,0 +1,251 @@
+model:
+  sources:
+  - id: black-forest-labs/FLUX.2-klein-4B
+    pattern: text_encoder/*.safetensors
+  - id: black-forest-labs/FLUX.2-klein-base-4B
+    pattern: transformer/*.safetensors
+  - id: black-forest-labs/FLUX.2-klein-4B
+    pattern: vae/diffusion_pytorch_model.safetensors
+  model_paths: null
+  tokenizer_path: black-forest-labs/FLUX.2-klein-4B:tokenizer/
+  audio_processor_path: null
+  fp8_models: []
+  offload_models: []
+  extra_inputs: []
+  dit:
+    _target_: diffsynth.models.flux2_dit_ds_asym.Flux2DiTDSAsym
+    guidance_embeds: false
+    joint_attention_dim: 7680
+    num_attention_heads: 24
+    num_layers: 5
+    num_single_layers: 20
+    attention_head_dim: 128
+    in_channels: 128
+    out_channels: 128
+    patch_size: 1
+    axes_dims_rope:
+    - 32
+    - 32
+    - 32
+    - 32
+    use_frac_rope: true
+    crpa_key_mode: merged_uniform
+    crpa_integer_rope: true
+    vrs_size_pct: 0
+    attention_size_bias: false
+    crpa_key_pool_mode: max
+    crpa_rope_scale_gate: 'off'
+    crpa_rope_scale_gate_strength: 0.0
+    crpa_rope_scale_gate_beta: 1.0
+    crpa_rope_scale_gate_power: 1.0
+    crpa_attention_backend: block_sparse_cuda
+    use_leaf_size_embed: true
+    procrustes_path: inference/flux/prompt_grid_retrieval/procrustes_subspace_gridrange_subnorm.pt
+    asym_calibrate: true
+    hr_head_type: linear
+dataset:
+  _target_: diffsynth.core.data.vrs_flux2_ds_asym_dataset.VRSFlux2DSAsymDataset
+  cache_dir: /scratch/m000051/george/videogen/datasets/aesthetic_vrs_pyramid_s0.05_a0.05
+  pyramid_cache_dir: /scratch/m000051/george/videogen/datasets/aesthetic_vrs_pyramid_s0.05_a0.05
+  metadata_csv: /scratch/m000051-pm05/brianchc_2/Aesthetic-Train-V2/metadata.csv
+  image_path_root: /scratch/m000051-pm05/brianchc_2/Aesthetic-Train-V2
+  val_fraction: 0.05
+  split_seed: 42
+  upsample_mode: nearest
+  target_mode: multires
+  latents_source: pyramid
+  partition_source: compute
+  vrs_grid: ${vrs_grid}
+  grid_augment_max_p: 0.5
+vrs_grid:
+  sensitivity:
+  - 0.01
+  - 0.1
+  ambient:
+  - 0.01
+  - 1.0
+  error_metric: variance_aniso
+  anisotropic: true
+  uniform_merge: false
+  max_leaves: null
+  max_compression_ratio: 0.125
+  max_aspect_ratio: 8
+  error_smooth_sigma: 0.0
+dataloader:
+  num_workers: 10
+  balance_leaves: true
+  balance_seed: 0
+  balance_leaf_count_grid: precompute/leaf_count_grid_gridrange_s001_01_a001_1.pt
+training:
+  lora:
+    base_model: dit
+    rank: 256
+    dropout: 0.05
+    target_modules:
+    - linear_in
+    - linear_out
+    - timestep_embedder.linear_1
+    - timestep_embedder.linear_2
+    - to_q
+    - to_k
+    - to_v
+    - add_q_proj
+    - add_k_proj
+    - add_v_proj
+    - to_out.0
+    - to_add_out
+    - to_qkv_mlp_proj
+    - single_transformer_blocks.0.attn.to_out
+    - single_transformer_blocks.1.attn.to_out
+    - single_transformer_blocks.2.attn.to_out
+    - single_transformer_blocks.3.attn.to_out
+    - single_transformer_blocks.4.attn.to_out
+    - single_transformer_blocks.5.attn.to_out
+    - single_transformer_blocks.6.attn.to_out
+    - single_transformer_blocks.7.attn.to_out
+    - single_transformer_blocks.8.attn.to_out
+    - single_transformer_blocks.9.attn.to_out
+    - single_transformer_blocks.10.attn.to_out
+    - single_transformer_blocks.11.attn.to_out
+    - single_transformer_blocks.12.attn.to_out
+    - single_transformer_blocks.13.attn.to_out
+    - single_transformer_blocks.14.attn.to_out
+    - single_transformer_blocks.15.attn.to_out
+    - single_transformer_blocks.16.attn.to_out
+    - single_transformer_blocks.17.attn.to_out
+    - single_transformer_blocks.18.attn.to_out
+    - single_transformer_blocks.19.attn.to_out
+    checkpoint: null
+    preset_path: null
+    preset_model: null
+  task: sft_ds
+  loss_type: asym_ds_distill_fm
+  learning_rate: 0.0001
+  num_epochs: 1
+  max_steps: 50000
+  batch_size: 4
+  weight_decay: 0.01
+  trainable_models: dit
+  prediction_type: clean
+  asym_loss_space: teacher_global_flow
+  asym_sigma_min: 0.05
+  asym_recover_sigma_min: 0.01
+  asym_teacher_velocity_aux_weight: 0.0
+  asym_teacher_velocity_aux_clip: 0.0
+  asym_teacher_global_scale: 1
+  asym_loss_shift: 0.3
+  asym_fm_loss_weight: 1.0
+  asym_distill_loss_weight: 1.0
+  shift_schedule: 'off'
+  shift_mu: 2.833213344056216
+  ema:
+    enable: true
+    gamma: 7.0
+    start_step: 100
+    interval: 1
+  val_every_n_steps: null
+  val_n_batches: 10
+  full_finetune_groups:
+  - name: hr_heads
+    base_model: dit
+    match:
+    - hr_heads
+    lr: 0.001
+    weight_decay: 0.0
+  - name: leaf_size_embed
+    base_model: dit
+    match:
+    - leaf_size_embed
+    lr: 0.001
+    weight_decay: 0.0
+  - name: x_embedder
+    base_model: dit
+    match:
+    - x_embedder
+    lr: 0.0001
+    weight_decay: 0.0
+  - name: norm_out
+    base_model: dit
+    match:
+    - norm_out
+    lr: 0.0001
+    weight_decay: 0.0
+logging:
+  wandb:
+    entity: georgenakayama
+    project: wan-video-training
+    run_name: ${run_name}
+    log_interval: 1
+  samples:
+    every_n_steps: 2500
+    num_train_samples: 6
+    num_val_samples: 6
+    cfg_scale: 3.5
+    negative_prompt: ''
+vae:
+  model:
+    _target_: diffsynth.models.flux2_vae.Flux2VAE
+    decode_mode: stitched
+    upsample_mode: nearest
+    fourier_pe:
+      enabled: false
+      num_freqs: 8
+      max_freq: null
+  weights: null
+  model_id: black-forest-labs/FLUX.2-klein-4B
+  pattern: vae/diffusion_pytorch_model.safetensors
+eval:
+  n_samples: null
+  num_strip_samples: null
+  batch_size: 1
+  gt_from_latents: false
+  shift_schedule: 'off'
+  shift_mu: 2.833213344056216
+  orthogonal_guidance: 0.0
+  use_ema: false
+  lora_checkpoint: null
+  lora_alpha_scale: 1.0
+  resume_from_dir: null
+  keep_images: false
+  nccl_timeout_min: 120
+  grid_source: cache
+  retrieval:
+    method: clip
+    cache_dir: null
+    split_seed: 42
+    val_fraction: 0.05
+    n_pool: 8000
+    pool_seed: 1234
+    openai_model: text-embedding-3-large
+    device: cpu
+    openai_key_file: null
+    emb_cache_dir: null
+output:
+  output_path: ./models
+  remove_prefix_in_ckpt: pipe.dit.
+  save_steps: 500
+  max_checkpoints: 1
+  permanent_save_steps: 2500
+out_dir: null
+run_name: vrs_flux_asymflow_distill_fm_globalflow_sg1_shift03_plainhead_ema_scratch_qkvattnfull_gridrange_poolmax_gridsubnorm_s001_01_a001_1
+image:
+  height: 1024
+  width: 1024
+  max_pixels: 1048576
+resume_from: /scratch/m000133/george/foveated_diffusion_3d_dump/train_flux_ds/vrs_flux_asymflow_distill_fm_globalflow_sg1_shift03_plainhead_ema_scratch_qkvattnfull_gridrange_poolmax_gridsubnorm_s001_01_a001_1/step-25000.pt
+gradient:
+  use_checkpointing: true
+  use_checkpointing_offload: false
+  accumulation_steps: 1
+  max_grad_norm: 200
+accelerate:
+  find_unused_parameters: true
+  initialize_model_on_cpu: false
+  use_fsdp: false
+timestep:
+  max_boundary: 1.0
+  min_boundary: 0.0
+inference:
+  cfg_scale: 1.0
+  num_inference_steps: 30
+  seed: 42