Training log
Browse files- training/train.log +132 -0
training/train.log
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[train] step 0: loss/diff: 0.789098, loss/stop: 0.503991, lr: 0.000000, epoch: 0.000000, grad_norm: 5.101328
|
| 2 |
+
[val] step 0: loss/total: 1.107430, loss/diff: 0.857099, loss/stop: 0.166888, log interval: 1.46s
|
| 3 |
+
[Audio] Starting audio generation for 2 samples at step 0
|
| 4 |
+
[Audio] Loaded reference audio for sample 0: duration=3.80s
|
| 5 |
+
[Audio] Generating sample 0 with text: 'ca yejudie nim zan ntable bun pa yehudia nim....'
|
| 6 |
+
[Audio] Generated audio for sample 0: duration=3.68s
|
| 7 |
+
[Audio] Created mel spectrogram figure for sample 0
|
| 8 |
+
[Audio] Loaded reference audio for sample 1: duration=1.07s
|
| 9 |
+
[Audio] Generating sample 1 with text: 'Nirigimi Graphic soli....'
|
| 10 |
+
[Audio] Generated audio for sample 1: duration=1.60s
|
| 11 |
+
[Audio] Created mel spectrogram figure for sample 1
|
| 12 |
+
[train] step 50: loss/diff: 0.788848, loss/stop: 0.335908, lr: 0.000001, epoch: 0.031728, grad_norm: 5.438103, log interval: 397.12s
|
| 13 |
+
[train] step 100: loss/diff: 0.828374, loss/stop: 0.069382, lr: 0.000003, epoch: 0.063456, grad_norm: 1.922445, log interval: 250.42s
|
| 14 |
+
[train] step 150: loss/diff: 0.923074, loss/stop: 0.125260, lr: 0.000004, epoch: 0.095183, grad_norm: 2.012486, log interval: 249.17s
|
| 15 |
+
[train] step 200: loss/diff: 0.932995, loss/stop: 0.035296, lr: 0.000005, epoch: 0.126911, grad_norm: 1.668936, log interval: 247.52s
|
| 16 |
+
[train] step 250: loss/diff: 0.796862, loss/stop: 0.029166, lr: 0.000006, epoch: 0.158639, grad_norm: 1.352380, log interval: 249.75s
|
| 17 |
+
[train] step 300: loss/diff: 0.927038, loss/stop: 0.107342, lr: 0.000008, epoch: 0.190367, grad_norm: 1.314299, log interval: 248.22s
|
| 18 |
+
[train] step 350: loss/diff: 0.852108, loss/stop: 0.013634, lr: 0.000009, epoch: 0.222094, grad_norm: 0.965734, log interval: 253.03s
|
| 19 |
+
[train] step 400: loss/diff: 0.909051, loss/stop: 0.102384, lr: 0.000010, epoch: 0.253822, grad_norm: 0.853521, log interval: 248.64s
|
| 20 |
+
[train] step 450: loss/diff: 0.693721, loss/stop: 0.060151, lr: 0.000010, epoch: 0.285550, grad_norm: 0.908810, log interval: 251.01s
|
| 21 |
+
[train] step 500: loss/diff: 0.726116, loss/stop: 0.012889, lr: 0.000010, epoch: 0.317278, grad_norm: 0.752108, log interval: 248.83s
|
| 22 |
+
[val] step 500: loss/total: 0.868330, loss/diff: 0.819919, loss/stop: 0.032275, log interval: 1.54s
|
| 23 |
+
[Audio] Starting audio generation for 2 samples at step 500
|
| 24 |
+
[Audio] Loaded reference audio for sample 0: duration=3.80s
|
| 25 |
+
[Audio] Generating sample 0 with text: 'ca yejudie nim zan ntable bun pa yehudia nim....'
|
| 26 |
+
[Audio] Generated audio for sample 0: duration=2.40s
|
| 27 |
+
[Audio] Created mel spectrogram figure for sample 0
|
| 28 |
+
[Audio] Loaded reference audio for sample 1: duration=1.07s
|
| 29 |
+
[Audio] Generating sample 1 with text: 'Nirigimi Graphic soli....'
|
| 30 |
+
[Audio] Generated audio for sample 1: duration=1.12s
|
| 31 |
+
[Audio] Created mel spectrogram figure for sample 1
|
| 32 |
+
[train] step 550: loss/diff: 0.806922, loss/stop: 0.007728, lr: 0.000010, epoch: 0.349006, grad_norm: 0.666208, log interval: 391.05s
|
| 33 |
+
[train] step 600: loss/diff: 0.701521, loss/stop: 0.008950, lr: 0.000010, epoch: 0.380733, grad_norm: 0.765606, log interval: 248.52s
|
| 34 |
+
[train] step 650: loss/diff: 0.924584, loss/stop: 0.232300, lr: 0.000010, epoch: 0.412461, grad_norm: 0.822268, log interval: 251.03s
|
| 35 |
+
[train] step 700: loss/diff: 0.931976, loss/stop: 0.198188, lr: 0.000010, epoch: 0.444189, grad_norm: 0.809766, log interval: 248.42s
|
| 36 |
+
[train] step 750: loss/diff: 0.635530, loss/stop: 0.122316, lr: 0.000010, epoch: 0.475917, grad_norm: 0.724147, log interval: 253.92s
|
| 37 |
+
[train] step 800: loss/diff: 0.805612, loss/stop: 0.008379, lr: 0.000010, epoch: 0.507644, grad_norm: 0.702823, log interval: 249.51s
|
| 38 |
+
[train] step 850: loss/diff: 0.812743, loss/stop: 0.067209, lr: 0.000010, epoch: 0.539372, grad_norm: 0.789986, log interval: 255.21s
|
| 39 |
+
[train] step 900: loss/diff: 0.797421, loss/stop: 0.035377, lr: 0.000010, epoch: 0.571100, grad_norm: 0.566310, log interval: 251.77s
|
| 40 |
+
[train] step 950: loss/diff: 0.740916, loss/stop: 0.003283, lr: 0.000010, epoch: 0.602828, grad_norm: 0.671982, log interval: 248.36s
|
| 41 |
+
[train] step 1000: loss/diff: 0.850881, loss/stop: 0.011768, lr: 0.000010, epoch: 0.634556, grad_norm: 0.944077, log interval: 252.65s
|
| 42 |
+
[val] step 1000: loss/total: 0.864477, loss/diff: 0.812996, loss/stop: 0.034321, log interval: 1.64s
|
| 43 |
+
[Audio] Starting audio generation for 2 samples at step 1000
|
| 44 |
+
[Audio] Loaded reference audio for sample 0: duration=3.80s
|
| 45 |
+
[Audio] Generating sample 0 with text: 'ca yejudie nim zan ntable bun pa yehudia nim....'
|
| 46 |
+
[Audio] Generated audio for sample 0: duration=4.16s
|
| 47 |
+
[Audio] Created mel spectrogram figure for sample 0
|
| 48 |
+
[Audio] Loaded reference audio for sample 1: duration=1.07s
|
| 49 |
+
[Audio] Generating sample 1 with text: 'Nirigimi Graphic soli....'
|
| 50 |
+
[Audio] Generated audio for sample 1: duration=1.12s
|
| 51 |
+
[Audio] Created mel spectrogram figure for sample 1
|
| 52 |
+
[train] step 1050: loss/diff: 0.739668, loss/stop: 0.007658, lr: 0.000010, epoch: 0.666283, grad_norm: 0.662255, log interval: 395.00s
|
| 53 |
+
[train] step 1100: loss/diff: 0.797663, loss/stop: 0.026830, lr: 0.000010, epoch: 0.698011, grad_norm: 0.709330, log interval: 248.76s
|
| 54 |
+
[train] step 1150: loss/diff: 0.789451, loss/stop: 0.163025, lr: 0.000010, epoch: 0.729739, grad_norm: 0.932318, log interval: 252.27s
|
| 55 |
+
[train] step 1200: loss/diff: 0.812164, loss/stop: 0.049388, lr: 0.000010, epoch: 0.761467, grad_norm: 0.767773, log interval: 245.93s
|
| 56 |
+
[train] step 1250: loss/diff: 0.737204, loss/stop: 0.003984, lr: 0.000010, epoch: 0.793194, grad_norm: 0.666597, log interval: 250.09s
|
| 57 |
+
[train] step 1300: loss/diff: 0.759161, loss/stop: 0.076431, lr: 0.000010, epoch: 0.824922, grad_norm: 0.672967, log interval: 249.81s
|
| 58 |
+
[train] step 1350: loss/diff: 0.762454, loss/stop: 0.012658, lr: 0.000010, epoch: 0.856650, grad_norm: 0.719600, log interval: 250.55s
|
| 59 |
+
[train] step 1400: loss/diff: 0.823602, loss/stop: 0.003835, lr: 0.000010, epoch: 0.888378, grad_norm: 0.610572, log interval: 253.05s
|
| 60 |
+
[train] step 1450: loss/diff: 0.842835, loss/stop: 0.021506, lr: 0.000010, epoch: 0.920105, grad_norm: 0.640176, log interval: 248.73s
|
| 61 |
+
[train] step 1500: loss/diff: 0.925031, loss/stop: 0.034859, lr: 0.000009, epoch: 0.951833, grad_norm: 0.671923, log interval: 252.37s
|
| 62 |
+
[val] step 1500: loss/total: 0.855272, loss/diff: 0.806172, loss/stop: 0.032734, log interval: 1.71s
|
| 63 |
+
[Audio] Starting audio generation for 2 samples at step 1500
|
| 64 |
+
[Audio] Loaded reference audio for sample 0: duration=3.80s
|
| 65 |
+
[Audio] Generating sample 0 with text: 'ca yejudie nim zan ntable bun pa yehudia nim....'
|
| 66 |
+
[Audio] Generated audio for sample 0: duration=4.48s
|
| 67 |
+
[Audio] Created mel spectrogram figure for sample 0
|
| 68 |
+
[Audio] Loaded reference audio for sample 1: duration=1.07s
|
| 69 |
+
[Audio] Generating sample 1 with text: 'Nirigimi Graphic soli....'
|
| 70 |
+
[Audio] Generated audio for sample 1: duration=1.12s
|
| 71 |
+
[Audio] Created mel spectrogram figure for sample 1
|
| 72 |
+
[train] step 1550: loss/diff: 0.826707, loss/stop: 0.042192, lr: 0.000009, epoch: 0.983561, grad_norm: 0.677085, log interval: 402.03s
|
| 73 |
+
[train] step 1600: loss/diff: 0.904812, loss/stop: 0.034896, lr: 0.000009, epoch: 1.015289, grad_norm: 0.656432, log interval: 250.20s
|
| 74 |
+
[train] step 1650: loss/diff: 0.863727, loss/stop: 0.099619, lr: 0.000009, epoch: 1.047017, grad_norm: 0.646881, log interval: 253.74s
|
| 75 |
+
[train] step 1700: loss/diff: 0.650111, loss/stop: 0.004085, lr: 0.000009, epoch: 1.078744, grad_norm: 0.746588, log interval: 249.99s
|
| 76 |
+
[train] step 1750: loss/diff: 0.778187, loss/stop: 0.023257, lr: 0.000009, epoch: 1.110472, grad_norm: 0.632165, log interval: 251.52s
|
| 77 |
+
[train] step 1800: loss/diff: 0.856697, loss/stop: 0.033471, lr: 0.000009, epoch: 1.142200, grad_norm: 0.651830, log interval: 254.93s
|
| 78 |
+
[train] step 1850: loss/diff: 0.729166, loss/stop: 0.011303, lr: 0.000009, epoch: 1.173928, grad_norm: 0.750736, log interval: 248.53s
|
| 79 |
+
[train] step 1900: loss/diff: 0.844783, loss/stop: 0.018158, lr: 0.000009, epoch: 1.205655, grad_norm: 1.068115, log interval: 256.72s
|
| 80 |
+
[train] step 1950: loss/diff: 0.749124, loss/stop: 0.042324, lr: 0.000009, epoch: 1.237383, grad_norm: 1.102926, log interval: 251.68s
|
| 81 |
+
[train] step 2000: loss/diff: 0.784226, loss/stop: 0.009501, lr: 0.000009, epoch: 1.269111, grad_norm: 0.748214, log interval: 250.90s
|
| 82 |
+
[val] step 2000: loss/total: 0.870591, loss/diff: 0.798133, loss/stop: 0.048305, log interval: 1.80s
|
| 83 |
+
[Audio] Starting audio generation for 2 samples at step 2000
|
| 84 |
+
[Audio] Loaded reference audio for sample 0: duration=3.80s
|
| 85 |
+
[Audio] Generating sample 0 with text: 'ca yejudie nim zan ntable bun pa yehudia nim....'
|
| 86 |
+
[Audio] Generated audio for sample 0: duration=4.48s
|
| 87 |
+
[Audio] Created mel spectrogram figure for sample 0
|
| 88 |
+
[Audio] Loaded reference audio for sample 1: duration=1.07s
|
| 89 |
+
[Audio] Generating sample 1 with text: 'Nirigimi Graphic soli....'
|
| 90 |
+
[Audio] Generated audio for sample 1: duration=1.12s
|
| 91 |
+
[Audio] Created mel spectrogram figure for sample 1
|
| 92 |
+
[train] step 2050: loss/diff: 0.859010, loss/stop: 0.034864, lr: 0.000009, epoch: 1.300839, grad_norm: 0.883937, log interval: 400.85s
|
| 93 |
+
[train] step 2100: loss/diff: 0.618100, loss/stop: 0.001664, lr: 0.000009, epoch: 1.332567, grad_norm: 0.638094, log interval: 249.98s
|
| 94 |
+
[train] step 2150: loss/diff: 0.703674, loss/stop: 0.003625, lr: 0.000009, epoch: 1.364294, grad_norm: 0.780594, log interval: 248.35s
|
| 95 |
+
[train] step 2200: loss/diff: 0.835902, loss/stop: 0.011624, lr: 0.000009, epoch: 1.396022, grad_norm: 0.912906, log interval: 248.75s
|
| 96 |
+
[train] step 2250: loss/diff: 0.756711, loss/stop: 0.037928, lr: 0.000009, epoch: 1.427750, grad_norm: 0.671276, log interval: 247.70s
|
| 97 |
+
[train] step 2300: loss/diff: 0.747084, loss/stop: 0.084500, lr: 0.000008, epoch: 1.459478, grad_norm: 0.797885, log interval: 253.25s
|
| 98 |
+
[train] step 2350: loss/diff: 0.832573, loss/stop: 0.001534, lr: 0.000008, epoch: 1.491205, grad_norm: 0.665750, log interval: 249.47s
|
| 99 |
+
[train] step 2400: loss/diff: 0.648599, loss/stop: 0.042551, lr: 0.000008, epoch: 1.522933, grad_norm: 0.743321, log interval: 249.53s
|
| 100 |
+
[train] step 2450: loss/diff: 0.743741, loss/stop: 0.008034, lr: 0.000008, epoch: 1.554661, grad_norm: 0.837090, log interval: 249.09s
|
| 101 |
+
[train] step 2500: loss/diff: 0.704607, loss/stop: 0.004469, lr: 0.000008, epoch: 1.586389, grad_norm: 0.678354, log interval: 253.81s
|
| 102 |
+
[val] step 2500: loss/total: 0.865124, loss/diff: 0.800481, loss/stop: 0.043095, log interval: 1.90s
|
| 103 |
+
[Audio] Starting audio generation for 2 samples at step 2500
|
| 104 |
+
[Audio] Loaded reference audio for sample 0: duration=3.80s
|
| 105 |
+
[Audio] Generating sample 0 with text: 'ca yejudie nim zan ntable bun pa yehudia nim....'
|
| 106 |
+
[Audio] Generated audio for sample 0: duration=4.16s
|
| 107 |
+
[Audio] Created mel spectrogram figure for sample 0
|
| 108 |
+
[Audio] Loaded reference audio for sample 1: duration=1.07s
|
| 109 |
+
[Audio] Generating sample 1 with text: 'Nirigimi Graphic soli....'
|
| 110 |
+
[Audio] Generated audio for sample 1: duration=1.12s
|
| 111 |
+
[Audio] Created mel spectrogram figure for sample 1
|
| 112 |
+
[train] step 2550: loss/diff: 0.803349, loss/stop: 0.027449, lr: 0.000008, epoch: 1.618117, grad_norm: 0.834545, log interval: 392.13s
|
| 113 |
+
[train] step 2600: loss/diff: 0.896383, loss/stop: 0.003762, lr: 0.000008, epoch: 1.649844, grad_norm: 0.757962, log interval: 250.09s
|
| 114 |
+
[train] step 2650: loss/diff: 0.828374, loss/stop: 0.038636, lr: 0.000008, epoch: 1.681572, grad_norm: 0.689122, log interval: 249.07s
|
| 115 |
+
[train] step 2700: loss/diff: 0.913328, loss/stop: 0.005403, lr: 0.000008, epoch: 1.713300, grad_norm: 0.760104, log interval: 253.34s
|
| 116 |
+
[train] step 2750: loss/diff: 0.729835, loss/stop: 0.023430, lr: 0.000008, epoch: 1.745028, grad_norm: 1.096714, log interval: 254.95s
|
| 117 |
+
[train] step 2800: loss/diff: 1.026032, loss/stop: 0.030930, lr: 0.000008, epoch: 1.776755, grad_norm: 0.810420, log interval: 252.27s
|
| 118 |
+
[train] step 2850: loss/diff: 0.623197, loss/stop: 0.056800, lr: 0.000008, epoch: 1.808483, grad_norm: 0.955575, log interval: 250.18s
|
| 119 |
+
[train] step 2900: loss/diff: 0.777317, loss/stop: 0.005526, lr: 0.000007, epoch: 1.840211, grad_norm: 1.080352, log interval: 250.65s
|
| 120 |
+
[train] step 2950: loss/diff: 0.707550, loss/stop: 0.125579, lr: 0.000007, epoch: 1.871939, grad_norm: 0.982410, log interval: 254.59s
|
| 121 |
+
[train] step 3000: loss/diff: 0.811348, loss/stop: 0.003190, lr: 0.000007, epoch: 1.903667, grad_norm: 0.711265, log interval: 251.38s
|
| 122 |
+
[val] step 3000: loss/total: 0.851482, loss/diff: 0.798783, loss/stop: 0.035133, log interval: 1.84s
|
| 123 |
+
[Audio] Starting audio generation for 2 samples at step 3000
|
| 124 |
+
[Audio] Loaded reference audio for sample 0: duration=3.80s
|
| 125 |
+
[Audio] Generating sample 0 with text: 'ca yejudie nim zan ntable bun pa yehudia nim....'
|
| 126 |
+
[Audio] Generated audio for sample 0: duration=3.68s
|
| 127 |
+
[Audio] Created mel spectrogram figure for sample 0
|
| 128 |
+
[Audio] Loaded reference audio for sample 1: duration=1.07s
|
| 129 |
+
[Audio] Generating sample 1 with text: 'Nirigimi Graphic soli....'
|
| 130 |
+
[Audio] Generated audio for sample 1: duration=1.12s
|
| 131 |
+
[Audio] Created mel spectrogram figure for sample 1
|
| 132 |
+
[train] step 3050: loss/diff: 0.917488, loss/stop: 0.087356, lr: 0.000007, epoch: 1.935394, grad_norm: 0.997853, log interval: 395.57s
|