File size: 65,261 Bytes
b02f1ad |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 |
Loading training set...
Num images: 216618
Image shape: [1, 256, 256]
Label shape: [0]
Constructing networks...
Setting up PyTorch plugin "bias_act_plugin"... Done.
Setting up PyTorch plugin "upfirdn2d_plugin"... Done.
Generator Parameters Buffers Output shape Datatype
--- --- --- --- ---
mapping.fc0 262656 - [16, 512] float32
mapping.fc1 262656 - [16, 512] float32
mapping - 512 [16, 14, 512] float32
synthesis.b4.conv1 2622465 32 [16, 512, 4, 4] float32
synthesis.b4.torgb 263169 - [16, 1, 4, 4] float32
synthesis.b4:0 8192 16 [16, 512, 4, 4] float32
synthesis.b4:1 - - [16, 512, 4, 4] float32
synthesis.b8.conv0 2622465 80 [16, 512, 8, 8] float32
synthesis.b8.conv1 2622465 80 [16, 512, 8, 8] float32
synthesis.b8.torgb 263169 - [16, 1, 8, 8] float32
synthesis.b8:0 - 16 [16, 512, 8, 8] float32
synthesis.b8:1 - - [16, 512, 8, 8] float32
synthesis.b16.conv0 2622465 272 [16, 512, 16, 16] float32
synthesis.b16.conv1 2622465 272 [16, 512, 16, 16] float32
synthesis.b16.torgb 263169 - [16, 1, 16, 16] float32
synthesis.b16:0 - 16 [16, 512, 16, 16] float32
synthesis.b16:1 - - [16, 512, 16, 16] float32
synthesis.b32.conv0 2622465 1040 [16, 512, 32, 32] float16
synthesis.b32.conv1 2622465 1040 [16, 512, 32, 32] float16
synthesis.b32.torgb 263169 - [16, 1, 32, 32] float16
synthesis.b32:0 - 16 [16, 512, 32, 32] float16
synthesis.b32:1 - - [16, 512, 32, 32] float32
synthesis.b64.conv0 1442561 4112 [16, 256, 64, 64] float16
synthesis.b64.conv1 721409 4112 [16, 256, 64, 64] float16
synthesis.b64.torgb 131585 - [16, 1, 64, 64] float16
synthesis.b64:0 - 16 [16, 256, 64, 64] float16
synthesis.b64:1 - - [16, 256, 64, 64] float32
synthesis.b128.conv0 426369 16400 [16, 128, 128, 128] float16
synthesis.b128.conv1 213249 16400 [16, 128, 128, 128] float16
synthesis.b128.torgb 65793 - [16, 1, 128, 128] float16
synthesis.b128:0 - 16 [16, 128, 128, 128] float16
synthesis.b128:1 - - [16, 128, 128, 128] float32
synthesis.b256.conv0 139457 65552 [16, 64, 256, 256] float16
synthesis.b256.conv1 69761 65552 [16, 64, 256, 256] float16
synthesis.b256.torgb 32897 - [16, 1, 256, 256] float16
synthesis.b256:0 - 16 [16, 64, 256, 256] float16
synthesis.b256:1 - - [16, 64, 256, 256] float32
--- --- --- --- ---
Total 23186516 175568 - -
Discriminator Parameters Buffers Output shape Datatype
--- --- --- --- ---
b256.fromrgb 128 16 [16, 64, 256, 256] float16
b256.skip 8192 16 [16, 128, 128, 128] float16
b256.conv0 36928 16 [16, 64, 256, 256] float16
b256.conv1 73856 16 [16, 128, 128, 128] float16
b256 - 16 [16, 128, 128, 128] float16
b128.skip 32768 16 [16, 256, 64, 64] float16
b128.conv0 147584 16 [16, 128, 128, 128] float16
b128.conv1 295168 16 [16, 256, 64, 64] float16
b128 - 16 [16, 256, 64, 64] float16
b64.skip 131072 16 [16, 512, 32, 32] float16
b64.conv0 590080 16 [16, 256, 64, 64] float16
b64.conv1 1180160 16 [16, 512, 32, 32] float16
b64 - 16 [16, 512, 32, 32] float16
b32.skip 262144 16 [16, 512, 16, 16] float16
b32.conv0 2359808 16 [16, 512, 32, 32] float16
b32.conv1 2359808 16 [16, 512, 16, 16] float16
b32 - 16 [16, 512, 16, 16] float16
b16.skip 262144 16 [16, 512, 8, 8] float32
b16.conv0 2359808 16 [16, 512, 16, 16] float32
b16.conv1 2359808 16 [16, 512, 8, 8] float32
b16 - 16 [16, 512, 8, 8] float32
b8.skip 262144 16 [16, 512, 4, 4] float32
b8.conv0 2359808 16 [16, 512, 8, 8] float32
b8.conv1 2359808 16 [16, 512, 4, 4] float32
b8 - 16 [16, 512, 4, 4] float32
b4.mbstd - - [16, 513, 4, 4] float32
b4.conv 2364416 16 [16, 512, 4, 4] float32
b4.fc 4194816 - [16, 512] float32
b4.out 513 - [16, 1] float32
--- --- --- --- ---
Total 24000961 416 - -
Setting up augmentation...
Distributing across 1 GPUs...
Setting up training phases...
Exporting sample images...
Initializing logs...
Skipping tfevents export: No module named 'tensorboard'
Training for 25000 kimg...
tick 0 kimg 0.0 time 15s sec/tick 2.3 sec/kimg 145.37 maintenance 12.3 cpumem 4.42 gpumem 9.10 augment 0.000
Evaluating metrics...
{"results": {"fid50k_full": 329.8177171102783}, "metric": "fid50k_full", "total_time": 451.604070186615, "total_time_str": "7m 32s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000000.pkl", "timestamp": 1694648098.9089189}
{"results": {"kid50k_full": 0.4050724111536537}, "metric": "kid50k_full", "total_time": 357.8517601490021, "total_time_str": "5m 58s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000000.pkl", "timestamp": 1694648456.760992}
tick 1 kimg 4.0 time 15m 44s sec/tick 117.2 sec/kimg 29.29 maintenance 811.9 cpumem 5.03 gpumem 4.96 augment 0.004
tick 2 kimg 8.0 time 17m 41s sec/tick 117.7 sec/kimg 29.42 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.010
tick 3 kimg 12.0 time 19m 39s sec/tick 117.2 sec/kimg 29.30 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.015
tick 4 kimg 16.0 time 21m 36s sec/tick 117.6 sec/kimg 29.39 maintenance 0.1 cpumem 5.03 gpumem 4.96 augment 0.020
tick 5 kimg 20.0 time 23m 34s sec/tick 117.5 sec/kimg 29.37 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.026
tick 6 kimg 24.0 time 25m 32s sec/tick 117.7 sec/kimg 29.43 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.031
tick 7 kimg 28.0 time 27m 29s sec/tick 117.6 sec/kimg 29.40 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.037
tick 8 kimg 32.0 time 29m 27s sec/tick 117.4 sec/kimg 29.36 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.044
tick 9 kimg 36.0 time 31m 24s sec/tick 117.4 sec/kimg 29.36 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.050
tick 10 kimg 40.0 time 33m 22s sec/tick 118.0 sec/kimg 29.50 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.056
tick 11 kimg 44.0 time 35m 20s sec/tick 117.6 sec/kimg 29.39 maintenance 0.1 cpumem 5.03 gpumem 4.98 augment 0.061
tick 12 kimg 48.0 time 37m 18s sec/tick 118.2 sec/kimg 29.54 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.065
tick 13 kimg 52.0 time 39m 16s sec/tick 117.5 sec/kimg 29.37 maintenance 0.1 cpumem 5.03 gpumem 4.98 augment 0.072
tick 14 kimg 56.0 time 41m 14s sec/tick 117.9 sec/kimg 29.46 maintenance 0.1 cpumem 5.03 gpumem 4.97 augment 0.079
tick 15 kimg 60.0 time 43m 12s sec/tick 117.6 sec/kimg 29.39 maintenance 0.1 cpumem 5.03 gpumem 4.99 augment 0.086
Evaluating metrics...
{"results": {"fid50k_full": 273.4672845272456}, "metric": "fid50k_full", "total_time": 250.90766167640686, "total_time_str": "4m 11s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000060.pkl", "timestamp": 1694650475.8410494}
{"results": {"kid50k_full": 0.3507548148898898}, "metric": "kid50k_full", "total_time": 235.7230269908905, "total_time_str": "3m 56s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000060.pkl", "timestamp": 1694650711.5643423}
tick 16 kimg 64.0 time 53m 19s sec/tick 117.9 sec/kimg 29.48 maintenance 489.8 cpumem 4.92 gpumem 4.99 augment 0.092
tick 17 kimg 68.0 time 55m 17s sec/tick 117.6 sec/kimg 29.41 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.098
tick 18 kimg 72.0 time 57m 15s sec/tick 118.0 sec/kimg 29.51 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.104
tick 19 kimg 76.0 time 59m 13s sec/tick 117.4 sec/kimg 29.35 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.111
tick 20 kimg 80.0 time 1h 01m 11s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.117
tick 21 kimg 84.0 time 1h 03m 09s sec/tick 117.9 sec/kimg 29.47 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.122
tick 22 kimg 88.0 time 1h 05m 07s sec/tick 117.8 sec/kimg 29.44 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.130
tick 23 kimg 92.0 time 1h 07m 05s sec/tick 117.9 sec/kimg 29.49 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.136
tick 24 kimg 96.0 time 1h 09m 03s sec/tick 117.9 sec/kimg 29.47 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.140
tick 25 kimg 100.0 time 1h 11m 00s sec/tick 117.7 sec/kimg 29.42 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.146
tick 26 kimg 104.0 time 1h 12m 59s sec/tick 118.1 sec/kimg 29.54 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.151
tick 27 kimg 108.0 time 1h 14m 56s sec/tick 117.6 sec/kimg 29.39 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.157
tick 28 kimg 112.0 time 1h 16m 55s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.92 gpumem 5.00 augment 0.164
tick 29 kimg 116.0 time 1h 18m 52s sec/tick 117.9 sec/kimg 29.46 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.168
tick 30 kimg 120.0 time 1h 20m 51s sec/tick 118.1 sec/kimg 29.52 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.167
Evaluating metrics...
{"results": {"fid50k_full": 208.02683845145685}, "metric": "fid50k_full", "total_time": 249.75050449371338, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000120.pkl", "timestamp": 1694652734.7345576}
{"results": {"kid50k_full": 0.2474209309609609}, "metric": "kid50k_full", "total_time": 236.35089206695557, "total_time_str": "3m 56s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000120.pkl", "timestamp": 1694652971.0857062}
tick 31 kimg 124.0 time 1h 30m 59s sec/tick 118.0 sec/kimg 29.51 maintenance 490.3 cpumem 4.92 gpumem 4.99 augment 0.168
tick 32 kimg 128.0 time 1h 32m 57s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.171
tick 33 kimg 132.0 time 1h 34m 55s sec/tick 117.8 sec/kimg 29.44 maintenance 0.1 cpumem 4.92 gpumem 5.05 augment 0.171
tick 34 kimg 136.0 time 1h 36m 53s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.171
tick 35 kimg 140.0 time 1h 38m 51s sec/tick 117.6 sec/kimg 29.41 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.171
tick 36 kimg 144.0 time 1h 40m 50s sec/tick 118.4 sec/kimg 29.61 maintenance 0.1 cpumem 4.92 gpumem 5.00 augment 0.170
tick 37 kimg 148.0 time 1h 42m 48s sec/tick 118.2 sec/kimg 29.56 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.165
tick 38 kimg 152.0 time 1h 44m 46s sec/tick 118.1 sec/kimg 29.53 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.163
tick 39 kimg 156.0 time 1h 46m 44s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.158
tick 40 kimg 160.0 time 1h 48m 43s sec/tick 118.1 sec/kimg 29.54 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.151
tick 41 kimg 164.0 time 1h 50m 41s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 4.92 gpumem 5.00 augment 0.144
tick 42 kimg 168.0 time 1h 52m 40s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.92 gpumem 5.00 augment 0.138
tick 43 kimg 172.0 time 1h 54m 38s sec/tick 117.9 sec/kimg 29.48 maintenance 0.1 cpumem 4.92 gpumem 4.99 augment 0.132
tick 44 kimg 176.0 time 1h 56m 36s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.92 gpumem 5.00 augment 0.126
tick 45 kimg 180.0 time 1h 58m 34s sec/tick 118.2 sec/kimg 29.54 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.119
Evaluating metrics...
{"results": {"fid50k_full": 80.07145157262984}, "metric": "fid50k_full", "total_time": 250.13644075393677, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000180.pkl", "timestamp": 1694654997.8041}
{"results": {"kid50k_full": 0.07736795149149157}, "metric": "kid50k_full", "total_time": 236.16426181793213, "total_time_str": "3m 56s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000180.pkl", "timestamp": 1694655233.9686294}
tick 46 kimg 184.0 time 2h 08m 41s sec/tick 117.6 sec/kimg 29.41 maintenance 489.4 cpumem 5.04 gpumem 4.99 augment 0.113
tick 47 kimg 188.0 time 2h 10m 40s sec/tick 118.1 sec/kimg 29.52 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.107
tick 48 kimg 192.0 time 2h 12m 38s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.04 gpumem 4.99 augment 0.101
tick 49 kimg 196.0 time 2h 14m 36s sec/tick 117.6 sec/kimg 29.40 maintenance 0.1 cpumem 5.04 gpumem 4.99 augment 0.095
tick 50 kimg 200.0 time 2h 16m 34s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 5.04 gpumem 4.99 augment 0.088
tick 51 kimg 204.0 time 2h 18m 32s sec/tick 117.9 sec/kimg 29.47 maintenance 0.1 cpumem 5.04 gpumem 5.00 augment 0.084
tick 52 kimg 208.0 time 2h 20m 30s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.079
tick 53 kimg 212.0 time 2h 22m 29s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 5.04 gpumem 4.99 augment 0.075
tick 54 kimg 216.0 time 2h 24m 27s sec/tick 117.6 sec/kimg 29.41 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.070
tick 55 kimg 220.0 time 2h 26m 25s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.064
tick 56 kimg 224.0 time 2h 28m 23s sec/tick 118.1 sec/kimg 29.52 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.061
tick 57 kimg 228.0 time 2h 30m 21s sec/tick 117.9 sec/kimg 29.48 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.057
tick 58 kimg 232.0 time 2h 32m 19s sec/tick 118.3 sec/kimg 29.58 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.054
tick 59 kimg 236.0 time 2h 34m 17s sec/tick 117.6 sec/kimg 29.40 maintenance 0.1 cpumem 5.04 gpumem 4.98 augment 0.051
tick 60 kimg 240.0 time 2h 36m 15s sec/tick 118.2 sec/kimg 29.54 maintenance 0.1 cpumem 5.04 gpumem 4.99 augment 0.049
Evaluating metrics...
{"results": {"fid50k_full": 52.725451648171486}, "metric": "fid50k_full", "total_time": 248.302996635437, "total_time_str": "4m 08s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000240.pkl", "timestamp": 1694657257.2457561}
{"results": {"kid50k_full": 0.044758790107607596}, "metric": "kid50k_full", "total_time": 236.0427005290985, "total_time_str": "3m 56s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000240.pkl", "timestamp": 1694657493.288714}
tick 61 kimg 244.0 time 2h 46m 21s sec/tick 118.2 sec/kimg 29.56 maintenance 487.7 cpumem 4.92 gpumem 4.97 augment 0.047
tick 62 kimg 248.0 time 2h 48m 20s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.044
tick 63 kimg 252.0 time 2h 50m 18s sec/tick 118.4 sec/kimg 29.61 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.041
tick 64 kimg 256.0 time 2h 52m 17s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.040
tick 65 kimg 260.0 time 2h 54m 15s sec/tick 118.1 sec/kimg 29.53 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.039
tick 66 kimg 264.0 time 2h 56m 14s sec/tick 118.7 sec/kimg 29.68 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.036
tick 67 kimg 268.0 time 2h 58m 12s sec/tick 118.1 sec/kimg 29.53 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.033
tick 68 kimg 272.0 time 3h 00m 10s sec/tick 118.2 sec/kimg 29.56 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.031
tick 69 kimg 276.0 time 3h 02m 09s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.027
tick 70 kimg 280.0 time 3h 04m 07s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.023
tick 71 kimg 284.0 time 3h 06m 06s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.022
tick 72 kimg 288.0 time 3h 08m 04s sec/tick 118.7 sec/kimg 29.67 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.021
tick 73 kimg 292.0 time 3h 10m 02s sec/tick 118.1 sec/kimg 29.51 maintenance 0.1 cpumem 4.92 gpumem 4.98 augment 0.022
tick 74 kimg 296.0 time 3h 12m 01s sec/tick 118.6 sec/kimg 29.66 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.021
tick 75 kimg 300.0 time 3h 13m 59s sec/tick 118.0 sec/kimg 29.50 maintenance 0.1 cpumem 4.92 gpumem 4.97 augment 0.020
Evaluating metrics...
{"results": {"fid50k_full": 40.158840884679236}, "metric": "fid50k_full", "total_time": 250.24572324752808, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000300.pkl", "timestamp": 1694659523.1340787}
{"results": {"kid50k_full": 0.028655740978478457}, "metric": "kid50k_full", "total_time": 236.63052415847778, "total_time_str": "3m 57s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000300.pkl", "timestamp": 1694659759.7648668}
tick 76 kimg 304.0 time 3h 24m 08s sec/tick 118.2 sec/kimg 29.54 maintenance 490.3 cpumem 4.93 gpumem 4.97 augment 0.016
tick 77 kimg 308.0 time 3h 26m 06s sec/tick 118.3 sec/kimg 29.58 maintenance 0.1 cpumem 4.93 gpumem 4.99 augment 0.017
tick 78 kimg 312.0 time 3h 28m 05s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.017
tick 79 kimg 316.0 time 3h 30m 03s sec/tick 118.1 sec/kimg 29.53 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.016
tick 80 kimg 320.0 time 3h 32m 02s sec/tick 118.7 sec/kimg 29.68 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.015
tick 81 kimg 324.0 time 3h 34m 00s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.014
tick 82 kimg 328.0 time 3h 35m 59s sec/tick 118.6 sec/kimg 29.66 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.013
tick 83 kimg 332.0 time 3h 37m 57s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.013
tick 84 kimg 336.0 time 3h 39m 56s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.013
tick 85 kimg 340.0 time 3h 41m 54s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.012
tick 86 kimg 344.0 time 3h 43m 52s sec/tick 118.3 sec/kimg 29.56 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.012
tick 87 kimg 348.0 time 3h 45m 51s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.011
tick 88 kimg 352.0 time 3h 47m 50s sec/tick 118.7 sec/kimg 29.68 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 89 kimg 356.0 time 3h 49m 48s sec/tick 118.1 sec/kimg 29.54 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.008
tick 90 kimg 360.0 time 3h 51m 47s sec/tick 118.7 sec/kimg 29.68 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.006
Evaluating metrics...
{"results": {"fid50k_full": 30.28247706883736}, "metric": "fid50k_full", "total_time": 249.87867975234985, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000360.pkl", "timestamp": 1694661790.3108518}
{"results": {"kid50k_full": 0.020121735555555572}, "metric": "kid50k_full", "total_time": 236.41930389404297, "total_time_str": "3m 56s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000360.pkl", "timestamp": 1694662026.7304013}
tick 91 kimg 364.0 time 4h 01m 55s sec/tick 118.3 sec/kimg 29.58 maintenance 489.7 cpumem 5.05 gpumem 4.97 augment 0.004
tick 92 kimg 368.0 time 4h 03m 54s sec/tick 118.7 sec/kimg 29.67 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 93 kimg 372.0 time 4h 05m 52s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.003
tick 94 kimg 376.0 time 4h 07m 51s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 95 kimg 380.0 time 4h 09m 49s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.003
tick 96 kimg 384.0 time 4h 11m 48s sec/tick 118.7 sec/kimg 29.67 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 97 kimg 388.0 time 4h 13m 46s sec/tick 118.2 sec/kimg 29.56 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.003
tick 98 kimg 392.0 time 4h 15m 45s sec/tick 118.6 sec/kimg 29.66 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.006
tick 99 kimg 396.0 time 4h 17m 43s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.007
tick 100 kimg 400.0 time 4h 19m 42s sec/tick 118.5 sec/kimg 29.64 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.009
tick 101 kimg 404.0 time 4h 21m 41s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.010
tick 102 kimg 408.0 time 4h 23m 39s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.009
tick 103 kimg 412.0 time 4h 25m 38s sec/tick 118.7 sec/kimg 29.68 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.010
tick 104 kimg 416.0 time 4h 27m 37s sec/tick 118.8 sec/kimg 29.70 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.007
tick 105 kimg 420.0 time 4h 29m 35s sec/tick 118.0 sec/kimg 29.51 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.007
Evaluating metrics...
{"results": {"fid50k_full": 28.32384051567694}, "metric": "fid50k_full", "total_time": 249.94679832458496, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000420.pkl", "timestamp": 1694664058.477075}
{"results": {"kid50k_full": 0.019354096509008996}, "metric": "kid50k_full", "total_time": 236.72545766830444, "total_time_str": "3m 57s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000420.pkl", "timestamp": 1694664295.2027884}
tick 106 kimg 424.0 time 4h 39m 43s sec/tick 118.1 sec/kimg 29.52 maintenance 490.1 cpumem 4.93 gpumem 4.97 augment 0.008
tick 107 kimg 428.0 time 4h 41m 42s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.98 augment 0.008
tick 108 kimg 432.0 time 4h 43m 40s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.009
tick 109 kimg 436.0 time 4h 45m 38s sec/tick 118.3 sec/kimg 29.58 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 110 kimg 440.0 time 4h 47m 36s sec/tick 118.0 sec/kimg 29.51 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 111 kimg 444.0 time 4h 49m 35s sec/tick 118.1 sec/kimg 29.53 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.011
tick 112 kimg 448.0 time 4h 51m 33s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.012
tick 113 kimg 452.0 time 4h 53m 31s sec/tick 118.0 sec/kimg 29.51 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.009
tick 114 kimg 456.0 time 4h 55m 30s sec/tick 118.6 sec/kimg 29.64 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.011
tick 115 kimg 460.0 time 4h 57m 28s sec/tick 118.3 sec/kimg 29.56 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 116 kimg 464.0 time 4h 59m 26s sec/tick 118.1 sec/kimg 29.52 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 117 kimg 468.0 time 5h 01m 25s sec/tick 118.6 sec/kimg 29.64 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 118 kimg 472.0 time 5h 03m 23s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 119 kimg 476.0 time 5h 05m 22s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.010
tick 120 kimg 480.0 time 5h 07m 20s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.008
Evaluating metrics...
{"results": {"fid50k_full": 26.00080113329641}, "metric": "fid50k_full", "total_time": 250.26559925079346, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000480.pkl", "timestamp": 1694666323.8536448}
{"results": {"kid50k_full": 0.019805346476476488}, "metric": "kid50k_full", "total_time": 236.81830716133118, "total_time_str": "3m 57s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000480.pkl", "timestamp": 1694666560.6722221}
tick 121 kimg 484.0 time 5h 17m 29s sec/tick 118.4 sec/kimg 29.59 maintenance 490.5 cpumem 4.93 gpumem 4.98 augment 0.008
tick 122 kimg 488.0 time 5h 19m 27s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.008
tick 123 kimg 492.0 time 5h 21m 26s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.008
tick 124 kimg 496.0 time 5h 23m 25s sec/tick 118.7 sec/kimg 29.68 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.007
tick 125 kimg 500.0 time 5h 25m 23s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.007
tick 126 kimg 504.0 time 5h 27m 22s sec/tick 118.6 sec/kimg 29.64 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.003
tick 127 kimg 508.0 time 5h 29m 20s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
tick 128 kimg 512.0 time 5h 31m 19s sec/tick 118.7 sec/kimg 29.66 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.003
tick 129 kimg 516.0 time 5h 33m 17s sec/tick 118.2 sec/kimg 29.56 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 130 kimg 520.0 time 5h 35m 16s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
tick 131 kimg 524.0 time 5h 37m 14s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
tick 132 kimg 528.0 time 5h 39m 12s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 133 kimg 532.0 time 5h 41m 11s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 134 kimg 536.0 time 5h 43m 10s sec/tick 118.6 sec/kimg 29.65 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.000
tick 135 kimg 540.0 time 5h 45m 08s sec/tick 118.4 sec/kimg 29.61 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
Evaluating metrics...
{"results": {"fid50k_full": 21.308802383348123}, "metric": "fid50k_full", "total_time": 251.03305459022522, "total_time_str": "4m 11s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000540.pkl", "timestamp": 1694668592.8447587}
{"results": {"kid50k_full": 0.01579188640890892}, "metric": "kid50k_full", "total_time": 236.91921520233154, "total_time_str": "3m 57s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000540.pkl", "timestamp": 1694668829.7642376}
tick 136 kimg 544.0 time 5h 55m 18s sec/tick 118.6 sec/kimg 29.64 maintenance 491.4 cpumem 5.05 gpumem 4.97 augment 0.002
tick 137 kimg 548.0 time 5h 57m 17s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 138 kimg 552.0 time 5h 59m 15s sec/tick 118.3 sec/kimg 29.59 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 139 kimg 556.0 time 6h 01m 13s sec/tick 118.3 sec/kimg 29.56 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 140 kimg 560.0 time 6h 03m 12s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.001
tick 141 kimg 564.0 time 6h 05m 10s sec/tick 118.2 sec/kimg 29.54 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.002
tick 142 kimg 568.0 time 6h 07m 09s sec/tick 118.6 sec/kimg 29.65 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.000
tick 143 kimg 572.0 time 6h 09m 07s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.000
tick 144 kimg 576.0 time 6h 11m 06s sec/tick 118.8 sec/kimg 29.71 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 145 kimg 580.0 time 6h 13m 05s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 146 kimg 584.0 time 6h 15m 03s sec/tick 118.5 sec/kimg 29.61 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 147 kimg 588.0 time 6h 17m 02s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 148 kimg 592.0 time 6h 19m 00s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 149 kimg 596.0 time 6h 20m 59s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 150 kimg 600.0 time 6h 22m 57s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
Evaluating metrics...
{"results": {"fid50k_full": 19.474795314716626}, "metric": "fid50k_full", "total_time": 249.82070064544678, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000600.pkl", "timestamp": 1694670860.4965193}
{"results": {"kid50k_full": 0.014893343260760753}, "metric": "kid50k_full", "total_time": 236.303875207901, "total_time_str": "3m 56s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000600.pkl", "timestamp": 1694671096.800674}
tick 151 kimg 604.0 time 6h 33m 05s sec/tick 118.4 sec/kimg 29.61 maintenance 489.5 cpumem 4.93 gpumem 4.97 augment 0.004
tick 152 kimg 608.0 time 6h 35m 04s sec/tick 118.6 sec/kimg 29.66 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.004
tick 153 kimg 612.0 time 6h 37m 02s sec/tick 118.3 sec/kimg 29.58 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.003
tick 154 kimg 616.0 time 6h 39m 01s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 155 kimg 620.0 time 6h 40m 59s sec/tick 118.3 sec/kimg 29.58 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.002
tick 156 kimg 624.0 time 6h 42m 58s sec/tick 118.8 sec/kimg 29.69 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.003
tick 157 kimg 628.0 time 6h 44m 56s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.003
tick 158 kimg 632.0 time 6h 46m 55s sec/tick 118.6 sec/kimg 29.65 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.000
tick 159 kimg 636.0 time 6h 48m 54s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
tick 160 kimg 640.0 time 6h 50m 52s sec/tick 118.6 sec/kimg 29.66 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.001
tick 161 kimg 644.0 time 6h 52m 51s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.000
tick 162 kimg 648.0 time 6h 54m 49s sec/tick 118.3 sec/kimg 29.57 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.000
tick 163 kimg 652.0 time 6h 56m 48s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.000
tick 164 kimg 656.0 time 6h 58m 46s sec/tick 118.3 sec/kimg 29.58 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 165 kimg 660.0 time 7h 00m 44s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.004
Evaluating metrics...
{"results": {"fid50k_full": 17.742923185581947}, "metric": "fid50k_full", "total_time": 250.11296224594116, "total_time_str": "4m 10s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000660.pkl", "timestamp": 1694673128.1005237}
{"results": {"kid50k_full": 0.013932279867367402}, "metric": "kid50k_full", "total_time": 236.46536302566528, "total_time_str": "3m 56s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000660.pkl", "timestamp": 1694673364.5661547}
tick 166 kimg 664.0 time 7h 10m 53s sec/tick 118.4 sec/kimg 29.59 maintenance 490.0 cpumem 4.93 gpumem 4.97 augment 0.001
tick 167 kimg 668.0 time 7h 12m 51s sec/tick 118.6 sec/kimg 29.66 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.001
tick 168 kimg 672.0 time 7h 14m 50s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
tick 169 kimg 676.0 time 7h 16m 48s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.001
tick 170 kimg 680.0 time 7h 18m 47s sec/tick 118.5 sec/kimg 29.63 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
tick 171 kimg 684.0 time 7h 20m 46s sec/tick 118.5 sec/kimg 29.62 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.000
tick 172 kimg 688.0 time 7h 22m 45s sec/tick 118.8 sec/kimg 29.71 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 173 kimg 692.0 time 7h 24m 43s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.000
tick 174 kimg 696.0 time 7h 26m 41s sec/tick 118.4 sec/kimg 29.61 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 175 kimg 700.0 time 7h 28m 40s sec/tick 118.1 sec/kimg 29.53 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.002
tick 176 kimg 704.0 time 7h 30m 38s sec/tick 118.6 sec/kimg 29.64 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.000
tick 177 kimg 708.0 time 7h 32m 37s sec/tick 118.4 sec/kimg 29.59 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.000
tick 178 kimg 712.0 time 7h 34m 35s sec/tick 118.5 sec/kimg 29.61 maintenance 0.1 cpumem 4.93 gpumem 4.96 augment 0.001
tick 179 kimg 716.0 time 7h 36m 34s sec/tick 118.5 sec/kimg 29.61 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.000
tick 180 kimg 720.0 time 7h 38m 32s sec/tick 118.3 sec/kimg 29.58 maintenance 0.1 cpumem 4.93 gpumem 4.97 augment 0.001
Evaluating metrics...
{"results": {"fid50k_full": 14.575906275298902}, "metric": "fid50k_full", "total_time": 250.86674618721008, "total_time_str": "4m 11s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000720.pkl", "timestamp": 1694675396.6673334}
{"results": {"kid50k_full": 0.010644822399899903}, "metric": "kid50k_full", "total_time": 236.76751255989075, "total_time_str": "3m 57s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000720.pkl", "timestamp": 1694675633.4351242}
tick 181 kimg 724.0 time 7h 48m 41s sec/tick 118.3 sec/kimg 29.56 maintenance 491.0 cpumem 5.05 gpumem 4.96 augment 0.000
tick 182 kimg 728.0 time 7h 50m 40s sec/tick 118.8 sec/kimg 29.71 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.000
tick 183 kimg 732.0 time 7h 52m 39s sec/tick 118.7 sec/kimg 29.67 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.001
tick 184 kimg 736.0 time 7h 54m 38s sec/tick 118.6 sec/kimg 29.66 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 185 kimg 740.0 time 7h 56m 36s sec/tick 118.4 sec/kimg 29.61 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 186 kimg 744.0 time 7h 58m 35s sec/tick 118.4 sec/kimg 29.60 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.001
tick 187 kimg 748.0 time 8h 00m 33s sec/tick 118.2 sec/kimg 29.55 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 188 kimg 752.0 time 8h 02m 32s sec/tick 118.8 sec/kimg 29.69 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 189 kimg 756.0 time 8h 04m 37s sec/tick 125.1 sec/kimg 31.28 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.000
tick 190 kimg 760.0 time 8h 06m 42s sec/tick 125.2 sec/kimg 31.29 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 191 kimg 764.0 time 8h 08m 48s sec/tick 125.7 sec/kimg 31.43 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.001
tick 192 kimg 768.0 time 8h 10m 54s sec/tick 125.3 sec/kimg 31.32 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 193 kimg 772.0 time 8h 12m 58s sec/tick 124.5 sec/kimg 31.12 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 194 kimg 776.0 time 8h 15m 04s sec/tick 125.6 sec/kimg 31.41 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 195 kimg 780.0 time 8h 17m 09s sec/tick 125.1 sec/kimg 31.27 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
Evaluating metrics...
{"results": {"fid50k_full": 14.727204616826384}, "metric": "fid50k_full", "total_time": 263.2710473537445, "total_time_str": "4m 23s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000780.pkl", "timestamp": 1694677725.919389}
{"results": {"kid50k_full": 0.01096397978478481}, "metric": "kid50k_full", "total_time": 248.44578909873962, "total_time_str": "4m 08s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000780.pkl", "timestamp": 1694677974.365463}
tick 196 kimg 784.0 time 8h 27m 49s sec/tick 125.0 sec/kimg 31.24 maintenance 515.2 cpumem 4.94 gpumem 4.97 augment 0.002
tick 197 kimg 788.0 time 8h 29m 54s sec/tick 124.7 sec/kimg 31.18 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.003
tick 198 kimg 792.0 time 8h 31m 58s sec/tick 124.4 sec/kimg 31.09 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.000
tick 199 kimg 796.0 time 8h 34m 03s sec/tick 124.5 sec/kimg 31.13 maintenance 0.1 cpumem 4.94 gpumem 4.96 augment 0.001
tick 200 kimg 800.0 time 8h 36m 09s sec/tick 125.7 sec/kimg 31.42 maintenance 0.1 cpumem 4.94 gpumem 4.96 augment 0.001
tick 201 kimg 804.0 time 8h 38m 13s sec/tick 124.3 sec/kimg 31.07 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.001
tick 202 kimg 808.0 time 8h 40m 17s sec/tick 124.3 sec/kimg 31.07 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.000
tick 203 kimg 812.0 time 8h 42m 23s sec/tick 125.2 sec/kimg 31.31 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.001
tick 204 kimg 816.0 time 8h 44m 28s sec/tick 125.0 sec/kimg 31.24 maintenance 0.1 cpumem 4.94 gpumem 4.96 augment 0.000
tick 205 kimg 820.0 time 8h 46m 32s sec/tick 124.0 sec/kimg 31.01 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.002
tick 206 kimg 824.0 time 8h 48m 37s sec/tick 124.7 sec/kimg 31.17 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.003
tick 207 kimg 828.0 time 8h 50m 42s sec/tick 125.3 sec/kimg 31.33 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.002
tick 208 kimg 832.0 time 8h 52m 47s sec/tick 124.4 sec/kimg 31.10 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.002
tick 209 kimg 836.0 time 8h 54m 51s sec/tick 124.3 sec/kimg 31.07 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.001
tick 210 kimg 840.0 time 8h 56m 57s sec/tick 125.6 sec/kimg 31.40 maintenance 0.1 cpumem 4.94 gpumem 4.96 augment 0.000
Evaluating metrics...
{"results": {"fid50k_full": 11.796006114092972}, "metric": "fid50k_full", "total_time": 261.69703674316406, "total_time_str": "4m 22s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000840.pkl", "timestamp": 1694680112.002921}
{"results": {"kid50k_full": 0.008275251468968983}, "metric": "kid50k_full", "total_time": 249.1113293170929, "total_time_str": "4m 09s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000840.pkl", "timestamp": 1694680361.1145208}
tick 211 kimg 844.0 time 9h 07m 35s sec/tick 124.2 sec/kimg 31.04 maintenance 514.3 cpumem 4.94 gpumem 4.96 augment 0.001
tick 212 kimg 848.0 time 9h 09m 41s sec/tick 125.4 sec/kimg 31.35 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.000
tick 213 kimg 852.0 time 9h 11m 46s sec/tick 125.2 sec/kimg 31.30 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.001
tick 214 kimg 856.0 time 9h 13m 50s sec/tick 124.3 sec/kimg 31.08 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.001
tick 215 kimg 860.0 time 9h 15m 55s sec/tick 125.2 sec/kimg 31.30 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.003
tick 216 kimg 864.0 time 9h 18m 01s sec/tick 125.5 sec/kimg 31.38 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.004
tick 217 kimg 868.0 time 9h 20m 05s sec/tick 123.9 sec/kimg 30.98 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.004
tick 218 kimg 872.0 time 9h 22m 10s sec/tick 125.0 sec/kimg 31.26 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.003
tick 219 kimg 876.0 time 9h 24m 15s sec/tick 125.2 sec/kimg 31.29 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.003
tick 220 kimg 880.0 time 9h 26m 20s sec/tick 124.7 sec/kimg 31.18 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.002
tick 221 kimg 884.0 time 9h 28m 25s sec/tick 124.3 sec/kimg 31.07 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.004
tick 222 kimg 888.0 time 9h 30m 30s sec/tick 125.2 sec/kimg 31.31 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.005
tick 223 kimg 892.0 time 9h 32m 35s sec/tick 125.1 sec/kimg 31.28 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.004
tick 224 kimg 896.0 time 9h 34m 40s sec/tick 124.4 sec/kimg 31.11 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.004
tick 225 kimg 900.0 time 9h 36m 45s sec/tick 125.1 sec/kimg 31.28 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.004
Evaluating metrics...
{"results": {"fid50k_full": 10.482408982956947}, "metric": "fid50k_full", "total_time": 261.8089416027069, "total_time_str": "4m 22s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000900.pkl", "timestamp": 1694682500.3033524}
{"results": {"kid50k_full": 0.0073491715640640765}, "metric": "kid50k_full", "total_time": 249.28055119514465, "total_time_str": "4m 09s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000900.pkl", "timestamp": 1694682749.584181}
tick 226 kimg 904.0 time 9h 47m 23s sec/tick 124.0 sec/kimg 31.01 maintenance 514.5 cpumem 5.05 gpumem 4.97 augment 0.004
tick 227 kimg 908.0 time 9h 49m 28s sec/tick 124.0 sec/kimg 31.01 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 228 kimg 912.0 time 9h 51m 33s sec/tick 125.6 sec/kimg 31.40 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.003
tick 229 kimg 916.0 time 9h 53m 37s sec/tick 124.1 sec/kimg 31.02 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 230 kimg 920.0 time 9h 55m 42s sec/tick 124.2 sec/kimg 31.04 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 231 kimg 924.0 time 9h 57m 47s sec/tick 125.3 sec/kimg 31.32 maintenance 0.1 cpumem 5.05 gpumem 4.96 augment 0.000
tick 232 kimg 928.0 time 9h 59m 51s sec/tick 124.4 sec/kimg 31.10 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 233 kimg 932.0 time 10h 01m 55s sec/tick 123.8 sec/kimg 30.94 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 234 kimg 936.0 time 10h 04m 00s sec/tick 124.6 sec/kimg 31.16 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.000
tick 235 kimg 940.0 time 10h 06m 05s sec/tick 124.7 sec/kimg 31.17 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 236 kimg 944.0 time 10h 08m 09s sec/tick 124.0 sec/kimg 30.99 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.001
tick 237 kimg 948.0 time 10h 10m 13s sec/tick 124.5 sec/kimg 31.12 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.002
tick 238 kimg 952.0 time 10h 12m 19s sec/tick 125.1 sec/kimg 31.28 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 239 kimg 956.0 time 10h 14m 23s sec/tick 124.1 sec/kimg 31.01 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
tick 240 kimg 960.0 time 10h 16m 27s sec/tick 124.5 sec/kimg 31.11 maintenance 0.1 cpumem 5.05 gpumem 4.97 augment 0.004
Evaluating metrics...
{"results": {"fid50k_full": 10.026805993782292}, "metric": "fid50k_full", "total_time": 260.7758901119232, "total_time_str": "4m 21s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000960.pkl", "timestamp": 1694684881.7145474}
{"results": {"kid50k_full": 0.0072147737312312525}, "metric": "kid50k_full", "total_time": 248.2493498325348, "total_time_str": "4m 08s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-000960.pkl", "timestamp": 1694685129.9641693}
tick 241 kimg 964.0 time 10h 27m 04s sec/tick 124.5 sec/kimg 31.12 maintenance 512.5 cpumem 4.94 gpumem 4.97 augment 0.006
tick 242 kimg 968.0 time 10h 29m 08s sec/tick 124.1 sec/kimg 31.02 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.007
tick 243 kimg 972.0 time 10h 31m 13s sec/tick 125.0 sec/kimg 31.25 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 244 kimg 976.0 time 10h 33m 19s sec/tick 125.4 sec/kimg 31.34 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.008
tick 245 kimg 980.0 time 10h 35m 23s sec/tick 124.3 sec/kimg 31.07 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.007
tick 246 kimg 984.0 time 10h 37m 28s sec/tick 124.8 sec/kimg 31.21 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.006
tick 247 kimg 988.0 time 10h 39m 33s sec/tick 124.9 sec/kimg 31.23 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.007
tick 248 kimg 992.0 time 10h 41m 38s sec/tick 124.6 sec/kimg 31.15 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.008
tick 249 kimg 996.0 time 10h 43m 42s sec/tick 124.0 sec/kimg 31.01 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.009
tick 250 kimg 1000.0 time 10h 45m 48s sec/tick 125.8 sec/kimg 31.45 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.009
tick 251 kimg 1004.0 time 10h 47m 53s sec/tick 124.5 sec/kimg 31.13 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.011
tick 252 kimg 1008.0 time 10h 49m 57s sec/tick 124.2 sec/kimg 31.06 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.009
tick 253 kimg 1012.0 time 10h 52m 02s sec/tick 125.4 sec/kimg 31.35 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 254 kimg 1016.0 time 10h 54m 07s sec/tick 124.8 sec/kimg 31.19 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.011
tick 255 kimg 1020.0 time 10h 56m 12s sec/tick 124.2 sec/kimg 31.06 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.011
Evaluating metrics...
{"results": {"fid50k_full": 8.745657893911403}, "metric": "fid50k_full", "total_time": 262.32836413383484, "total_time_str": "4m 22s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001020.pkl", "timestamp": 1694687267.5379338}
{"results": {"kid50k_full": 0.00554922357857858}, "metric": "kid50k_full", "total_time": 247.56754326820374, "total_time_str": "4m 08s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001020.pkl", "timestamp": 1694687515.1057637}
tick 256 kimg 1024.0 time 11h 06m 50s sec/tick 125.5 sec/kimg 31.37 maintenance 513.3 cpumem 4.94 gpumem 4.97 augment 0.013
tick 257 kimg 1028.0 time 11h 08m 55s sec/tick 124.0 sec/kimg 31.01 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.013
tick 258 kimg 1032.0 time 11h 10m 59s sec/tick 124.7 sec/kimg 31.17 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.014
tick 259 kimg 1036.0 time 11h 13m 05s sec/tick 125.3 sec/kimg 31.32 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.013
tick 260 kimg 1040.0 time 11h 15m 10s sec/tick 124.8 sec/kimg 31.21 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.014
tick 261 kimg 1044.0 time 11h 17m 14s sec/tick 124.3 sec/kimg 31.08 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.015
tick 262 kimg 1048.0 time 11h 19m 19s sec/tick 124.7 sec/kimg 31.18 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.014
tick 263 kimg 1052.0 time 11h 21m 24s sec/tick 125.1 sec/kimg 31.27 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.012
tick 264 kimg 1056.0 time 11h 23m 28s sec/tick 124.2 sec/kimg 31.05 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.012
tick 265 kimg 1060.0 time 11h 25m 33s sec/tick 124.7 sec/kimg 31.17 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 266 kimg 1064.0 time 11h 27m 39s sec/tick 125.4 sec/kimg 31.36 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 267 kimg 1068.0 time 11h 29m 42s sec/tick 123.6 sec/kimg 30.90 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 268 kimg 1072.0 time 11h 31m 47s sec/tick 124.7 sec/kimg 31.17 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 269 kimg 1076.0 time 11h 33m 52s sec/tick 125.0 sec/kimg 31.26 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.012
tick 270 kimg 1080.0 time 11h 35m 56s sec/tick 124.3 sec/kimg 31.08 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.012
Evaluating metrics...
{"results": {"fid50k_full": 8.780460742630755}, "metric": "fid50k_full", "total_time": 261.4188401699066, "total_time_str": "4m 21s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001080.pkl", "timestamp": 1694689651.5038993}
{"results": {"kid50k_full": 0.005656544454454474}, "metric": "kid50k_full", "total_time": 246.95897698402405, "total_time_str": "4m 07s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001080.pkl", "timestamp": 1694689898.463157}
tick 271 kimg 1084.0 time 11h 46m 33s sec/tick 124.7 sec/kimg 31.17 maintenance 511.8 cpumem 5.06 gpumem 4.97 augment 0.011
tick 272 kimg 1088.0 time 11h 48m 37s sec/tick 124.4 sec/kimg 31.09 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.012
tick 273 kimg 1092.0 time 11h 50m 41s sec/tick 123.5 sec/kimg 30.86 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.013
tick 274 kimg 1096.0 time 11h 52m 46s sec/tick 124.5 sec/kimg 31.14 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.013
tick 275 kimg 1100.0 time 11h 54m 50s sec/tick 124.4 sec/kimg 31.09 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.014
tick 276 kimg 1104.0 time 11h 56m 54s sec/tick 124.0 sec/kimg 31.00 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.011
tick 277 kimg 1108.0 time 11h 58m 58s sec/tick 123.5 sec/kimg 30.86 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.013
tick 278 kimg 1112.0 time 12h 01m 03s sec/tick 124.8 sec/kimg 31.20 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.011
tick 279 kimg 1116.0 time 12h 03m 07s sec/tick 123.9 sec/kimg 30.98 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.013
tick 280 kimg 1120.0 time 12h 05m 12s sec/tick 125.3 sec/kimg 31.34 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.014
tick 281 kimg 1124.0 time 12h 07m 20s sec/tick 128.0 sec/kimg 31.99 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.012
tick 282 kimg 1128.0 time 12h 09m 24s sec/tick 124.1 sec/kimg 31.02 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.012
tick 283 kimg 1132.0 time 12h 11m 28s sec/tick 123.7 sec/kimg 30.93 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.012
tick 284 kimg 1136.0 time 12h 13m 33s sec/tick 124.6 sec/kimg 31.16 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.011
tick 285 kimg 1140.0 time 12h 15m 38s sec/tick 124.9 sec/kimg 31.21 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.011
Evaluating metrics...
{"results": {"fid50k_full": 10.5000946189002}, "metric": "fid50k_full", "total_time": 261.2173035144806, "total_time_str": "4m 21s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001140.pkl", "timestamp": 1694692032.4746609}
{"results": {"kid50k_full": 0.007626752780280285}, "metric": "kid50k_full", "total_time": 248.74727129936218, "total_time_str": "4m 09s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001140.pkl", "timestamp": 1694692281.222202}
tick 286 kimg 1144.0 time 12h 26m 15s sec/tick 124.2 sec/kimg 31.04 maintenance 513.4 cpumem 4.94 gpumem 4.97 augment 0.009
tick 287 kimg 1148.0 time 12h 28m 20s sec/tick 125.2 sec/kimg 31.30 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 288 kimg 1152.0 time 12h 30m 25s sec/tick 124.3 sec/kimg 31.08 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.010
tick 289 kimg 1156.0 time 12h 32m 29s sec/tick 124.1 sec/kimg 31.03 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.011
tick 290 kimg 1160.0 time 12h 34m 34s sec/tick 125.2 sec/kimg 31.30 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.016
tick 291 kimg 1164.0 time 12h 36m 39s sec/tick 124.7 sec/kimg 31.17 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.018
tick 292 kimg 1168.0 time 12h 38m 43s sec/tick 124.2 sec/kimg 31.05 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.018
tick 293 kimg 1172.0 time 12h 40m 48s sec/tick 125.0 sec/kimg 31.25 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.020
tick 294 kimg 1176.0 time 12h 42m 54s sec/tick 125.2 sec/kimg 31.31 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.021
tick 295 kimg 1180.0 time 12h 44m 58s sec/tick 124.1 sec/kimg 31.01 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.020
tick 296 kimg 1184.0 time 12h 47m 03s sec/tick 125.0 sec/kimg 31.26 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.021
tick 297 kimg 1188.0 time 12h 49m 08s sec/tick 125.0 sec/kimg 31.24 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.019
tick 298 kimg 1192.0 time 12h 51m 13s sec/tick 125.0 sec/kimg 31.24 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.019
tick 299 kimg 1196.0 time 12h 53m 17s sec/tick 124.1 sec/kimg 31.03 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.020
tick 300 kimg 1200.0 time 12h 55m 23s sec/tick 125.5 sec/kimg 31.37 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.019
Evaluating metrics...
{"results": {"fid50k_full": 10.557362148535166}, "metric": "fid50k_full", "total_time": 262.5542676448822, "total_time_str": "4m 23s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001200.pkl", "timestamp": 1694694419.0747936}
{"results": {"kid50k_full": 0.007641803688688688}, "metric": "kid50k_full", "total_time": 249.22258710861206, "total_time_str": "4m 09s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001200.pkl", "timestamp": 1694694668.2976696}
tick 301 kimg 1204.0 time 13h 06m 02s sec/tick 124.2 sec/kimg 31.04 maintenance 515.2 cpumem 4.99 gpumem 4.98 augment 0.018
tick 302 kimg 1208.0 time 13h 08m 07s sec/tick 125.0 sec/kimg 31.24 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.017
tick 303 kimg 1212.0 time 13h 10m 12s sec/tick 124.8 sec/kimg 31.20 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.017
tick 304 kimg 1216.0 time 13h 12m 17s sec/tick 124.6 sec/kimg 31.15 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.015
tick 305 kimg 1220.0 time 13h 14m 21s sec/tick 124.3 sec/kimg 31.08 maintenance 0.1 cpumem 4.99 gpumem 4.98 augment 0.014
tick 306 kimg 1224.0 time 13h 16m 27s sec/tick 125.7 sec/kimg 31.43 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.017
tick 307 kimg 1228.0 time 13h 18m 31s sec/tick 124.1 sec/kimg 31.04 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.015
tick 308 kimg 1232.0 time 13h 20m 36s sec/tick 124.7 sec/kimg 31.17 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.015
tick 309 kimg 1236.0 time 13h 22m 42s sec/tick 125.6 sec/kimg 31.39 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.015
tick 310 kimg 1240.0 time 13h 24m 46s sec/tick 124.6 sec/kimg 31.14 maintenance 0.1 cpumem 4.99 gpumem 4.98 augment 0.014
tick 311 kimg 1244.0 time 13h 26m 51s sec/tick 124.3 sec/kimg 31.08 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.014
tick 312 kimg 1248.0 time 13h 28m 56s sec/tick 125.1 sec/kimg 31.26 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.020
tick 313 kimg 1252.0 time 13h 31m 01s sec/tick 124.8 sec/kimg 31.20 maintenance 0.1 cpumem 4.99 gpumem 4.98 augment 0.022
tick 314 kimg 1256.0 time 13h 33m 05s sec/tick 124.5 sec/kimg 31.13 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.023
tick 315 kimg 1260.0 time 13h 35m 10s sec/tick 124.6 sec/kimg 31.16 maintenance 0.1 cpumem 4.99 gpumem 4.97 augment 0.023
Evaluating metrics...
{"results": {"fid50k_full": 8.515544360031178}, "metric": "fid50k_full", "total_time": 265.43890023231506, "total_time_str": "4m 25s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001260.pkl", "timestamp": 1694696809.1560202}
{"results": {"kid50k_full": 0.005208328668668682}, "metric": "kid50k_full", "total_time": 251.16270399093628, "total_time_str": "4m 11s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001260.pkl", "timestamp": 1694697060.319028}
tick 316 kimg 1264.0 time 13h 45m 55s sec/tick 124.9 sec/kimg 31.24 maintenance 520.1 cpumem 5.06 gpumem 5.00 augment 0.023
tick 317 kimg 1268.0 time 13h 48m 00s sec/tick 124.8 sec/kimg 31.20 maintenance 0.1 cpumem 5.06 gpumem 4.99 augment 0.025
tick 318 kimg 1272.0 time 13h 50m 05s sec/tick 125.2 sec/kimg 31.31 maintenance 0.1 cpumem 5.06 gpumem 4.98 augment 0.024
tick 319 kimg 1276.0 time 13h 52m 12s sec/tick 126.2 sec/kimg 31.56 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.021
tick 320 kimg 1280.0 time 13h 54m 16s sec/tick 124.3 sec/kimg 31.08 maintenance 0.1 cpumem 5.06 gpumem 4.98 augment 0.022
tick 321 kimg 1284.0 time 13h 56m 21s sec/tick 125.1 sec/kimg 31.27 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.020
tick 322 kimg 1288.0 time 13h 58m 26s sec/tick 125.2 sec/kimg 31.31 maintenance 0.1 cpumem 5.06 gpumem 4.98 augment 0.021
tick 323 kimg 1292.0 time 14h 00m 31s sec/tick 124.4 sec/kimg 31.11 maintenance 0.1 cpumem 5.06 gpumem 4.98 augment 0.021
tick 324 kimg 1296.0 time 14h 02m 35s sec/tick 124.0 sec/kimg 31.00 maintenance 0.1 cpumem 5.06 gpumem 4.98 augment 0.023
tick 325 kimg 1300.0 time 14h 04m 43s sec/tick 127.6 sec/kimg 31.90 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.024
tick 326 kimg 1304.0 time 14h 06m 51s sec/tick 128.4 sec/kimg 32.09 maintenance 0.1 cpumem 5.06 gpumem 4.98 augment 0.025
tick 327 kimg 1308.0 time 14h 08m 56s sec/tick 125.0 sec/kimg 31.26 maintenance 0.1 cpumem 5.06 gpumem 4.98 augment 0.027
tick 328 kimg 1312.0 time 14h 11m 07s sec/tick 130.7 sec/kimg 32.68 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.025
tick 329 kimg 1316.0 time 14h 13m 16s sec/tick 129.1 sec/kimg 32.27 maintenance 0.1 cpumem 5.06 gpumem 4.99 augment 0.026
tick 330 kimg 1320.0 time 14h 15m 26s sec/tick 129.5 sec/kimg 32.37 maintenance 0.1 cpumem 5.06 gpumem 4.97 augment 0.027
Evaluating metrics...
{"results": {"fid50k_full": 8.767082366260123}, "metric": "fid50k_full", "total_time": 275.49366569519043, "total_time_str": "4m 35s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001320.pkl", "timestamp": 1694699235.1012852}
{"results": {"kid50k_full": 0.005630771308808789}, "metric": "kid50k_full", "total_time": 251.07139682769775, "total_time_str": "4m 11s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001320.pkl", "timestamp": 1694699486.172953}
tick 331 kimg 1324.0 time 14h 26m 19s sec/tick 123.5 sec/kimg 30.88 maintenance 530.2 cpumem 4.94 gpumem 4.97 augment 0.028
tick 332 kimg 1328.0 time 14h 28m 21s sec/tick 121.1 sec/kimg 30.28 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.026
tick 333 kimg 1332.0 time 14h 30m 24s sec/tick 123.2 sec/kimg 30.80 maintenance 0.1 cpumem 4.94 gpumem 4.99 augment 0.026
tick 334 kimg 1336.0 time 14h 32m 28s sec/tick 124.4 sec/kimg 31.09 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.028
tick 335 kimg 1340.0 time 14h 34m 32s sec/tick 123.3 sec/kimg 30.82 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.027
tick 336 kimg 1344.0 time 14h 36m 33s sec/tick 121.3 sec/kimg 30.31 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.031
tick 337 kimg 1348.0 time 14h 38m 34s sec/tick 120.5 sec/kimg 30.13 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.033
tick 338 kimg 1352.0 time 14h 40m 37s sec/tick 123.7 sec/kimg 30.93 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.034
tick 339 kimg 1356.0 time 14h 42m 48s sec/tick 130.6 sec/kimg 32.65 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.033
tick 340 kimg 1360.0 time 14h 44m 55s sec/tick 126.4 sec/kimg 31.60 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.033
tick 341 kimg 1364.0 time 14h 47m 00s sec/tick 125.5 sec/kimg 31.38 maintenance 0.1 cpumem 4.94 gpumem 4.97 augment 0.033
tick 342 kimg 1368.0 time 14h 49m 07s sec/tick 127.0 sec/kimg 31.76 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.035
tick 343 kimg 1372.0 time 14h 51m 13s sec/tick 125.4 sec/kimg 31.34 maintenance 0.1 cpumem 4.94 gpumem 5.00 augment 0.035
tick 344 kimg 1376.0 time 14h 53m 18s sec/tick 125.1 sec/kimg 31.28 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.033
tick 345 kimg 1380.0 time 14h 55m 22s sec/tick 124.3 sec/kimg 31.07 maintenance 0.1 cpumem 4.94 gpumem 4.98 augment 0.033
Evaluating metrics...
{"results": {"fid50k_full": 9.515935129334032}, "metric": "fid50k_full", "total_time": 262.34165048599243, "total_time_str": "4m 22s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001380.pkl", "timestamp": 1694701618.4407206}
{"results": {"kid50k_full": 0.006682737022022029}, "metric": "kid50k_full", "total_time": 249.27070689201355, "total_time_str": "4m 09s", "num_gpus": 1, "snapshot_pkl": "network-snapshot-001380.pkl", "timestamp": 1694701867.7117002}
tick 346 kimg 1384.0 time 15h 06m 03s sec/tick 125.5 sec/kimg 31.38 maintenance 515.1 cpumem 4.99 gpumem 4.97 augment 0.032
tick 347 kimg 1388.0 time 15h 08m 08s sec/tick 125.1 sec/kimg 31.27 maintenance 0.1 cpumem 4.99 gpumem 5.02 augment 0.035
Aborted!
|