You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
Re-Tested. It was this commit that broke training samples: f7e680b
Resulting in error:
`INFO:main:height: 640
INFO:main:width: 512
INFO:main:frame count: 25
INFO:main:sample steps: 25
INFO:main:guidance scale: 6.0
INFO:main:discrete flow shift: 12.0
Sampling timesteps for prompt 1: 0%| | 0/25 [00:00<?, ?it/s]
Traceback (most recent call last): 0%| | 0/25 [00:00<?, ?it/s]
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 2462, in
trainer.train(args)
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 1780, in train
sample_images(accelerator, args, None, global_step, vae, transformer, sample_parameters, dit_dtype)
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 374, in sample_images
sample_image_inference(accelerator, args, transformer, dit_dtype, vae, save_dir, sample_parameter, epoch, steps)
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 532, in sample_image_inference
if image_latents is not None:
UnboundLocalError: local variable 'image_latents' referenced before assignment
steps: 1%| | 10/1400 [01:46<4:06:22, 10.64s/it, Average key norm=0.000441, Keys Scaled=0, avr_loss=0.165]
Traceback (most recent call last):
File "C:\Program Files\Python310\lib\runpy.py", line 196, in run_module_as_main
return run_code(code, main_globals, None,
File "C:\Program Files\Python310\lib\runpy.py", line 86, in run_code
exec(code, run_globals)
File "D:\ML\SD\trainers\musubi-latest\venv\Scripts\accelerate.exe_main.py", line 7, in
File "D:\ML\SD\trainers\musubi-latest\venv\lib\site-packages\accelerate\commands\accelerate_cli.py", line 48, in main
args.func(args)
File "D:\ML\SD\trainers\musubi-latest\venv\lib\site-packages\accelerate\commands\launch.py", line 1172, in launch_command
simple_launcher(args)
File "D:\ML\SD\trainers\musubi-latest\venv\lib\site-packages\accelerate\commands\launch.py", line 762, in simple_launcher
raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
subprocess.CalledProcessError: Command '['D:\ML\SD\trainers\musubi-latest\venv\Scripts\python.exe', 'hv_train_network.py', '--config_file', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23\config.toml', '--dataset_config', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23\test_mixset_v2_aspect_no2_448.toml', '--sdpa', '--split_attn', '--mixed_precision', 'bf16', '--gradient_checkpointing', '--max_data_loader_n_workers', '8', '--persistent_data_loader_workers', '--output_dir', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23', '--output_name', '2025-02-27_1805_test_mixset_v2_aspect_no2_448', '--logging_dir', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23/_logs', '--vae_chunk_size', '32', '--vae_spatial_tile_sample_min_size', '128', '--sample_prompts', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23\prompt.txt']' returned non-zero exit status 1.`
The text was updated successfully, but these errors were encountered:
Re-Tested. It was this commit that broke training samples:
f7e680b
Resulting in error:
`INFO:main:height: 640
INFO:main:width: 512
INFO:main:frame count: 25
INFO:main:sample steps: 25
INFO:main:guidance scale: 6.0
INFO:main:discrete flow shift: 12.0
Sampling timesteps for prompt 1: 0%| | 0/25 [00:00<?, ?it/s]
Traceback (most recent call last): 0%| | 0/25 [00:00<?, ?it/s]
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 2462, in
trainer.train(args)
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 1780, in train
sample_images(accelerator, args, None, global_step, vae, transformer, sample_parameters, dit_dtype)
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 374, in sample_images
sample_image_inference(accelerator, args, transformer, dit_dtype, vae, save_dir, sample_parameter, epoch, steps)
File "D:\ML\SD\trainers\musubi-latest\hv_train_network.py", line 532, in sample_image_inference
if image_latents is not None:
UnboundLocalError: local variable 'image_latents' referenced before assignment
steps: 1%| | 10/1400 [01:46<4:06:22, 10.64s/it, Average key norm=0.000441, Keys Scaled=0, avr_loss=0.165]
Traceback (most recent call last):
File "C:\Program Files\Python310\lib\runpy.py", line 196, in run_module_as_main
return run_code(code, main_globals, None,
File "C:\Program Files\Python310\lib\runpy.py", line 86, in run_code
exec(code, run_globals)
File "D:\ML\SD\trainers\musubi-latest\venv\Scripts\accelerate.exe_main.py", line 7, in
File "D:\ML\SD\trainers\musubi-latest\venv\lib\site-packages\accelerate\commands\accelerate_cli.py", line 48, in main
args.func(args)
File "D:\ML\SD\trainers\musubi-latest\venv\lib\site-packages\accelerate\commands\launch.py", line 1172, in launch_command
simple_launcher(args)
File "D:\ML\SD\trainers\musubi-latest\venv\lib\site-packages\accelerate\commands\launch.py", line 762, in simple_launcher
raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
subprocess.CalledProcessError: Command '['D:\ML\SD\trainers\musubi-latest\venv\Scripts\python.exe', 'hv_train_network.py', '--config_file', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23\config.toml', '--dataset_config', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23\test_mixset_v2_aspect_no2_448.toml', '--sdpa', '--split_attn', '--mixed_precision', 'bf16', '--gradient_checkpointing', '--max_data_loader_n_workers', '8', '--persistent_data_loader_workers', '--output_dir', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23', '--output_name', '2025-02-27_1805_test_mixset_v2_aspect_no2_448', '--logging_dir', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23/_logs', '--vae_chunk_size', '32', '--vae_spatial_tile_sample_min_size', '128', '--sample_prompts', 'D:\ML\SD\workspace\musubi\2025-02-24\test_v23\prompt.txt']' returned non-zero exit status 1.`
The text was updated successfully, but these errors were encountered: