Merge pull request #2478 from bmaltais/dev

v24.1.3
bmaltais · May 9, 2024 · 8316941 · 8316941
2 parents e2206e0 + dd083f9
commit 8316941
Show file tree

Hide file tree

Showing 23 changed files with 67 additions and 55 deletions.
diff --git a/.release b/.release
@@ -1 +1 @@
-v24.1.2
+v24.1.3
diff --git a/README.md b/README.md
@@ -47,8 +47,6 @@ The GUI allows you to set the training parameters and generate and run the requi
   - [SDXL training](#sdxl-training)
   - [Masked loss](#masked-loss)
   - [Change History](#change-history)
-    - [v24.1.1](#v2411)
-    - [v24.1.0](#v2410)
 
 ## 🦒 Colab
 
@@ -442,14 +440,4 @@ ControlNet dataset is used to specify the mask. The mask images should be the RG
 
 ## Change History
 
-### v24.1.1
-
-- Fix small issue with VAE file path validation
-
-### v24.1.0
-
-- To ensure cross-platform compatibility and security, the GUI now defaults to using "shell=False" when running subprocesses. This is based on documentation and should not cause issues on most platforms. However, some users have reported issues on specific platforms such as runpod and colab. PLease open an issue if you encounter any issues.
-- Add support for custom LyCORIS toml config files. Simply type the path to the config file in the LyCORIS preset dropdown.
-- Improve files and folders validation
-- Added a new setup-3.10.bat file to set the venv to specifically use python 3.10.x instead of the default python the system might use.
-- Relocate toml training config file to same folder as the model output directory.
+See release information.
diff --git a/kohya_gui/basic_caption_gui.py b/kohya_gui/basic_caption_gui.py
@@ -85,7 +85,7 @@ def caption_images(
         log.info(f"Executing command: {command_to_run}")
 
         # Set the environment variable for the Python path
-        env = setup_environment(scriptdir=scriptdir)
+        env = setup_environment()
 
         # Run the command in the sd-scripts folder context
         subprocess.run(run_cmd, env=env, shell=False)

diff --git a/kohya_gui/blip_caption_gui.py b/kohya_gui/blip_caption_gui.py
@@ -87,7 +87,7 @@ def caption_images(
     )
 
     # Set up the environment
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/class_tensorboard.py b/kohya_gui/class_tensorboard.py
@@ -73,7 +73,7 @@ def start_tensorboard(self, logging_dir=None):
 
         self.log.info("Starting TensorBoard on port {}".format(self.tensorboard_port))
         try:
-            env = setup_environment(scriptdir=scriptdir)
+            env = setup_environment()
             self.tensorboard_proc = subprocess.Popen(run_cmd, env=env)
         except Exception as e:
             self.log.error("Failed to start Tensorboard:", e)

diff --git a/kohya_gui/common_gui.py b/kohya_gui/common_gui.py
@@ -1368,17 +1368,21 @@ def validate_file_path(file_path: str) -> bool:
     return True
 
 
-def validate_folder_path(folder_path: str, can_be_written_to: bool = False) -> bool:
+def validate_folder_path(folder_path: str, can_be_written_to: bool = False, create_if_not_exists: bool = False) -> bool:
     if folder_path == "":
         return True
     msg = f"Validating {folder_path} existence{' and writability' if can_be_written_to else ''}..."
     if not os.path.isdir(folder_path):
-        log.error(f"{msg} FAILED: does not exist")
-        return False
-    if can_be_written_to:
-        if not os.access(folder_path, os.W_OK):
-            log.error(f"{msg} FAILED: is not writable.")
+        if create_if_not_exists:
+            os.makedirs(folder_path)
+            log.info(f"{msg} SUCCESS")
+            return True
+        else:
+            log.error(f"{msg} FAILED: does not exist")
             return False
+    if can_be_written_to and not os.access(folder_path, os.W_OK):
+        log.error(f"{msg} FAILED: is not writable.")
+        return False
     log.info(f"{msg} SUCCESS")
     return True
 
@@ -1485,7 +1489,7 @@ def validate_args_setting(input_string):
         )
         return False
 
-def setup_environment(scriptdir: str):
+def setup_environment():
     env = os.environ.copy()
     env["PYTHONPATH"] = (
         fr"{scriptdir}{os.pathsep}{scriptdir}/sd-scripts{os.pathsep}{env.get('PYTHONPATH', '')}"

diff --git a/kohya_gui/convert_lcm_gui.py b/kohya_gui/convert_lcm_gui.py
@@ -62,7 +62,7 @@ def convert_lcm(
         run_cmd.append("--ssd-1b")
 
     # Set up the environment
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/convert_model_gui.py b/kohya_gui/convert_model_gui.py
@@ -99,7 +99,7 @@ def convert_model(
     # Log the command
     log.info(" ".join(run_cmd))
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Run the command
     subprocess.run(run_cmd, env=env, shell=False)

diff --git a/kohya_gui/dreambooth_gui.py b/kohya_gui/dreambooth_gui.py
@@ -524,10 +524,10 @@ def train_model(
     if not validate_file_path(log_tracker_config):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(logging_dir, can_be_written_to=True):
+    if not validate_folder_path(logging_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(output_dir, can_be_written_to=True):
+    if not validate_folder_path(output_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
     if not validate_model_path(pretrained_model_name_or_path):
@@ -677,7 +677,12 @@ def train_model(
     log.info(max_train_steps_info)
     log.info(f"lr_warmup_steps = {lr_warmup_steps}")
 
-    run_cmd = [rf'{get_executable_path("accelerate")}', "launch"]
+    accelerate_path = get_executable_path("accelerate")
+    if accelerate_path == "":
+        log.error("accelerate not found")
+        return TRAIN_BUTTON_VISIBLE
+
+    run_cmd = [rf'{accelerate_path}', "launch"]
 
     run_cmd = AccelerateLaunch.run_cmd(
         run_cmd=run_cmd,
@@ -899,7 +904,7 @@ def train_model(
 
         # log.info(run_cmd)
 
-        env = setup_environment(scriptdir=scriptdir)
+        env = setup_environment()
 
         # Run the command
 

diff --git a/kohya_gui/extract_lora_from_dylora_gui.py b/kohya_gui/extract_lora_from_dylora_gui.py
@@ -59,7 +59,7 @@ def extract_dylora(
         str(unit),
     ]
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/extract_lora_gui.py b/kohya_gui/extract_lora_gui.py
@@ -109,7 +109,7 @@ def extract_lora(
         run_cmd.append("--load_tuned_model_to")
         run_cmd.append(load_tuned_model_to)
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/extract_lycoris_locon_gui.py b/kohya_gui/extract_lycoris_locon_gui.py
@@ -123,7 +123,7 @@ def extract_lycoris_locon(
     run_cmd.append(fr"{db_model}")
     run_cmd.append(fr"{output_name}")
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/finetune_gui.py b/kohya_gui/finetune_gui.py
@@ -569,10 +569,10 @@ def train_model(
     if not validate_file_path(log_tracker_config):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(logging_dir, can_be_written_to=True):
+    if not validate_folder_path(logging_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(output_dir, can_be_written_to=True):
+    if not validate_folder_path(output_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
     if not validate_model_path(pretrained_model_name_or_path):
@@ -639,7 +639,7 @@ def train_model(
             log.info(" ".join(run_cmd))
 
             # Prepare environment variables
-            env = setup_environment(scriptdir=scriptdir)
+            env = setup_environment()
 
         # create images buckets
         if generate_image_buckets:
@@ -677,7 +677,7 @@ def train_model(
             log.info(" ".join(run_cmd))
 
             # Copy and modify environment variables
-            env = setup_environment(scriptdir=scriptdir)
+            env = setup_environment()
 
             # Execute the command if not just for printing
             if not print_only:
@@ -729,7 +729,12 @@ def train_model(
         lr_warmup_steps = 0
     log.info(f"lr_warmup_steps = {lr_warmup_steps}")
 
-    run_cmd = [rf'{get_executable_path("accelerate")}', "launch"]
+    accelerate_path = get_executable_path("accelerate")
+    if accelerate_path == "":
+        log.error("accelerate not found")
+        return TRAIN_BUTTON_VISIBLE
+
+    run_cmd = [rf'{accelerate_path}', "launch"]
 
     run_cmd = AccelerateLaunch.run_cmd(
         run_cmd=run_cmd,
@@ -953,7 +958,7 @@ def train_model(
 
         # log.info(run_cmd)
 
-        env = setup_environment(scriptdir=scriptdir)
+        env = setup_environment()
 
         # Run the command
         executor.execute_command(run_cmd=run_cmd, env=env)

diff --git a/kohya_gui/git_caption_gui.py b/kohya_gui/git_caption_gui.py
@@ -58,7 +58,7 @@ def caption_images(
     # Add the directory containing the training data
     run_cmd.append(fr"{train_data_dir}")
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/group_images_gui.py b/kohya_gui/group_images_gui.py
@@ -51,7 +51,7 @@ def group_images(
             run_cmd.append("--caption_ext")
             run_cmd.append(caption_ext)
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/lora_gui.py b/kohya_gui/lora_gui.py
@@ -707,7 +707,7 @@ def train_model(
     if not validate_file_path(log_tracker_config):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(logging_dir, can_be_written_to=True):
+    if not validate_folder_path(logging_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
     if LyCORIS_preset not in LYCORIS_PRESETS_CHOICES:
@@ -717,7 +717,7 @@ def train_model(
     if not validate_file_path(network_weights):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(output_dir, can_be_written_to=True):
+    if not validate_folder_path(output_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
     if not validate_model_path(pretrained_model_name_or_path):
@@ -920,7 +920,12 @@ def train_model(
     log.info(f"stop_text_encoder_training = {stop_text_encoder_training}")
     log.info(f"lr_warmup_steps = {lr_warmup_steps}")
 
-    run_cmd = [rf'{get_executable_path("accelerate")}', "launch"]
+    accelerate_path = get_executable_path("accelerate")
+    if accelerate_path == "":
+        log.error("accelerate not found")
+        return TRAIN_BUTTON_VISIBLE
+
+    run_cmd = [rf'{accelerate_path}', "launch"]
 
     run_cmd = AccelerateLaunch.run_cmd(
         run_cmd=run_cmd,
@@ -1282,7 +1287,7 @@ def train_model(
         )
 
         # log.info(run_cmd)
-        env = setup_environment(scriptdir=scriptdir)
+        env = setup_environment()
 
         # Run the command
 

diff --git a/kohya_gui/merge_lora_gui.py b/kohya_gui/merge_lora_gui.py
@@ -451,7 +451,7 @@ def merge_lora(
                 map(str, valid_ratios)
             )  # Convert ratios to strings and include them as separate arguments
 
-        env = setup_environment(scriptdir=scriptdir)
+        env = setup_environment()
 
         # Reconstruct the safe command string for display
         command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/merge_lycoris_gui.py b/kohya_gui/merge_lycoris_gui.py
@@ -59,7 +59,7 @@ def merge_lycoris(
         run_cmd.append("--is_v2")
 
     # Copy and update the environment variables
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/resize_lora_gui.py b/kohya_gui/resize_lora_gui.py
@@ -90,7 +90,7 @@ def resize_lora(
     if verbose:
         run_cmd.append("--verbose")
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)

diff --git a/kohya_gui/svd_merge_lora_gui.py b/kohya_gui/svd_merge_lora_gui.py
@@ -95,7 +95,7 @@ def add_model(model_path, ratio):
     # Log the command
     log.info(" ".join(run_cmd))
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Run the command
     subprocess.run(run_cmd, env=env)

diff --git a/kohya_gui/textual_inversion_gui.py b/kohya_gui/textual_inversion_gui.py
@@ -524,10 +524,10 @@ def train_model(
     if not validate_file_path(log_tracker_config):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(logging_dir, can_be_written_to=True):
+    if not validate_folder_path(logging_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
-    if not validate_folder_path(output_dir, can_be_written_to=True):
+    if not validate_folder_path(output_dir, can_be_written_to=True, create_if_not_exists=True):
         return TRAIN_BUTTON_VISIBLE
 
     if not validate_model_path(pretrained_model_name_or_path):
@@ -703,7 +703,12 @@ def train_model(
     log.info(f"stop_text_encoder_training = {stop_text_encoder_training}")
     log.info(f"lr_warmup_steps = {lr_warmup_steps}")
 
-    run_cmd = [rf'{get_executable_path("accelerate")}', "launch"]
+    accelerate_path = get_executable_path("accelerate")
+    if accelerate_path == "":
+        log.error("accelerate not found")
+        return TRAIN_BUTTON_VISIBLE
+
+    run_cmd = [rf'{accelerate_path}', "launch"]
 
     run_cmd = AccelerateLaunch.run_cmd(
         run_cmd=run_cmd,
@@ -916,7 +921,7 @@ def train_model(
             exclusion=["file_path", "save_as", "headless", "print_only"],
         )
 
-        env = setup_environment(scriptdir=scriptdir)
+        env = setup_environment()
 
         # Run the command
 

diff --git a/kohya_gui/verify_lora_gui.py b/kohya_gui/verify_lora_gui.py
@@ -46,7 +46,7 @@ def verify_lora(
     log.info(f"Executing command: {command_to_run}")
 
     # Set the environment variable for the Python path
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Run the command using subprocess.Popen for asynchronous handling
     process = subprocess.Popen(

diff --git a/kohya_gui/wd14_caption_gui.py b/kohya_gui/wd14_caption_gui.py
@@ -117,7 +117,7 @@ def caption_images(
     # Add the directory containing the training data
     run_cmd.append(rf"{train_data_dir}")
 
-    env = setup_environment(scriptdir=scriptdir)
+    env = setup_environment()
 
     # Reconstruct the safe command string for display
     command_to_run = " ".join(run_cmd)