PuzzleTuning/utils/Experiment_script_helper.py

"""
Experimental Script Generator    Script  ver： Oct 5th 16:30

for linux servers

todo fix train and test alternatively
"""
import argparse
import os.path


def zero_trans_mystrlr_to_float(in_str):
    # EG: '305' -> 0.0005
    front = '0.'
    num_of_zero = int(in_str[0])
    end = in_str[-1]
    for i in range(num_of_zero):
        front = front + '0'
    front = front + end

    out_float = float(front)

    return out_float


def zero_trans_floatlr_to_mystrlr(in_float):
    # EG: 0.0005 -> '305'
    in_string = "%.20f" % in_float
    zero_counts = 0

    for i in range(len(in_string) - 2):
        # print(string[i+2])
        if in_string[i + 2] == '0':
            zero_counts += 1
        else:
            cut = i
            break

    trans_output = str(zero_counts) + '0' + in_string[(cut + 2):]

    last_zeros = 0
    for i in trans_output[::-1]:
        if i == '0':
            last_zeros += 1
        else:
            break
    trans_output = trans_output[0:0 - last_zeros]

    return trans_output


def remove_nohup_ignoring_input_at_first_line(directory='./'):
    """
    read the .sh files at the directory, remove the first line if it's 'nohup: ignoring input\n'
    """
    for root, _, files in os.walk(directory):
        for file_name in files:
            if file_name.endswith(".sh"):
                file_path = os.path.join(root, file_name)

                with open(file_path, 'r') as file:
                    lines = file.readlines()
                    # print(lines)

                modified_lines = [line for line in lines if line != "nohup: ignoring input\n"]
                with open(file_path, 'w') as file:
                    file.writelines(modified_lines)

                print('file_path:', file_path, 'has been cleaned')


def concatenate_the_lines_from_several_files(directory='./', cat_file='0.sh'):
    cat_file_path = os.path.join(directory, cat_file)
    all_lines = ["#!/bin/sh\n", ]

    for root, _, files in os.walk(directory):
        for file_name in files:
            if file_name.endswith(".sh"):
                file_path = os.path.join(root, file_name)

                with open(file_path, 'r') as file:
                    lines = file.readlines()
                    # print(lines)

                modified_lines = [line for line in lines if line != "#!/bin/sh\n"]
                all_lines.extend(modified_lines)
                print('file_path:', file_path, 'has taken')

    with open(cat_file_path, 'w') as file:
        file.writelines(all_lines)


def print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_Token_num=20,
                                           Prompt_input=False):
    Pre_Trained_model_path = os.path.join(Pre_Trained_model_path_PATH, model_weight_name)
    VPT_backbone_model_path = os.path.join(Pre_Trained_model_path_PATH, 'ViT_b16_224_Imagenet.pth')
    if not Prompt_input:
        # send a ViT model inside and then do the ViT + finetuning;
        # In VPT versions: we build VPT backbone with the ViT weight, then do finetuning and prompting
        # ViT + finetuning
        print(
            'python Train.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --lr ' + lr + ' --lrf ' + lrf + ' --enable_tensorboard --model_idx ViT_base_' + model_weight_idx + '_'
            + lr_mystr + '_lf' + lrf_mystr + '_finetuning_' + dataset_name + '_CLS --dataroot ' + str(dataroot)
            + ' --draw_root ' + draw_root + ' --Pre_Trained_model_path ' + Pre_Trained_model_path
            + ' --model_path ' + save_model_PATH)
        print(
            'python Test.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --model_idx ViT_base_' + model_weight_idx + '_' + lr_mystr + '_lf' + lrf_mystr + '_finetuning_'
            + dataset_name + '_CLS --dataroot ' + str(dataroot) + ' --draw_root ' + draw_root + ' --model_path '
            + save_model_PATH)
        # VPT + prompting
        print(
            'python Train.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --lr ' + lr + ' --lrf ' + lrf + ' --enable_tensorboard --model_idx ViT_base_' + model_weight_idx
            + '_PromptDeep_' + str(Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr + '_prompting_' + dataset_name
            + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(Prompt_Token_num) + ' --dataroot ' + str(
                dataroot) + ' --draw_root ' + draw_root
            + ' --Pre_Trained_model_path ' + Pre_Trained_model_path + ' --model_path ' + save_model_PATH)
        print(
            'python Test.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --model_idx ViT_base_' + model_weight_idx + '_PromptDeep_' + str(
                Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr
            + '_prompting_' + dataset_name + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(
                Prompt_Token_num) + ' --dataroot ' + str(dataroot) + ' --draw_root '
            + draw_root + ' --Pre_Trained_model_path ' + Pre_Trained_model_path + ' --model_path ' + save_model_PATH)
        # VPT + finetuning
        print(
            'python Train.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --lr ' + lr + ' --lrf ' + lrf + ' --enable_tensorboard --model_idx ViT_base_' + model_weight_idx
            + '_PromptDeep_' + str(
                Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr + '_finetuning_' + dataset_name
            + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(
                Prompt_Token_num) + ' --PromptUnFreeze --dataroot ' + str(dataroot) + ' --draw_root ' + draw_root
            + ' --Pre_Trained_model_path ' + Pre_Trained_model_path + ' --model_path ' + save_model_PATH)
        print(
            'python Test.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --model_idx ViT_base_' + model_weight_idx + '_PromptDeep_' + str(
                Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr
            + '_finetuning_' + dataset_name + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(
                Prompt_Token_num) + ' --PromptUnFreeze --dataroot ' + str(dataroot)
            + ' --draw_root ' + draw_root + ' --model_path ' + save_model_PATH)
    else:
        # send a VPT prompt state inside to build the prompt tokens
        # we build VPT backbone with the ViT-timm weight, then do finetuning and prompting
        # fixme notice here Pre_Trained_model_path is actually the trained prompt state path
        # VPT + prompting
        print(
            'python Train.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --lr ' + lr + ' --lrf ' + lrf + ' --enable_tensorboard --model_idx ViT_base_' + model_weight_idx
            + '_PromptDeep_' + str(Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr + '_prompting_' + dataset_name
            + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(Prompt_Token_num) + ' --dataroot ' + str(
                dataroot) + ' --draw_root ' + draw_root
            + ' --Pre_Trained_model_path ' + VPT_backbone_model_path + ' --Prompt_state_path ' + Pre_Trained_model_path + ' --model_path ' + save_model_PATH)
        print(
            'python Test.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --model_idx ViT_base_' + model_weight_idx + '_PromptDeep_' + str(
                Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr
            + '_prompting_' + dataset_name + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(
                Prompt_Token_num) + ' --dataroot ' + str(dataroot) + ' --draw_root '
            + draw_root + ' --Pre_Trained_model_path ' + VPT_backbone_model_path + ' --model_path ' + save_model_PATH)
        # VPT + finetuning
        print(
            'python Train.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --lr ' + lr + ' --lrf ' + lrf + ' --enable_tensorboard --model_idx ViT_base_' + model_weight_idx
            + '_PromptDeep_' + str(
                Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr + '_finetuning_' + dataset_name
            + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(
                Prompt_Token_num) + ' --PromptUnFreeze --dataroot ' + str(dataroot) + ' --draw_root ' + draw_root
            + ' --Pre_Trained_model_path ' + VPT_backbone_model_path + ' --Prompt_state_path ' + Pre_Trained_model_path + ' --model_path ' + save_model_PATH)
        print(
            'python Test.py --gpu_idx ' + GPU_idx + ' --edge_size 224 --data_augmentation_mode ' + data_augmentation_mode
            + ' --model_idx ViT_base_' + model_weight_idx + '_PromptDeep_' + str(
                Prompt_Token_num) + '_' + lr_mystr + '_lf' + lrf_mystr
            + '_finetuning_' + dataset_name + '_CLS --PromptTuning Deep --Prompt_Token_num ' + str(
                Prompt_Token_num) + ' --PromptUnFreeze --dataroot ' + str(dataroot)
            + ' --draw_root ' + draw_root + ' --model_path ' + save_model_PATH)

    print('')


def write_PuzzleTuning_comparison_script(lr_mystr, lrf_mystr, data_augmentation_mode, dataset_name, GPU_idx='0'):
    """
    In PuzzleTuning comparison experiments we put
    datasets at: --dataroot /root/autodl-tmp/datasets
    Pre_Trained_model_path /root/autodl-tmp/pre_trained_models  # output_models (not applicable for comparison)
    Prompt_state_path (not applicable for comparison) /root/autodl-tmp/output_models
    save the training model at: model_path /root/autodl-tmp/saved_models
    draw_root /root/autodl-tmp/PuzzleTuning_Comparison/[*lr*_*lrf*_*dataset_name*]

    """
    dataroot_PATH = '/root/autodl-tmp/datasets'
    Pre_Trained_model_path_PATH = '/root/autodl-tmp/pre_trained_models'
    save_model_PATH = '/root/autodl-tmp/saved_models'
    draw_root_PATH = '/root/autodl-tmp/PuzzleTuning_Comparison'

    data_augmentation_mode = str(data_augmentation_mode)
    GPU_idx = str(GPU_idx)

    lr = str(zero_trans_mystrlr_to_float(lr_mystr))
    lrf = '0.' + str(lrf_mystr)

    experiment_idx = lr_mystr + '_lf' + lrf_mystr + '_' + dataset_name

    dataroot = os.path.join(dataroot_PATH, dataset_name + '_CLS')
    draw_root = os.path.join(draw_root_PATH, experiment_idx)

    # PuzzleTuning official version:
    # we pre-trained VPT prompt tokens, and use the timm ViT as backbone
    print('#SAE-timm-start_promptstate')  # SAE+VPT start with timm
    model_weight_idx = 'ViT_base_timm_PuzzleTuning_SAE_E_199_promptstate'
    model_weight_name = 'ViT_b16_224_timm_PuzzleTuning_SAE_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=True)

    # Comparison methods:

    # For the comparison methods: we trained ViT, so we use ViT + ft first,
    # and then, put it as vpt 's backbone in prompting and VPT finetuning.
    print('#空白对比')
    model_weight_idx = 'random'
    model_weight_name = 'ViT_b16_224_Random_Init.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#timm对比')
    model_weight_idx = 'timm'
    model_weight_name = 'ViT_b16_224_Imagenet.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#MAEImageNet对比')
    model_weight_idx = 'MAEImageNet'
    model_weight_name = 'ViT_b16_224_MAEImageNet_Init.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#mae对比')
    model_weight_idx = 'timm_mae_CPIAm_E100'
    model_weight_name = 'ViT_b16_224_timm_mae_ALL_100.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#moco对比')
    model_weight_idx = 'timm_moco_CPIAm_E100'
    model_weight_name = 'ViT_b16_224_timm_moco_ALL_100.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#dino对比')
    model_weight_idx = 'timm_dino_CPIAm_E100'
    model_weight_name = 'ViT_b16_224_timm_dino_ALL_100.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#BYOL对比')
    model_weight_idx = 'timm_BYOL_CPIAm_E50'
    model_weight_name = 'ViT_b16_224_timm_BYOL_ALL_50.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#GCMAE对比')
    model_weight_idx = 'timm_GCMAE_CPIAm_E80'
    model_weight_name = 'ViT_b16_224_timm_GCMAE_ALL_80.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#SDMAE对比')
    model_weight_idx = 'timm_SDMAE_CPIAm_E80'
    model_weight_name = 'ViT_b16_224_timm_SDMAE_ALL_80.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#SIMMIM对比')
    model_weight_idx = 'timm_SIMMIM_CPIAm_E200'
    model_weight_name = 'ViT_b16_224_timm_SIMMIM_ALL_200.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#SIMCLR对比')
    model_weight_idx = 'timm_SIMCLR_CPIAm_E100'
    model_weight_name = 'ViT_b16_224_timm_SIMCLR_ALL_100.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    # Ablation versions:

    # For ablation SAE-ViT version, we pre-trained ViT, so we use ViT + ft first,
    # and then, put it as vpt 's backbone in prompting and VPT finetuning.
    print('#PuzzleTuning_SAE_ViT-CPIA对比')
    model_weight_idx = 'timm_PuzzleTuning_SAE_E_199'
    model_weight_name = 'ViT_b16_224_timm_PuzzleTuning_SAE_CPIAm_E_199.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#SAE_fixp16fixr25-timm-start')  # SAE_fixp16fixr25+ViT start with timm
    model_weight_idx = 'ViT_base_timm_PuzzleTuning_SAE_fixp16fixr25_E_199'
    model_weight_name = 'ViT_b16_224_timm_PuzzleTuning_SAE_fixp16fixr25_CPIAm_E_199.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    print('#SAE_fixp16ratiodecay-timm-start')  # SAE_fixp16ratiodecay+ViT start with timm
    model_weight_idx = 'ViT_base_timm_PuzzleTuning_SAE_fixp16ratiodecay_E_199'
    model_weight_name = 'ViT_b16_224_timm_PuzzleTuning_SAE_fixp16ratiodecay_CPIAm_E_199.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=False)

    # For ablation SAE-VPT version, we pre-trained VPT prompt tokens, and use the timm ViT as backbone
    print('#MAE-VPT_promptstate')  # MAE+VPT
    model_weight_idx = 'timm_mae_Prompt_CPIAm_E199_promptstate'
    model_weight_name = 'ViT_b16_224_timm_PuzzleTuning_MAE_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=True)

    print('#SAE-MAE-start_promptstate')  # SAE+VPT start with MAEImageNet
    model_weight_idx = 'ViT_base_MAEImageNet_PuzzleTuning_SAE_E_199_promptstate'
    model_weight_name = 'ViT_b16_224_MAEImageNet_PuzzleTuning_SAE_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=True)

    print('#SAE-Random-start_promptstate')  # SAE+VPT start with Random
    model_weight_idx = 'ViT_base_Random_PuzzleTuning_SAE_E_199_promptstate'
    model_weight_name = 'ViT_b16_224_Random_PuzzleTuning_SAE_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=True)

    print('#SAE_fixp16fixr25-timm-start_promptstate')  # SAE_fixp16fixr25+VPT start with timm
    model_weight_idx = 'ViT_base_timm_PuzzleTuning_SAE_fixp16fixr25_E_199_promptstate'
    model_weight_name = 'ViT_b16_224_timm_PuzzleTuning_SAE_fixp16fixr25_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=True)

    print('#SAE_fixp16ratiodecay-timm-start_promptstate')  # SAE_fixp16ratiodecay+VPT start with timm
    model_weight_idx = 'ViT_base_timm_PuzzleTuning_SAE_fixp16ratiodecay_E_199_promptstate'
    model_weight_name = 'ViT_b16_224_timm_PuzzleTuning_SAE_fixp16ratiodecay_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth'
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=True)

    print('')
    print('cd /home/pancreatic-cancer-diagnosis-tansformer/code/utils')
    record_dir = os.path.join(draw_root, 'CSV_logs')
    print('python check_log_json.py --enable_notify --draw_root ' + draw_root + ' --record_dir ' + record_dir)
    print('cd /home/pancreatic-cancer-diagnosis-tansformer/code')


def write_additional_PuzzleTuning_comparison_script(add_idx, lr_mystr, lrf_mystr, data_augmentation_mode, dataset_name,
                                                    model_weight_idx='timm_mae_CPIAm_E100',
                                                    model_weight_name='ViT_b16_224_timm_mae_ALL_100.pth',
                                                    GPU_idx='0', Prompt_input=False):
    """
    In PuzzleTuning comparison experiments we put
    datasets at: --dataroot /root/autodl-tmp/datasets
    Pre_Trained_model_path /root/autodl-tmp/pre_trained_models  # output_models (not applicable for comparison)
    Prompt_state_path (not applicable for comparison) /root/autodl-tmp/output_models
    save the training model at: model_path /root/autodl-tmp/saved_models
    draw_root /root/autodl-tmp/PuzzleTuning_Comparison/[*lr*_*lrf*_*dataset_name*]

    # fixme the additional experiments settings need to manually set!!!
    in the additional experiments, we save the runs to
    draw_root /root/autodl-tmp/runs/[*lr*_*lrf*_*dataset_name*]
    and then copy a duplicates to /root/autodl-tmp/PuzzleTuning_Comparison/[*lr*_*lrf*_*dataset_name*]

    """
    dataroot_PATH = '/root/autodl-tmp/datasets'
    Pre_Trained_model_path_PATH = '/root/autodl-tmp/pre_trained_models'
    save_model_PATH = '/root/autodl-tmp/saved_models'
    draw_root_PATH = '/root/autodl-tmp/runs'
    copy_to_draw_root_PATH = '/root/autodl-tmp/PuzzleTuning_Comparison'

    data_augmentation_mode = str(data_augmentation_mode)
    GPU_idx = str(GPU_idx)

    lr = str(zero_trans_mystrlr_to_float(lr_mystr))
    lrf = '0.' + str(lrf_mystr)

    experiment_idx = lr_mystr + '_lf' + lrf_mystr + '_' + dataset_name
    add_experiment_idx = add_idx + '_' + lr_mystr + '_lf' + lrf_mystr + '_' + dataset_name

    dataroot = os.path.join(dataroot_PATH, dataset_name + '_CLS')
    # additional exp runs path
    draw_root = os.path.join(draw_root_PATH, add_experiment_idx)
    # basic all exp runs path
    copy_draw_root = os.path.join(copy_to_draw_root_PATH, experiment_idx)

    print('# Additional ' + add_idx)
    print_a_PuzzleTuning_comparison_script(model_weight_idx, model_weight_name, lr, lrf, lr_mystr, lrf_mystr,
                                           dataset_name, dataroot, draw_root, Pre_Trained_model_path_PATH,
                                           save_model_PATH, data_augmentation_mode, GPU_idx, Prompt_input=Prompt_input)
    print('')
    print('cd /home/pancreatic-cancer-diagnosis-tansformer/code/utils')
    # update the total record
    print('')
    print('cp -r ' + draw_root + '/*' + ' ' + copy_draw_root)
    record_dir = os.path.join(copy_draw_root, 'CSV_logs')
    print('python check_log_json.py --draw_root ' + copy_draw_root + ' --record_dir ' + record_dir)

    # update the additional runs and send to notify
    record_dir = os.path.join(draw_root, add_experiment_idx)
    print('python check_log_json.py --enable_notify --draw_root ' + draw_root + ' --record_dir ' + record_dir)

    print('cd /home/pancreatic-cancer-diagnosis-tansformer/code')


def write_CLS_script(model_idxs, data_augmentation_mode, edge_size, batch_size, lr, lrf, enable_tensorboard,
                     test_enable_attention_check, dataset_name, dataroot, model_path, draw_root):
    data_augmentation_mode = str(data_augmentation_mode)
    edge_size_ipt = str(edge_size)
    batch_size = str(batch_size)
    lr_name = zero_trans_floatlr_to_mystrlr(lr)
    lr = str(lr)
    lf_name = str(int(100 * lrf))
    lrf = str(lrf)
    dataroot = dataroot + dataset_name + '_CLS'

    for model_idx in model_idxs:

        # alter the edge size for certain models
        if model_idx in ['cross_former', 'convit', 'visformer', 'ViT_h']:
            edge_size = '224'
        else:
            edge_size = edge_size_ipt

        if enable_tensorboard is True:
            print('python Train.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_CLS --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode + ' --batch_size ' + batch_size +
                  ' --lr ' + lr + ' --lrf ' + lrf + ' --enable_tensorboard --dataroot ' + dataroot +
                  ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')
        else:
            print('python Train.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_CLS --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode + ' --batch_size ' + batch_size +
                  ' --lr ' + lr + ' --lrf ' + lrf + ' --dataroot ' + dataroot +
                  ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')

    for model_idx in model_idxs:

        # alter the edge size for certain models
        if model_idx in ['cross_former', 'convit', 'visformer', 'ViT_h']:
            edge_size = '224'
        else:
            edge_size = edge_size_ipt

        if test_enable_attention_check is True:
            print('python Test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_CLS --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode + ' --enable_attention_check --dataroot '
                  + dataroot + ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')
        else:
            print('python Test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_CLS --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode + ' --dataroot '
                  + dataroot + ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')


def write_CLS_AUG_script(model_idx, augmentation_names, data_augmentation_mode, edge_size, batch_size, lr, lrf,
                         test_enable_attention_check, enable_tensorboard, dataset_name, dataroot, model_path,
                         draw_root):
    data_augmentation_mode = str(data_augmentation_mode)
    data_augmentation_mode = str(data_augmentation_mode)
    edge_size = str(edge_size)
    batch_size = str(batch_size)
    lr_name = zero_trans_floatlr_to_mystrlr(lr)
    lr = str(lr)
    lf_name = str(int(100 * lrf))
    lrf = str(lrf)
    dataroot = dataroot + dataset_name + '_CLS'

    for augmentation_name in augmentation_names:
        if enable_tensorboard is True:
            print('python Train.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_' + augmentation_name + '_CLS' +
                  ' --augmentation_name ' + augmentation_name + ' --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode + ' --batch_size ' + batch_size +
                  ' --lr ' + lr + ' --lrf ' + lrf + ' --enable_tensorboard --dataroot ' + dataroot +
                  ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')
        else:
            print('python Train.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_' + augmentation_name + '_CLS' +
                  ' --augmentation_name ' + augmentation_name + ' --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode + ' --batch_size ' + batch_size +
                  ' --lr ' + lr + ' --lrf ' + lrf + ' --dataroot ' + dataroot +
                  ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')

    for augmentation_name in augmentation_names:
        if test_enable_attention_check is True:
            print('python Test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_' + augmentation_name + '_CLS' +
                  ' --edge_size ' + edge_size + ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --enable_attention_check --dataroot ' + dataroot + ' --model_path ' + model_path +
                  ' --draw_root ' + draw_root)
            print('')
        else:
            print('python Test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_' + dataset_name + '_' + augmentation_name + '_CLS' +
                  ' --edge_size ' + edge_size + ' --data_augmentation_mode ' + data_augmentation_mode + ' --dataroot '
                  + dataroot + ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')


def write_MIL_script(model_idxs, data_augmentation_mode, edge_size, batch_size, patch_size, lr, lrf, enable_tensorboard,
                     test_enable_attention_check, dataset_name, dataroot, model_path, draw_root, imaging_root=None):
    # imaging_root 是放画图的检查的路径，可以和draw一样
    if imaging_root == None:
        imaging_root = draw_root

    data_augmentation_mode = str(data_augmentation_mode)
    edge_size = str(edge_size)
    batch_size = str(batch_size)
    patch_size = str(patch_size)
    lr_name = zero_trans_floatlr_to_mystrlr(lr)
    lr = str(lr)
    lf_name = str(int(100 * lrf))
    lrf = str(lrf)
    dataroot = dataroot + dataset_name + '_MIL'
    CLS_dataroot = dataroot + dataset_name + '_CLS'

    for model_idx in model_idxs:
        if enable_tensorboard is True:
            print('python MIL_train.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
                  '_MIL --edge_size ' + edge_size + ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --batch_size ' + batch_size + ' --patch_size ' + patch_size + ' --lr ' + lr + ' --lrf '
                  + lrf + ' --enable_tensorboard --dataroot ' + dataroot + ' --model_path ' + model_path
                  + ' --draw_root ' + draw_root)
            print('')
        else:
            print('python MIL_train.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
                  '_MIL --edge_size ' + edge_size + ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --batch_size ' + batch_size + ' --patch_size ' + patch_size + ' --lr ' + lr + ' --lrf '
                  + lrf + ' --dataroot ' + dataroot + ' --model_path ' + model_path + ' --draw_root ' + draw_root)
            print('')

    for model_idx in model_idxs:
        print('python MIL_test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
              + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
              '_MIL --edge_size ' + edge_size + ' --patch_size ' + patch_size +
              ' --batch_size 1 --data_augmentation_mode ' + data_augmentation_mode + ' --dataroot ' +
              dataroot + ' --model_path ' + model_path + ' --draw_root ' + draw_root)
        print('')

        if test_enable_attention_check is True:  # 设置多个batch的实验
            print('python Test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
                  '_MIL --edge_size ' + edge_size + ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --MIL_Stripe --enable_attention_check --check_minibatch 10' +
                  ' --dataroot ' + CLS_dataroot + ' --model_path ' + model_path +
                  ' --draw_root ' + imaging_root)
            print('')
            print('python MIL_test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
                  '_MIL --shuffle_attention_check --MIL_Stripe --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --shuffle_dataloader --batch_size 4 --check_minibatch 10' + ' --patch_size ' + patch_size +
                  ' --dataroot ' + dataroot + ' --model_path ' + model_path +
                  ' --draw_root ' + imaging_root)
            print('')
            print('python MIL_test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
                  '_MIL --shuffle_attention_check --MIL_Stripe --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --batch_size 4 --check_minibatch 10' + ' --patch_size ' + patch_size +
                  ' --dataroot ' + dataroot + ' --model_path ' + model_path +
                  ' --draw_root ' + imaging_root)
            print('')
            print('python MIL_test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
                  '_MIL --shuffle_attention_check --MIL_Stripe --edge_size ' + edge_size +
                  ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --batch_size 1 --check_minibatch 10' + ' --patch_size ' + patch_size +
                  ' --dataroot ' + dataroot + ' --model_path ' + model_path +
                  ' --draw_root ' + imaging_root)
            print('')

        else:
            print('python Test.py --model_idx ' + model_idx + '_' + edge_size + '_' + lr_name
                  + '_PT_lf' + lf_name + '_b' + batch_size + '_p' + patch_size + '_' + dataset_name +
                  '_MIL --edge_size ' + edge_size + ' --data_augmentation_mode ' + data_augmentation_mode +
                  ' --MIL_Stripe --dataroot ' + CLS_dataroot + ' --model_path ' + model_path +
                  ' --draw_root ' + draw_root)
            print('')


'''
if __name__ == '__main__':

    print('#!/bin/sh')
    print('')
    # CLS-MIL调参的第一步是使用一个经验参数进行简单摸索，看看大家结果大概是多少，同时和文献进行对比
    # 首先摸索CLS对比实验结果
    model_idxs = ['ViT', 'vgg16', 'vgg19', 'mobilenetv3', 'inceptionv3', 'xception',
                  'ResNet50', 'efficientnet_b3', 'swin_b', 'ResN50_ViT', 'conformer', 'cross_former']

    batch_size = 8
    dataset_name = 'NCT-CRC-HE-100K'

    write_CLS_script(model_idxs=model_idxs,
                     data_augmentation_mode=3,
                     edge_size=384,
                     batch_size=batch_size,
                     lr=0.000007,
                     lrf=0.35,
                     enable_tensorboard=True,
                     test_enable_attention_check=True,
                     dataset_name=dataset_name,
                     dataroot='/root/autodl-tmp/datasets/',
                     model_path='/root/autodl-tmp/saved_models',
                     draw_root='/root/autodl-tmp/runs')

    # 正式实验的时候，后面还需要做各种MIL的消融实验
    # TODO 更多write_MIL_script
    # 其次摸索CLS+特定模型vit+不同数据增强 对比实验结果
    augmentation_names = ['Cutout', 'Mixup', 'CutMix']
    write_CLS_AUG_script(model_idx='ViT',
                         augmentation_names=augmentation_names,
                         data_augmentation_mode=3,
                         edge_size=384,
                         batch_size=batch_size,
                         lr=0.000007,
                         lrf=0.35,
                         enable_tensorboard=True,
                         test_enable_attention_check=True,
                         dataset_name=dataset_name,
                         dataroot='/root/autodl-tmp/datasets/',
                         model_path='/root/autodl-tmp/saved_models',
                         draw_root='/root/autodl-tmp/runs')

    # 最后摸索MIL+ViT的实验结果
    MIL_model_idxs = ['ViT', ]
    # MIL ablations
    write_MIL_script(model_idxs=MIL_model_idxs,
                     data_augmentation_mode=3,
                     edge_size=384,
                     batch_size=batch_size,
                     patch_size=16,
                     lr=0.000007,
                     lrf=0.35,
                     enable_tensorboard=True,
                     test_enable_attention_check=False,
                     dataset_name=dataset_name,
                     dataroot='/root/autodl-tmp/datasets/',
                     model_path='/root/autodl-tmp/saved_models',
                     draw_root='/root/autodl-tmp/runs',
                     imaging_root='/root/autodl-tmp/imaging_results')
    write_MIL_script(model_idxs=MIL_model_idxs,
                     data_augmentation_mode=3,
                     edge_size=384,
                     batch_size=batch_size,
                     patch_size=64,
                     lr=0.000007,
                     lrf=0.35,
                     enable_tensorboard=True,
                     test_enable_attention_check=False,
                     dataset_name=dataset_name,
                     dataroot='/root/autodl-tmp/datasets/',
                     model_path='/root/autodl-tmp/saved_models',
                     draw_root='/root/autodl-tmp/runs',
                     imaging_root='/root/autodl-tmp/imaging_results')
    write_MIL_script(model_idxs=MIL_model_idxs,
                     data_augmentation_mode=3,
                     edge_size=384,
                     batch_size=batch_size,
                     patch_size=48,
                     lr=0.000007,
                     lrf=0.35,
                     enable_tensorboard=True,
                     test_enable_attention_check=False,
                     dataset_name=dataset_name,
                     dataroot='/root/autodl-tmp/datasets/',
                     model_path='/root/autodl-tmp/saved_models',
                     draw_root='/root/autodl-tmp/runs',
                     imaging_root='/root/autodl-tmp/imaging_results')
    write_MIL_script(model_idxs=MIL_model_idxs,
                     data_augmentation_mode=3,
                     edge_size=384,
                     batch_size=batch_size,
                     patch_size=96,
                     lr=0.000007,
                     lrf=0.35,
                     enable_tensorboard=True,
                     test_enable_attention_check=False,
                     dataset_name=dataset_name,
                     dataroot='/root/autodl-tmp/datasets/',
                     model_path='/root/autodl-tmp/saved_models',
                     draw_root='/root/autodl-tmp/runs',
                     imaging_root='/root/autodl-tmp/imaging_results')
    write_MIL_script(model_idxs=MIL_model_idxs,
                     data_augmentation_mode=3,
                     edge_size=384,
                     batch_size=batch_size,
                     patch_size=128,
                     lr=0.000007,
                     lrf=0.35,
                     enable_tensorboard=True,
                     test_enable_attention_check=False,
                     dataset_name=dataset_name,
                     dataroot='/root/autodl-tmp/datasets/',
                     model_path='/root/autodl-tmp/saved_models',
                     draw_root='/root/autodl-tmp/runs',
                     imaging_root='/root/autodl-tmp/imaging_results')

    # 调参实验的时候，先调MIL到最好，然后用参数去跑CLS实验看结果

    print('cd /home/pancreatic-cancer-diagnosis-tansformer/code/utils')
    print('')
    print(
        'python check_log_json.py --enable_notify --draw_root /root/autodl-tmp/runs --record_dir /root/autodl-tmp/CSV_logs')
    print('')
    print('shutdown')
'''


def get_args_parser():
    parser = argparse.ArgumentParser(description='Automatically write shell script for training')

    # Model Name or index
    parser.add_argument('--lr_mystr', default=None, type=str, help='Model lr EG: 506 -> 0.000006')
    parser.add_argument('--lrf_mystr', default=None, type=str, help='Model lrf EG: 50 -> cosine decay to 50%')
    parser.add_argument('--data_augmentation_mode', default=None, type=str, help='ROSE,pRCC:0; CAM16,WBC:3')
    parser.add_argument('--dataset_name', default=None, type=str, help='ROSE,pRCC,CAM16,WBC ?')
    parser.add_argument('--GPU_idx', default='0', type=str, help='Experiment GPU_idx EG: 0')

    return parser


if __name__ == '__main__':
    parser = get_args_parser()
    args = parser.parse_args()

    print('#!/bin/sh')
    print('')
    # add DropPos-CPIA
    write_additional_PuzzleTuning_comparison_script(add_idx='DropPos-CPIA', lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='timm_DropPos_CPIAm_E200',
                                                    model_weight_name='ViT_b16_224_timm_DropPos_ALL_200.pth',
                                                    GPU_idx=args.GPU_idx, Prompt_input=False)

    '''
    # add MAE-CPIA
    write_additional_PuzzleTuning_comparison_script(add_idx='MAE-CPIA', lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name, 
                                                    model_weight_idx='timm_mae_CPIAm_E100',
                                                    model_weight_name='ViT_b16_224_timm_mae_ALL_100.pth', 
                                                    GPU_idx=args.GPU_idx, Prompt_input=False)
    # add SDMAE-CPIA
    write_additional_PuzzleTuning_comparison_script(add_idx='SDMAE-CPIA', lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='timm_SDMAE_CPIAm_E80',
                                                    model_weight_name='ViT_b16_224_timm_SDMAE_ALL_80.pth',
                                                    GPU_idx=args.GPU_idx, Prompt_input=False)
    # add GCMAE-CPIA
    write_additional_PuzzleTuning_comparison_script(add_idx='GCMAE-CPIA', lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='timm_GCMAE_CPIAm_E80',
                                                    model_weight_name='ViT_b16_224_timm_GCMAE_ALL_80.pth',
                                                    GPU_idx=args.GPU_idx, Prompt_input=False)
    # add JIGSAW-CPIA
    write_additional_PuzzleTuning_comparison_script(add_idx='JIGSAW-CPIA', lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='timm_JIGSAW_CPIAm_E50',
                                                    model_weight_name='ViT_b16_224_timm_JIGSAW_ALL_50.pth',
                                                    GPU_idx=args.GPU_idx, Prompt_input=False)
                                                    
    # add DropPos-CPIA
    write_additional_PuzzleTuning_comparison_script(add_idx='DropPos-CPIA', lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='timm_DropPos_CPIAm_E200',
                                                    model_weight_name='ViT_b16_224_timm_DropPos_ALL_200.pth',
                                                    GPU_idx=args.GPU_idx, Prompt_input=False)
                                                    
    # add MAE+VPT
    write_additional_PuzzleTuning_comparison_script(add_idx='MAE-VPT_promptstate',
                                                    lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='timm_mae_Prompt_CPIAm_E199_promptstate',
                                                    model_weight_name='ViT_b16_224_timm_PuzzleTuning_MAE_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth',
                                                    GPU_idx='0', Prompt_input=True)
    # add SAE-MAE-start
    write_additional_PuzzleTuning_comparison_script(add_idx='SAE-MAE-start_promptstate',
                                                    lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='ViT_base_MAEImageNet_PuzzleTuning_SAE_E_199_promptstate',
                                                    model_weight_name='ViT_b16_224_MAEImageNet_PuzzleTuning_SAE_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth',
                                                    GPU_idx='0', Prompt_input=True)
    # add SAE-Random-start
    write_additional_PuzzleTuning_comparison_script(add_idx='SAE-Random-start_promptstate',
                                                    lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='ViT_base_Random_PuzzleTuning_SAE_E_199_promptstate',
                                                    model_weight_name='ViT_b16_224_Random_PuzzleTuning_SAE_CPIAm_Prompt_Deep_tokennum_20_E_199_promptstate.pth',
                                                    GPU_idx='0', Prompt_input=True)
                                                    
    # add PuzzleTuning_SAE_ViT_to_VPT-CPIA
    write_additional_PuzzleTuning_comparison_script(add_idx='PuzzleTuning_SAE_ViT-CPIA', lr_mystr=args.lr_mystr,
                                                    lrf_mystr=args.lrf_mystr,
                                                    data_augmentation_mode=args.data_augmentation_mode,
                                                    dataset_name=args.dataset_name,
                                                    model_weight_idx='timm_PuzzleTuning_SAE_E_199',
                                                    model_weight_name='ViT_b16_224_timm_PuzzleTuning_SAE_CPIAm_E_199.pth',
                                                    GPU_idx=args.GPU_idx, Prompt_input=False)
    '''

    # rewrite all
    '''
    write_PuzzleTuning_comparison_script(lr_mystr=args.lr_mystr, lrf_mystr=args.lrf_mystr,
                                         data_augmentation_mode=args.data_augmentation_mode,
                                         dataset_name=args.dataset_name, GPU_idx=args.GPU_idx)
    '''


    '''
    we can use the following codes to generates the additional exp scripts
    
    # read and auto generate task info
    import os
    path='/root/autodl-tmp/PuzzleTuning_Comparison'
    data_augmentation_dic = {'ROSE': '0', 'pRCC': '0', 'CAM16': '3', 'WBC': '3'}
    for exp_root in os.listdir(path):
        out_sh_name = exp_root + '.sh'
        lr_mystr = exp_root.split('_')[0]
        lrf_mystr = exp_root.split('_')[1].split('lf')[-1]
        dataset_name = exp_root.split('_')[-1]
        data_augmentation_mode = data_augmentation_dic[dataset_name]
        print('nohup python Experiment_script_helper.py --lr_mystr ' + lr_mystr + ' --lrf_mystr ' + lrf_mystr
              + ' --data_augmentation_mode ' + data_augmentation_mode + ' --dataset_name ' + dataset_name + ' > '
              + out_sh_name + ' 2>&1 &')
    
    # then, we use the shell to run this code with the generated lines
    
    # the generate sh files has a nohup line at their first lines, so we can use this to erase
    remove_nohup_ignoring_input_at_first_line(directory='./')
    
    # we can use the func to combine the sh files:
    concatenate_the_lines_from_several_files(directory='./', cat_file='0.sh')
    '''