File size: 7,658 Bytes
772f69e 5544a13 772f69e 5544a13 772f69e 5544a13 772f69e 5544a13 772f69e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 |
SYSTEM = ''
accumulative_counts = 1
batch_size = 16
betas = (
0.9,
0.999,
)
custom_hooks = [
dict(
tokenizer=dict(
padding_side='right',
pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained'),
type='xtuner.engine.DatasetInfoHook'),
dict(
evaluation_images='https://llava-vl.github.io/static/images/view.jpg',
evaluation_inputs=[
'请描述一下这张照片',
'Please describe this picture',
],
every_n_iters=500,
image_processor=dict(
pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
trust_remote_code=True,
type='transformers.CLIPImageProcessor.from_pretrained'),
prompt_template='xtuner.utils.PROMPT_TEMPLATE.vicuna',
system='',
tokenizer=dict(
padding_side='right',
pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained'),
type='xtuner.engine.EvaluateChatHook'),
]
data_path = './data/llava_data/LLaVA-Instruct-150K/llava_v1_5_mix665k.json'
dataloader_num_workers = 0
default_hooks = dict(
checkpoint=dict(interval=1, type='mmengine.hooks.CheckpointHook'),
logger=dict(interval=10, type='mmengine.hooks.LoggerHook'),
param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
timer=dict(type='mmengine.hooks.IterTimerHook'))
env_cfg = dict(
cudnn_benchmark=False,
dist_cfg=dict(backend='nccl'),
mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
evaluation_freq = 500
evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg'
evaluation_inputs = [
'请描述一下这张照片',
'Please describe this picture',
]
image_folder = './data/llava_data/llava_images'
launcher = 'pytorch'
llava_data_root = './data/llava_data/'
llava_dataset = dict(
data_path='./data/llava_data/LLaVA-Instruct-150K/llava_v1_5_mix665k.json',
dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
image_folder='./data/llava_data/llava_images',
max_length=1472,
pad_image_to_square=True,
image_processor=dict(
pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
trust_remote_code=True,
type='transformers.CLIPImageProcessor.from_pretrained'),
template_map_fn=dict(
template='xtuner.utils.PROMPT_TEMPLATE.vicuna',
type='xtuner.dataset.map_fns.template_map_fn_factory'),
tokenizer=dict(
padding_side='right',
pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained'),
type='xtuner.dataset.LLaVADataset')
llm_name_or_path = 'lmsys/vicuna-7b-v1.5'
load_from = None
log_level = 'INFO'
lr = 0.0002
max_epochs = 1
max_length = 1472
max_norm = 1
model = dict(
freeze_llm=True,
freeze_visual_encoder=True,
llm=dict(
pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
quantization_config=dict(
bnb_4bit_compute_dtype='torch.float16',
bnb_4bit_quant_type='nf4',
bnb_4bit_use_double_quant=True,
llm_int8_has_fp16_weight=False,
llm_int8_threshold=6.0,
load_in_4bit=True,
load_in_8bit=False,
type='transformers.BitsAndBytesConfig'),
torch_dtype='torch.float16',
trust_remote_code=True,
type='transformers.AutoModelForCausalLM.from_pretrained'),
llm_lora=dict(
bias='none',
lora_alpha=256,
lora_dropout=0.05,
r=512,
task_type='CAUSAL_LM',
type='peft.LoraConfig'),
pretrained_pth=
'./work_dirs/llava_vicuna_7b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain/epoch_1.pth',
type='xtuner.model.LLaVAModel',
visual_encoder=dict(
pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
type='transformers.CLIPVisionModel.from_pretrained'),
visual_encoder_lora=dict(
bias='none',
lora_alpha=16,
lora_dropout=0.05,
r=64,
type='peft.LoraConfig'))
optim_type = 'torch.optim.AdamW'
optim_wrapper = dict(
optimizer=dict(
betas=(
0.9,
0.999,
),
lr=0.0002,
type='torch.optim.AdamW',
weight_decay=0),
type='DeepSpeedOptimWrapper')
param_scheduler = [
dict(
begin=0,
by_epoch=True,
convert_to_iter_based=True,
end=0.03,
start_factor=1e-05,
type='mmengine.optim.LinearLR'),
dict(
T_max=1,
begin=0.03,
by_epoch=True,
convert_to_iter_based=True,
eta_min=0.0,
type='mmengine.optim.CosineAnnealingLR'),
]
pretrained_pth = './work_dirs/llava_vicuna_7b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain/epoch_1.pth'
image_processor = dict(
pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
trust_remote_code=True,
type='transformers.CLIPImageProcessor.from_pretrained')
prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.vicuna'
randomness = dict(deterministic=False, seed=None)
resume = False
runner_type = 'FlexibleRunner'
strategy = dict(
config=dict(
bf16=dict(enabled=True),
fp16=dict(enabled=False, initial_scale_power=16),
gradient_accumulation_steps='auto',
gradient_clipping='auto',
train_micro_batch_size_per_gpu='auto',
zero_allow_untested_optimizer=True,
zero_force_ds_cpu_optimizer=False,
zero_optimization=dict(overlap_comm=True, stage=2)),
exclude_frozen_parameters=True,
gradient_accumulation_steps=1,
gradient_clipping=1,
train_micro_batch_size_per_gpu=16,
type='xtuner.engine.DeepSpeedStrategy')
tokenizer = dict(
padding_side='right',
pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained')
train_cfg = dict(by_epoch=True, max_epochs=1, val_interval=1)
train_dataloader = dict(
batch_size=16,
collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
dataset=dict(
data_path=
'./data/llava_data/LLaVA-Instruct-150K/llava_v1_5_mix665k.json',
dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
image_folder='./data/llava_data/llava_images',
max_length=1472,
pad_image_to_square=True,
image_processor=dict(
pretrained_model_name_or_path='openai/clip-vit-large-patch14-336',
trust_remote_code=True,
type='transformers.CLIPImageProcessor.from_pretrained'),
template_map_fn=dict(
template='xtuner.utils.PROMPT_TEMPLATE.vicuna',
type='xtuner.dataset.map_fns.template_map_fn_factory'),
tokenizer=dict(
padding_side='right',
pretrained_model_name_or_path='lmsys/vicuna-7b-v1.5',
trust_remote_code=True,
type='transformers.AutoTokenizer.from_pretrained'),
type='xtuner.dataset.LLaVADataset'),
num_workers=0,
sampler=dict(
length_property='modality_length',
per_device_batch_size=16,
type='xtuner.dataset.samplers.LengthGroupedSampler'))
visual_encoder_name_or_path = 'openai/clip-vit-large-patch14-336'
visualizer = None
warmup_ratio = 0.03
weight_decay = 0
work_dir = './work_dirs/llava_vicuna_7b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune'
|