Update README.md
Browse files
README.md
CHANGED
@@ -111,4 +111,385 @@ deepspeed llava/train/train_mem.py \
|
|
111 |
--lazy_preprocess True \
|
112 |
--report_to wandb \
|
113 |
--run_name $BASE_RUN_NAME \
|
114 |
-
--attn_implementation flash_attention_2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
111 |
--lazy_preprocess True \
|
112 |
--report_to wandb \
|
113 |
--run_name $BASE_RUN_NAME \
|
114 |
+
--attn_implementation flash_attention_2
|
115 |
+
|
116 |
+
|
117 |
+
|
118 |
+
here is my pip list for my system. if you use flash attention you must download it after installig above and compile and install the wheel from source. Other wise when you isntall it will download the older version of transformers etc pytorch
|
119 |
+
pip3 list
|
120 |
+
Package Version Editable project location
|
121 |
+
----------------------------- ------------------------ --------------------------------
|
122 |
+
absl-py 2.1.0
|
123 |
+
accelerate 0.34.2
|
124 |
+
aiofiles 22.1.0
|
125 |
+
aiohappyeyeballs 2.4.0
|
126 |
+
aiohttp 3.10.5
|
127 |
+
aiosignal 1.3.1
|
128 |
+
aiosqlite 0.20.0
|
129 |
+
altair 5.4.1
|
130 |
+
annotated-types 0.7.0
|
131 |
+
anyio 4.4.0
|
132 |
+
appdirs 1.4.4
|
133 |
+
argon2-cffi 23.1.0
|
134 |
+
argon2-cffi-bindings 21.2.0
|
135 |
+
arrow 1.3.0
|
136 |
+
asttokens 2.4.1
|
137 |
+
astunparse 1.6.3
|
138 |
+
async-lru 2.0.4
|
139 |
+
async-timeout 4.0.3
|
140 |
+
attrs 24.2.0
|
141 |
+
audioread 3.0.1
|
142 |
+
av 13.0.0
|
143 |
+
babel 2.16.0
|
144 |
+
beartype 0.14.1
|
145 |
+
beautifulsoup4 4.12.3
|
146 |
+
better-abc 0.0.3
|
147 |
+
bidict 0.23.1
|
148 |
+
bitsandbytes 0.43.3
|
149 |
+
black 24.1.0
|
150 |
+
bleach 6.1.0
|
151 |
+
Brotli 1.1.0
|
152 |
+
cachetools 5.5.0
|
153 |
+
certifi 2024.8.30
|
154 |
+
cffi 1.17.1
|
155 |
+
cfgv 3.4.0
|
156 |
+
chardet 5.2.0
|
157 |
+
charset-normalizer 3.3.2
|
158 |
+
click 8.1.7
|
159 |
+
cmake 3.30.2
|
160 |
+
colorama 0.4.6
|
161 |
+
comm 0.2.2
|
162 |
+
contourpy 1.3.0
|
163 |
+
crcmod 1.7
|
164 |
+
cryptography 43.0.1
|
165 |
+
cuda-python 12.4.0 /home/myles/cuda-python-12.4.0
|
166 |
+
cycler 0.12.1
|
167 |
+
Cython 3.0.11
|
168 |
+
DataProperty 1.0.1
|
169 |
+
datasets 2.16.1
|
170 |
+
debugpy 1.8.5
|
171 |
+
decorator 5.1.1
|
172 |
+
decord 0.6.0
|
173 |
+
deepspeed 0.15.2+fc22d960
|
174 |
+
deepspeed-kernels 0.0.1.dev1698255861
|
175 |
+
defusedxml 0.7.1
|
176 |
+
Deprecated 1.2.14
|
177 |
+
dill 0.3.7
|
178 |
+
distlib 0.3.8
|
179 |
+
distro 1.9.0
|
180 |
+
dnspython 2.6.1
|
181 |
+
docker-pycreds 0.4.0
|
182 |
+
docopt 0.6.2
|
183 |
+
docstring_parser 0.16
|
184 |
+
e 1.4.5
|
185 |
+
einops 0.8.0
|
186 |
+
einops-exts 0.0.4
|
187 |
+
entrypoints 0.4
|
188 |
+
et-xmlfile 1.1.0
|
189 |
+
eval_type_backport 0.2.0
|
190 |
+
evaluate 0.4.2
|
191 |
+
exceptiongroup 1.2.2
|
192 |
+
executing 2.1.0
|
193 |
+
fancy-einsum 0.0.3
|
194 |
+
fastapi 0.112.4
|
195 |
+
fastjsonschema 2.20.0
|
196 |
+
ffmpeg-python 0.2.0
|
197 |
+
ffmpy 0.4.0
|
198 |
+
filelock 3.16.0
|
199 |
+
flash_attn 2.6.3
|
200 |
+
flatbuffers 24.3.25
|
201 |
+
fonttools 4.53.1
|
202 |
+
fqdn 1.5.1
|
203 |
+
frozenlist 1.4.1
|
204 |
+
fsspec 2023.10.0
|
205 |
+
ftfy 6.2.3
|
206 |
+
future 1.0.0
|
207 |
+
gast 0.6.0
|
208 |
+
gitdb 4.0.11
|
209 |
+
GitPython 3.1.43
|
210 |
+
google-pasta 0.2.0
|
211 |
+
gradio 4.43.0
|
212 |
+
gradio_client 1.3.0
|
213 |
+
graphviz 0.20.3
|
214 |
+
grpcio 1.66.1
|
215 |
+
h11 0.14.0
|
216 |
+
h5py 3.11.0
|
217 |
+
hf_transfer 0.1.8
|
218 |
+
hjson 3.1.0
|
219 |
+
httpcore 1.0.5
|
220 |
+
httpx 0.27.2
|
221 |
+
huggingface-hub 0.24.6
|
222 |
+
identify 2.6.0
|
223 |
+
idna 3.8
|
224 |
+
importlib_metadata 8.4.0
|
225 |
+
importlib_resources 6.4.4
|
226 |
+
iniconfig 2.0.0
|
227 |
+
ipaddress 1.0.23
|
228 |
+
ipykernel 6.29.5
|
229 |
+
ipython 8.27.0
|
230 |
+
ipython-genutils 0.2.0
|
231 |
+
ipywidgets 8.1.5
|
232 |
+
isoduration 20.11.0
|
233 |
+
isort 5.13.2
|
234 |
+
jaxtyping 0.2.34
|
235 |
+
jedi 0.19.1
|
236 |
+
Jinja2 3.1.4
|
237 |
+
jiter 0.5.0
|
238 |
+
joblib 1.4.2
|
239 |
+
json5 0.9.25
|
240 |
+
jsonlines 4.0.0
|
241 |
+
jsonpointer 3.0.0
|
242 |
+
jsonschema 4.23.0
|
243 |
+
jsonschema-specifications 2023.12.1
|
244 |
+
jupyter 1.1.1
|
245 |
+
jupyter_client 8.6.2
|
246 |
+
jupyter-console 6.6.3
|
247 |
+
jupyter_core 5.7.2
|
248 |
+
jupyter-events 0.10.0
|
249 |
+
jupyter-lsp 2.2.5
|
250 |
+
jupyter_server 2.14.2
|
251 |
+
jupyter_server_fileid 0.9.3
|
252 |
+
jupyter_server_terminals 0.5.3
|
253 |
+
jupyter_server_ydoc 0.8.0
|
254 |
+
jupyter-ydoc 0.3.4
|
255 |
+
jupyterlab 4.2.5
|
256 |
+
jupyterlab_pygments 0.3.0
|
257 |
+
jupyterlab_server 2.27.3
|
258 |
+
jupyterlab_widgets 3.0.13
|
259 |
+
keras 3.5.0
|
260 |
+
kiwisolver 1.4.7
|
261 |
+
latex2mathml 3.77.0
|
262 |
+
lazy_loader 0.4
|
263 |
+
Levenshtein 0.25.1
|
264 |
+
libclang 18.1.1
|
265 |
+
librosa 0.10.2.post1
|
266 |
+
linkify-it-py 2.0.3
|
267 |
+
llava 1.7.0.dev0 /home/myles/LLaVA-NeXT
|
268 |
+
llvmlite 0.43.0
|
269 |
+
lmms_eval 0.2.3 /home/myles/lmms-eval
|
270 |
+
loguru 0.7.2
|
271 |
+
lxml 5.3.0
|
272 |
+
Markdown 3.7
|
273 |
+
markdown-it-py 3.0.0
|
274 |
+
markdown2 2.5.0
|
275 |
+
MarkupSafe 2.1.5
|
276 |
+
matplotlib 3.9.2
|
277 |
+
matplotlib-inline 0.1.7
|
278 |
+
mbstrdecoder 1.1.3
|
279 |
+
mdit-py-plugins 0.4.1
|
280 |
+
mdurl 0.1.2
|
281 |
+
mistune 3.0.2
|
282 |
+
ml-dtypes 0.4.0
|
283 |
+
mpmath 1.3.0
|
284 |
+
msgpack 1.0.8
|
285 |
+
multidict 6.0.5
|
286 |
+
multiprocess 0.70.15
|
287 |
+
mutagen 1.47.0
|
288 |
+
mypy-extensions 1.0.0
|
289 |
+
namex 0.0.8
|
290 |
+
narwhals 1.6.2
|
291 |
+
nbclassic 1.1.0
|
292 |
+
nbclient 0.10.0
|
293 |
+
nbconvert 7.16.4
|
294 |
+
nbformat 5.10.4
|
295 |
+
nest-asyncio 1.6.0
|
296 |
+
networkx 3.3
|
297 |
+
ninja 1.11.1.1
|
298 |
+
nltk 3.9.1
|
299 |
+
nodeenv 1.9.1
|
300 |
+
notebook 7.2.2
|
301 |
+
notebook_shim 0.2.4
|
302 |
+
num2words 0.5.13
|
303 |
+
numba 0.60.0
|
304 |
+
numexpr 2.10.1
|
305 |
+
numpy 1.26.4
|
306 |
+
nvidia-cublas-cu12 12.4.5.8
|
307 |
+
nvidia-cuda-cupti-cu12 12.4.127
|
308 |
+
nvidia-cuda-nvrtc-cu12 12.4.127
|
309 |
+
nvidia-cuda-runtime-cu12 12.4.127
|
310 |
+
nvidia-cudnn-cu12 9.1.0.70
|
311 |
+
nvidia-cufft-cu12 11.2.1.3
|
312 |
+
nvidia-curand-cu12 10.3.5.147
|
313 |
+
nvidia-cusolver-cu12 11.6.1.9
|
314 |
+
nvidia-cusparse-cu12 12.3.1.170
|
315 |
+
nvidia-cutlass 3.5.1.0 /home/myles/cutlass
|
316 |
+
nvidia-ml-py 12.560.30
|
317 |
+
nvidia-nccl-cu12 2.21.5
|
318 |
+
nvidia-nvjitlink-cu12 12.4.127
|
319 |
+
nvidia-nvtx-cu12 12.4.127
|
320 |
+
nvidia-pyindex 1.0.9
|
321 |
+
open_clip_torch 2.26.1
|
322 |
+
openai 1.44.0
|
323 |
+
opencv-python 4.10.0.84
|
324 |
+
opencv-python-headless 4.10.0.84
|
325 |
+
openpyxl 3.1.5
|
326 |
+
opt-einsum 3.3.0
|
327 |
+
optree 0.12.1
|
328 |
+
orjson 3.10.7
|
329 |
+
overrides 7.7.0
|
330 |
+
packaging 24.1
|
331 |
+
pandas 2.2.2
|
332 |
+
pandocfilters 1.5.1
|
333 |
+
parso 0.8.4
|
334 |
+
pathlib2 2.3.7.post1
|
335 |
+
pathspec 0.12.1
|
336 |
+
pathvalidate 3.2.1
|
337 |
+
peft 0.12.0
|
338 |
+
pexpect 4.9.0
|
339 |
+
Pillow 10.1.0
|
340 |
+
pip 24.2
|
341 |
+
platformdirs 4.3.1
|
342 |
+
pluggy 1.5.0
|
343 |
+
ply 3.11
|
344 |
+
pooch 1.8.2
|
345 |
+
portalocker 2.10.1
|
346 |
+
pre-commit 3.8.0
|
347 |
+
prometheus_client 0.20.0
|
348 |
+
promise 2.3
|
349 |
+
prompt_toolkit 3.0.47
|
350 |
+
protobuf 4.25.4
|
351 |
+
psutil 6.0.0
|
352 |
+
ptyprocess 0.7.0
|
353 |
+
pure_eval 0.2.3
|
354 |
+
py 1.11.0
|
355 |
+
py-cpuinfo 9.0.0
|
356 |
+
py-spy 0.3.14
|
357 |
+
pyarrow 17.0.0
|
358 |
+
pyarrow-hotfix 0.6
|
359 |
+
pybind11 2.13.5
|
360 |
+
pycocoevalcap 1.2
|
361 |
+
pycocotools 2.0.8
|
362 |
+
pycparser 2.22
|
363 |
+
pycryptodomex 3.20.0
|
364 |
+
pydantic 2.9.0
|
365 |
+
pydantic_core 2.23.2
|
366 |
+
pydot 3.0.1
|
367 |
+
pydub 0.25.1
|
368 |
+
Pygments 2.18.0
|
369 |
+
PyJWT 2.9.0
|
370 |
+
pynndescent 0.5.13
|
371 |
+
pynvml 11.5.3
|
372 |
+
pyOpenSSL 24.2.1
|
373 |
+
pyparsing 3.1.4
|
374 |
+
pyproject-api 1.7.1
|
375 |
+
pytablewriter 1.2.0
|
376 |
+
pytest 8.3.2
|
377 |
+
python-consul 1.1.0
|
378 |
+
python-dateutil 2.9.0.post0
|
379 |
+
python-engineio 4.9.1
|
380 |
+
python-etcd 0.4.5
|
381 |
+
python-json-logger 2.0.7
|
382 |
+
python-multipart 0.0.9
|
383 |
+
python-socketio 5.11.4
|
384 |
+
pytorch-triton 3.0.0+757b6a61e7
|
385 |
+
pytz 2024.1
|
386 |
+
PyYAML 6.0.2
|
387 |
+
pyzmq 26.2.0
|
388 |
+
qtconsole 5.6.0
|
389 |
+
QtPy 2.4.1
|
390 |
+
rapidfuzz 3.9.7
|
391 |
+
referencing 0.35.1
|
392 |
+
regex 2024.7.24
|
393 |
+
requests 2.32.3
|
394 |
+
responses 0.25.3
|
395 |
+
rfc3339-validator 0.1.4
|
396 |
+
rfc3986-validator 0.1.1
|
397 |
+
rich 13.8.0
|
398 |
+
ring_flash_attn 0.1 /home/myles/ring-flash-attention
|
399 |
+
rouge_score 0.1.2
|
400 |
+
rpds-py 0.20.0
|
401 |
+
ruff 0.6.4
|
402 |
+
sacrebleu 2.4.3
|
403 |
+
safetensors 0.4.5
|
404 |
+
schedule 1.2.2
|
405 |
+
scikit-learn 1.5.1
|
406 |
+
scipy 1.14.1
|
407 |
+
semantic-version 2.10.0
|
408 |
+
Send2Trash 1.8.3
|
409 |
+
sentencepiece 0.2.0
|
410 |
+
sentry-sdk 2.13.0
|
411 |
+
setproctitle 1.3.3
|
412 |
+
setuptools 70.2.0
|
413 |
+
shellingham 1.5.4
|
414 |
+
shortuuid 1.0.13
|
415 |
+
shtab 1.7.1
|
416 |
+
simple-websocket 1.0.0
|
417 |
+
six 1.16.0
|
418 |
+
smmap 5.0.1
|
419 |
+
sniffio 1.3.1
|
420 |
+
sounddevice 0.5.0
|
421 |
+
soundfile 0.12.1
|
422 |
+
soupsieve 2.6
|
423 |
+
soxr 0.5.0.post1
|
424 |
+
sqlitedict 2.1.0
|
425 |
+
stack-data 0.6.3
|
426 |
+
starlette 0.38.4
|
427 |
+
svgwrite 1.4.3
|
428 |
+
sympy 1.13.1
|
429 |
+
tabledata 1.3.3
|
430 |
+
tabulate 0.9.0
|
431 |
+
tcolorpy 0.1.6
|
432 |
+
tenacity 9.0.0
|
433 |
+
tensorboard 2.17.1
|
434 |
+
tensorboard-data-server 0.7.2
|
435 |
+
tensorflow 2.17.0
|
436 |
+
termcolor 2.4.0
|
437 |
+
terminado 0.18.1
|
438 |
+
threadpoolctl 3.5.0
|
439 |
+
thriftpy2 0.5.2
|
440 |
+
tiktoken 0.7.0
|
441 |
+
timm 1.0.9
|
442 |
+
tinycss2 1.3.0
|
443 |
+
tokenizers 0.19.1
|
444 |
+
toml 0.10.2
|
445 |
+
tomli 2.0.1
|
446 |
+
tomlkit 0.12.0
|
447 |
+
toolz 0.12.1
|
448 |
+
torch 2.5.0.dev20240907+cu124
|
449 |
+
torchaudio 2.5.0.dev20240907+cu124
|
450 |
+
torchvision 0.20.0.dev20240907+cu124
|
451 |
+
tornado 6.4.1
|
452 |
+
tox 4.18.1
|
453 |
+
tqdm 4.66.5
|
454 |
+
tqdm-multiprocess 0.0.11
|
455 |
+
traitlets 5.14.3
|
456 |
+
transformer-lens 2.4.1
|
457 |
+
transformers 4.45.0.dev0 /home/myles/transformers
|
458 |
+
transformers-stream-generator 0.0.5
|
459 |
+
treelib 1.7.0
|
460 |
+
triton 3.0.0
|
461 |
+
typeguard 2.13.3
|
462 |
+
typepy 1.3.2
|
463 |
+
typer 0.12.5
|
464 |
+
types-python-dateutil 2.9.0.20240906
|
465 |
+
typing_extensions 4.12.2
|
466 |
+
tyro 0.8.10
|
467 |
+
tzdata 2024.1
|
468 |
+
uc-micro-py 1.0.3
|
469 |
+
umap-learn 0.5.6
|
470 |
+
Unidecode 1.3.8
|
471 |
+
uri-template 1.3.0
|
472 |
+
urllib3 2.2.2
|
473 |
+
uvicorn 0.30.6
|
474 |
+
virtualenv 20.26.4
|
475 |
+
wandb 0.17.9
|
476 |
+
watchdog 5.0.2
|
477 |
+
wavedrom 2.0.3.post3
|
478 |
+
wcwidth 0.2.13
|
479 |
+
webcolors 24.8.0
|
480 |
+
webencodings 0.5.1
|
481 |
+
websocket-client 1.8.0
|
482 |
+
websockets 12.0
|
483 |
+
Werkzeug 3.0.4
|
484 |
+
wheel 0.44.0
|
485 |
+
widgetsnbextension 4.0.13
|
486 |
+
wrapt 1.16.0
|
487 |
+
wsproto 1.2.0
|
488 |
+
xxhash 3.5.0
|
489 |
+
y-py 0.6.2
|
490 |
+
yarl 1.10.0
|
491 |
+
ypy-websocket 0.8.4
|
492 |
+
yt-dlp 2024.8.6
|
493 |
+
zipp 3.20.1
|
494 |
+
zss 1.2.0
|
495 |
+
zstandard 0.23.0
|