Spaces:
Running
Running
Update convert.py
Browse files- convert.py +17 -0
convert.py
CHANGED
@@ -12,6 +12,10 @@ from huggingface_hub import CommitInfo, CommitOperationAdd, Discussion, HfApi, h
|
|
12 |
from huggingface_hub.file_download import repo_folder_name
|
13 |
from safetensors.torch import _find_shared_tensors, _is_complete, load_file, save_file
|
14 |
|
|
|
|
|
|
|
|
|
15 |
|
16 |
COMMIT_DESCRIPTION = """
|
17 |
This is an automated PR created with https://huggingface.co/spaces/safetensors/convert
|
@@ -292,22 +296,35 @@ def convert(
|
|
292 |
new_pr = pr
|
293 |
raise AlreadyExists(f"Model {model_id} already has an open PR check out {url}")
|
294 |
elif library_name == "transformers":
|
|
|
295 |
|
296 |
discard_names = get_discard_names(model_id, revision=revision, folder=folder, token=api.token)
|
|
|
|
|
297 |
if "pytorch_model.bin" in filenames:
|
|
|
298 |
operations, errors = convert_single(
|
299 |
model_id, revision=revision, folder=folder, token=api.token, discard_names=discard_names
|
300 |
)
|
|
|
|
|
301 |
elif "pytorch_model.bin.index.json" in filenames:
|
|
|
302 |
operations, errors = convert_multi(
|
303 |
model_id, revision=revision, folder=folder, token=api.token, discard_names=discard_names
|
304 |
)
|
|
|
|
|
305 |
else:
|
|
|
306 |
raise RuntimeError(f"Model {model_id} doesn't seem to be a valid pytorch model. Cannot convert")
|
307 |
else:
|
|
|
308 |
operations, errors = convert_generic(
|
309 |
model_id, revision=revision, folder=folder, filenames=filenames, token=api.token
|
310 |
)
|
|
|
|
|
311 |
|
312 |
if operations:
|
313 |
new_pr = api.create_commit(
|
|
|
12 |
from huggingface_hub.file_download import repo_folder_name
|
13 |
from safetensors.torch import _find_shared_tensors, _is_complete, load_file, save_file
|
14 |
|
15 |
+
from transformers.utils import logging
|
16 |
+
|
17 |
+
logging.set_verbosity_info()
|
18 |
+
logger = logging.get_logger("transformers")
|
19 |
|
20 |
COMMIT_DESCRIPTION = """
|
21 |
This is an automated PR created with https://huggingface.co/spaces/safetensors/convert
|
|
|
296 |
new_pr = pr
|
297 |
raise AlreadyExists(f"Model {model_id} already has an open PR check out {url}")
|
298 |
elif library_name == "transformers":
|
299 |
+
logger.info(f"Detected library: {library_name} for model: {model_id}, starting conversion.")
|
300 |
|
301 |
discard_names = get_discard_names(model_id, revision=revision, folder=folder, token=api.token)
|
302 |
+
logger.debug(f"Discard names for {model_id}: {discard_names}")
|
303 |
+
|
304 |
if "pytorch_model.bin" in filenames:
|
305 |
+
logger.info(f"Found 'pytorch_model.bin' for {model_id}, proceeding with single file conversion.")
|
306 |
operations, errors = convert_single(
|
307 |
model_id, revision=revision, folder=folder, token=api.token, discard_names=discard_names
|
308 |
)
|
309 |
+
if errors:
|
310 |
+
logger.error(f"Errors occurred during single file conversion for {model_id}: {errors}")
|
311 |
elif "pytorch_model.bin.index.json" in filenames:
|
312 |
+
logger.info(f"Found 'pytorch_model.bin.index.json' for {model_id}, proceeding with multi file conversion.")
|
313 |
operations, errors = convert_multi(
|
314 |
model_id, revision=revision, folder=folder, token=api.token, discard_names=discard_names
|
315 |
)
|
316 |
+
if errors:
|
317 |
+
logger.error(f"Errors occurred during multi file conversion for {model_id}: {errors}")
|
318 |
else:
|
319 |
+
logger.error(f"Model {model_id} doesn't seem to be a valid pytorch model. Cannot convert")
|
320 |
raise RuntimeError(f"Model {model_id} doesn't seem to be a valid pytorch model. Cannot convert")
|
321 |
else:
|
322 |
+
logger.info(f"No specific library matched for {model_id}, using generic conversion method.")
|
323 |
operations, errors = convert_generic(
|
324 |
model_id, revision=revision, folder=folder, filenames=filenames, token=api.token
|
325 |
)
|
326 |
+
if errors:
|
327 |
+
logger.error(f"Errors occurred during generic conversion for {model_id}: {errors}")
|
328 |
|
329 |
if operations:
|
330 |
new_pr = api.create_commit(
|