Trying to handle endpoint errors

src/backend/inference_endpoint.py

@@ -20,32 +20,38 @@ def create_endpoint(endpoint_name, repository, framework="pytorch", task="text-g
         logger.debug("Hit the following exception:")
         logger.debug(e)
         logger.debug("Attempting to continue.")
-        try:
-            endpoint = get_inference_endpoint(endpoint_name)
-            endpoint.update(repository=repository, framework=framework, task=task, accelerator=accelerator,  instance_size=instance_size, instance_type=instance_type)
-        except:
-            if instance_type == "nvidia-l4":
-                # Try a larger, different, more expensive GPU.
-                endpoint = create_inference_endpoint(endpoint_name,
-                                                     repository=repository,
-                                                     framework=framework, task=task,
-                                                     accelerator=accelerator,
-                                                     vendor=vendor, region=region,
-                                                     type=type,
-                                                     instance_size="x1",
-                                                     instance_type="nvidia-a100")
-            elif instance_type == "a100" and instance_size == "x1":
-                endpoint = create_inference_endpoint(endpoint_name,
-                                                     repository=repository,
-                                                     framework=framework, task=task,
-                                                     accelerator=accelerator,
-                                                     vendor=vendor, region=region,
-                                                     type=type,
-                                                     instance_size="x4",
-                                                     instance_type="nvidia-a10g")
-            else:
-                logger.info("Getting expensive to run this model without human oversight. Exiting.")
-                sys.exit()
+        endpoint = get_inference_endpoint(endpoint_name)
+        endpoint.update(repository=repository, framework=framework, task=task, accelerator=accelerator,  instance_size=instance_size, instance_type=instance_type)
+    except huggingface_hub.utils._errors.BadRequestError as e:
+        logger.debug("Hit the following exception:")
+        logger.debug(e)
+        logger.debug("Attempting a new instance type.")
+        if instance_type == "nvidia-l4":
+            # Try a larger, different, more expensive GPU.
+            endpoint = create_inference_endpoint(endpoint_name,
+                                                 repository=repository,
+                                                 framework=framework, task=task,
+                                                 accelerator=accelerator,
+                                                 vendor=vendor, region=region,
+                                                 type=type,
+                                                 instance_size="x1",
+                                                 instance_type="nvidia-a100")
+        elif instance_type == "a100" and instance_size == "x1":
+            endpoint = create_inference_endpoint(endpoint_name,
+                                                 repository=repository,
+                                                 framework=framework, task=task,
+                                                 accelerator=accelerator,
+                                                 vendor=vendor, region=region,
+                                                 type=type,
+                                                 instance_size="x4",
+                                                 instance_type="nvidia-a10g")
+        else:
+            logger.info("Getting expensive to try to run this model without human oversight. Exiting.")
+            sys.exit()
+    except Exception as e:
+        logger.debug("Hit error")
+        logger.debug(e)
+        sys.exit()
 
     endpoint.fetch()
     logger.info("Endpoint status: %s." % (endpoint.status))