fully app first commit

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+save_model/best_model.pth filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,91 @@
+import gradio as gr
+import torch
+import torch.nn as nn
+import onnx
+import engine, data, utils
+from typing import Tuple, Dict
+from train import NUM_CLASSES
+from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+import torchvision.transforms as T
+import onnxruntime as ort
+import numpy as np
+from timeit import default_timer as timer
+from pathlib import Path
+
+PATH = "save_model/food_cpu.onnx"
+
+
+model = efficientnet_b0(weights=EfficientNet_B0_Weights.DEFAULT)
+model.classifier = nn.Sequential(
+                                    nn.Dropout(p = 0.2, inplace = True),
+                                    nn.Linear(1280, NUM_CLASSES),
+                                    # nn.Softmax()
+                                )
+model = utils.load_model(model, "save_model/best_model.pth")
+
+utils.onnx_inference(model, PATH, "cpu")
+onnx_model = onnx.load(PATH)
+onnx_check = onnx.checker.check_model(onnx_model)
+
+classes = data.train_datasets.classes
+
+trn = T.ToPILImage()
+
+def predict(img) -> Tuple[Dict, float]:
+    """Transforms and performs a prediction on img and return prediction and time take."""
+    # Start the timer
+    start_time = timer()
+
+    # transform the target image and add a batch dimension
+    img = data.transform(img).unsqueeze(dim = 0)
+
+    # inference using onnx
+    ort_sess = ort.InferenceSession(PATH)
+    outputs = ort_sess.run(None, {'input': img.numpy()})
+
+    predicted = classes[outputs[0][0].argmax(0)]
+
+    # print("\n", outputs[0][0], "\n")
+    outputs = np.array(torch.softmax(torch.from_numpy(outputs[0]), dim = 1))
+    pred_labels_and_prob = {classes[i]: float(outputs[0][i]) for i in range(len(classes))}
+    # print(f'Predicted: "{predicted}"')
+
+    # Calculate the predicion time
+    pred_time = round(timer() - start_time, 5)
+
+    # Return the prediction dictionary and prediction time
+    return pred_labels_and_prob, pred_time
+
+
+image = trn(data.test_datasets[3][0])
+
+exp_dir = "./example_data/"
+test_data_paths = list(Path(exp_dir).glob("*.jpg"))
+# print(test_data_paths)
+example_list = [[str(filepath)] for filepath in test_data_paths]
+# print(example_list)
+
+
+# pred_dict, pred_time = predict(img = image)
+# print(f"Predicted label and probability: {pred_dict}")
+# print(f"Prediction time: {pred_time}")
+
+title = "FoodVision 🍕🥩🍣"
+description = "An EfficientNetB2 feature extractor computer vision model to classify images of food as pizza, steak or sushi."
+article = "Created at [09. PyTorch Model Deployment](https://www.learnpytorch.io/09_pytorch_model_deployment/)."
+
+
+# Create the Gradio demo
+demo = gr.Interface(fn=predict, # mapping function from input to output
+                    inputs=gr.Image(type="pil"), # what are the inputs?
+                    outputs=[gr.Label(num_top_classes=101, label="Predictions"), # what are the outputs?
+                             gr.Number(label="Prediction time (s)")], # our fn has two outputs, therefore we have two outputs
+                    examples=example_list, 
+                    title=title,
+                    description=description,
+                    article=article)
+
+# Launch the demo!
+# demo.launch(debug=False, # print errors locally?
+#             share=True) # generate a publically shareable URL?
+demo.launch(share=True)

data.py

@@ -0,0 +1,35 @@
+import torch
+import torchvision
+from torchvision.datasets import Food101, MNIST
+from torch.utils.data import Subset, DataLoader
+import torchvision.transforms as transforms
+
+transform = transforms.Compose([
+    transforms.Resize((224, 224)),
+    transforms.ToTensor(),
+    transforms.Normalize(mean=[0.485, 0.456, 0.406],
+                         std=[0.229, 0.224, 0.225]),
+])
+
+split = 0.25
+batch_size = 32
+
+train_datasets = Food101(root = "./data", split = "train", transform = transform, download = True)
+test_datasets = Food101(root = "./data", split = "test", transform = transform, download = True)
+
+num_samples_train = int(split * len(train_datasets))
+train_indices = torch.randperm(len(train_datasets))[:num_samples_train]
+
+num_samples_valid = int(split * len(train_datasets))
+valid_indices = torch.randperm(len(train_datasets))[:num_samples_valid]
+
+num_samples_test = int(split * len(test_datasets))
+test_indices = torch.randperm(len(test_datasets))[:num_samples_test]
+
+split_tarin_data = Subset(train_datasets, train_indices)
+split_valid_data = Subset(train_datasets, valid_indices)
+split_test_data = Subset(test_datasets, test_indices)
+
+train_dataloader = DataLoader(split_tarin_data, batch_size = batch_size, shuffle = True, drop_last = True)
+valid_dataloader = DataLoader(split_valid_data, batch_size = batch_size, shuffle = True, drop_last = True)
+test_dataloader = DataLoader(split_test_data, batch_size = batch_size, shuffle = False, drop_last = True)

engine.py

@@ -0,0 +1,203 @@
+import torch
+import wandb
+from tqdm.auto import tqdm
+
+API_KEY = "881252af31786a1cf813449b9b4124955f54703e"
+
+
+def train_loop(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module,
+               optimizer: torch.optim.Optimizer, accuracy_fn, device: torch.device):
+
+    """
+    Function for Model Training
+
+    Args:
+        model: A pytorch model you want to train
+        dataloader: A dataloader for intance for model training
+        loss_fn: A loss function for calculate model loss
+        accuracy_fn: A Accuracy function to check how model is accurate
+        device: A device on which model run i.e.: "cuda" or "cpu"
+
+    Return:
+        list of train loss and accuracy and also model weights
+
+    Example usage:
+        train_loop(model = mymodel, dataloader = train_dataloader, loss_fn = loss_fn, 
+                    accuracy_fn = accuracy_fn, device = device)
+    """
+
+    train_loss, train_acc = 0, 0 
+
+    model.train()
+
+    for batch, (x_train, y_train) in enumerate(dataloader):
+        x_train, y_train = x_train.to(device), y_train.to(device)
+
+        # 1. Forwrad Pass
+        logits = model(x_train)
+
+        # 2. Loss
+        loss = loss_fn(logits, y_train)
+
+        # 3. Gradzero step
+        optimizer.zero_grad()
+
+        # 4. Backward
+        loss.backward()
+
+        # 5. Optimizer step
+        optimizer.step()
+
+        acc = accuracy_fn(torch.argmax(logits, dim = 1), y_train)
+
+        train_acc += acc
+        train_loss += loss
+
+    train_loss /= len(dataloader)
+    train_acc /= len(dataloader)
+
+    return model, train_loss, train_acc
+
+
+def test_loop(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module,
+              accuracy_fn, device: torch.device):
+
+    """
+    A Funtion to test the model after traininig
+
+    Args:
+        model: A model which you want to test on test intance
+        dataloader: A dataloader intance on which you test model
+        loss_fn: A loss function to calculate the model loss
+        accuracy_fn : A accuracy function to check model accuracy on dataloader intance
+        device: A device on whic you want to run model i.e.: "cuda" or "cpu"
+
+    Return:
+        A list of test loss and Accuracy
+
+    Example Usage:
+        test_loop(model = mymodel, dataloader = test_datloader, loss_fn = loss_fn,
+                  accuracy_fn = accuracy+fn, device = device)
+    """
+
+    test_loss, test_acc = 0, 0
+
+    model.eval()
+    with torch.inference_mode():
+        for x_test, y_test in dataloader:
+            x_test, y_test = x_test.to(device), y_test.to(device)
+
+            # 1. Forward Pass
+            logits = model(x_test)
+
+            # 2. Loss
+            test_loss += loss_fn(logits, y_test)
+
+            test_acc += accuracy_fn(torch.argmax(logits, dim = 1), y_test)
+
+        test_acc /= len(dataloader)
+        test_loss /= len(dataloader)
+
+    return test_loss, test_acc
+
+
+def log_image_table(images, predicted, labels, probs):
+    "Log a wandb.Table with (img, pred, target, scores)"
+    wandb.login(key=API_KEY)
+    print("[LOG]: Login Succesfull.")
+    # 🐝 Create a wandb Table to log images, labels and predictions to
+    table = wandb.Table(columns=["image", "pred", "target"]+[f"score_{i}" for i in range(10)])
+    for img, pred, targ, prob in zip(images.to("cpu"), predicted.to("cpu"), labels.to("cpu"), probs.to("cpu")):
+        table.add_data(wandb.Image(img[0].numpy()*255), pred, targ, *prob.numpy())
+    wandb.log({"predictions_table":table}, commit=False)
+
+
+
+def validation(model: torch.nn.Module, dataloader: torch.utils.data.DataLoader, loss_fn: torch.nn.Module,
+               accuracy_fn, log_images: bool, device: torch.device, batch_idx=0):
+    """
+    A Function for hyperparameter tuning
+
+    Args:
+        model: A model the you want tune its hyperparameter
+        dataloder: Adataloader intance for hyperparameter tuning
+        loss_fn: A loss funtion to calcualte model loss
+        Accuracy_fn: a accuracy function to calcultae accuracy for model perforamnce
+        device: A device on which model run i.e.: "cuda" or "cpu"
+
+    Return:
+        A list of accuracy and loss
+
+    Example usage:
+        validation(model = mymodel, dataloader = valid_dataloader, loss_fn = loss_fn,
+                   accuracy_fn = accuracy_fn, device = device)
+    """
+
+    val_loss, val_acc = 0, 0
+
+    model.eval()
+    with torch.inference_mode():
+        for i, (x_val, y_val) in enumerate(dataloader):
+            x_val, y_val = x_val.to(device), y_val.to(device)
+
+            logits = model(x_val)
+
+            val_loss += loss_fn(logits, y_val)
+
+            val_acc += accuracy_fn(torch.argmax(logits, dim = 1), y_val)
+
+            # Log one batch of images to the dashboard, always same batch_idx.
+            if i==batch_idx and log_images:
+                log_image_table(x_val, torch.max(logits.data, 1)[0], y_val, logits.softmax(dim=1))
+
+        val_loss /= len(dataloader)
+        val_acc /= len(dataloader)
+
+    return val_loss, val_acc
+
+def train(model: torch.nn.Module, train_dataloader: torch.utils.data.DataLoader, test_dataloader: torch.utils.data.DataLoader,
+          optimizer: torch.optim.Optimizer, loss_fn: torch.nn.Module, accuracy_fn, epochs: int, device: torch.device):
+
+    """
+    A function to train and test the pytorch model
+
+    Args:
+        model: A model to train and test
+        train_dataloader: A dataloader intance for train model
+        test_dataloader: A dataloader intance for test model
+        optimizer: A optimizer funtion to optimize the model
+        loss_fn: A loss function to calculate model loss
+        accuracy_fn: An accuracy  to calculate model performance
+        epochs: number of iteration to run the loop
+        device: A device on which model run i.e.: "cuda" or "cpu"
+
+    Return:
+        train model, List of train and test losses and accuracy
+
+    Example usage:
+        train(model = mymodel, train_dataloader = train_dataloader, test_dataloader = test_dataloader, optimizer = optimizer,
+              loss_fn = loss_fn, acuuracy_fn = accuracy_fn, epochs = epochs, device = device)
+    """
+
+    train_losses, test_losses = [], []
+    train_accs, test_accs = [], []
+
+    for epoch in tqdm(range(epochs)):
+
+        print(f"\nEpoch: {epoch+1}")
+
+        train_model, train_loss, train_acc = train_loop(model = model, dataloader = train_dataloader, 
+                                                        loss_fn = loss_fn, optimizer = optimizer, 
+                                                        accuracy_fn = accuracy_fn, device = device)
+
+        test_loss, test_acc = test_loop(model = model, dataloader = test_dataloader, loss_fn = loss_fn,
+                                        accuracy_fn = accuracy_fn, device = device)
+
+        print(f"Train Loss: {train_loss:.5f} | Test Loss: {test_loss:.5f} || Train Accuracy: {train_acc:.5f} | Test Accuracy: {test_acc:.5f}")
+
+        train_losses.append(train_loss.item())
+        test_losses.append(test_loss.item())
+        train_accs.append(train_acc.item())
+        test_accs.append(test_acc.item())
+
+    return train_losses, test_losses, train_accs, test_accs, train_model 

hyperpara.py

@@ -0,0 +1,127 @@
+import os
+import torch
+import torch.nn as nn
+import optuna
+from optuna.trial import TrialState
+from torch import optim
+import engine, data, utils
+from train import device, LEARNIGN_RATE, NUM_EPOCH, NUM_CLASSES
+from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+from torchmetrics.classification import MulticlassAccuracy
+
+import wandb
+
+# WanDB login
+
+API_KEY = "881252af31786a1cf813449b9b4124955f54703e"
+
+wandb.login(key=API_KEY)
+print("[LOG]: Login Succesfull.")
+
+
+
+
+def objective(trial, n_trials=100):
+    """Objective function to be optimized by Optuna.
+    Hyperparameters chosen to be optimized: optimizer, learning rate,
+    dropout values, number of convolutional layers, number of filters of
+    convolutional layers, number of neurons of fully connected layers.
+    Inputs:
+        - trial (optuna.trial._trial.Trial): Optuna trial
+    Returns:
+        - accuracy(torch.Tensor): The test accuracy. Parameter to be maximized.
+    """
+
+    lr = trial.suggest_float("lr", LEARNIGN_RATE*1e-2, LEARNIGN_RATE, log=True)                                 # Learning rates
+    n_epochs = trial.suggest_int('n_estimators', NUM_EPOCH//2, NUM_EPOCH)
+
+    optimizer_name = trial.suggest_categorical("optimizer", ["Adam", "RMSprop", "SGD"])
+    optimizer = getattr(optim, optimizer_name)(model.parameters(), lr=lr) #torch.optim.Adam(model.parameters(), lr = lr)# getattr(optim, optimizer_name)(model.parameters(), lr=lr)
+
+
+    loss_fn = torch.nn.CrossEntropyLoss()
+    accuracy_fn = MulticlassAccuracy(num_classes = NUM_CLASSES).to(device)
+
+    wandb.init(
+                    # set the wandb project where this run will be logged
+                    project="food-app",
+                    
+                    # track hyperparameters and run metadata
+                    config={
+                    "optimizer": trial.params["optimizer"],
+                    "architecture": "Efficientnet B0",
+                    "dataset": "Food101",
+                    "epochs": trial.params["n_estimators"],
+                    }
+                )
+
+    # Training of the model
+    best_loss = 100
+    patience = 5
+    early_stop = 0
+    for epoch in range(n_epochs):
+
+        train_model, train_loss, train_acc = engine.train_loop(model = model, dataloader = data.train_dataloader, 
+                                                        loss_fn = loss_fn, optimizer = optimizer, 
+                                                        accuracy_fn = accuracy_fn, device = device)
+
+        val_loss, val_acc = engine.validation(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn,
+                                        accuracy_fn = accuracy_fn, log_images=(epoch==(wandb.config.epochs-1)), device = device)
+        
+        
+        # For pruning (stops trial early if not promising)
+        trial.report(val_acc, epoch)
+        # Handle pruning based on the intermediate value.
+        if trial.should_prune():
+            raise optuna.exceptions.TrialPruned()
+
+        if val_acc < best_loss:
+            early_stop = 0
+            best_loss = val_acc
+            utils.save_model(model = train_model, target_dir = "./save_model", model_name = f"best_model.pth")
+
+        else:
+            early_stop += 1
+
+        if early_stop == patience:
+            break
+
+    wandb.log({"Train Loss": train_loss, "Train Accuracy": train_acc, "Validation Loss": val_loss, "Validation Accuracy": val_acc})
+
+    wandb.finish()
+
+    return val_acc
+
+
+model = efficientnet_b0(weights=EfficientNet_B0_Weights.DEFAULT)
+model.classifier = nn.Sequential(
+                                    nn.Dropout(p = 0.2, inplace = True),
+                                    nn.Linear(1280, NUM_CLASSES),
+                                    # nn.Softmax()
+                                )
+
+# isExist = os.path.exists("save_model/train_model_4e-06.pth")
+# print(isExist)
+model = utils.load_model(model, "save_model/train_model_4e-06.pth").to(device)
+# print(model)
+
+# Create an Optuna study to maximize test accuracy
+study = optuna.create_study(direction="maximize")
+study.optimize(objective, n_trials=11)
+
+# Find number of pruned and completed trials
+pruned_trials = study.get_trials(deepcopy=False, states=[TrialState.PRUNED])
+complete_trials = study.get_trials(deepcopy=False, states=[TrialState.COMPLETE])
+
+# Display the study statistics
+print("\nStudy statistics: ")
+print("  Number of finished trials: ", len(study.trials))
+print("  Number of pruned trials: ", len(pruned_trials))
+print("  Number of complete trials: ", len(complete_trials))
+
+trial = study.best_trial
+print("Best trial:")
+print("  Value: ", trial.value)
+print("  Params: ")
+for key, value in trial.params.items():
+    print("    {}: {}".format(key, value))

inference.py

@@ -0,0 +1,36 @@
+import torch
+import torch.nn as nn
+
+import onnx
+import engine, data, utils
+from train import device, NUM_CLASSES
+from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+from torchmetrics.classification import MulticlassAccuracy
+import onnxruntime as ort
+import numpy as np
+
+
+
+model = efficientnet_b0(weights=EfficientNet_B0_Weights.IMAGENET1K_V1)
+model.classifier = nn.Sequential(
+                                    nn.Dropout(p = 0.2, inplace = True),
+                                    nn.Linear(1280, NUM_CLASSES),
+                                    # nn.Softmax()
+                                )
+model = utils.load_model(model, "save_model/best_model.pth").to(device)
+
+PATH = "save_model/food_cpu.onnx"
+# onnx inference
+utils.onnx_inference(model, PATH, "cpu")
+onnx_model = onnx.load(PATH)
+onnx_check = onnx.checker.check_model(onnx_model)
+
+# print(onnx_check)
+x, y = data.test_datasets[0][0], data.test_datasets[0][1]
+ort_sess = ort.InferenceSession(PATH)
+outputs = ort_sess.run(None, {'input': x.unsqueeze(dim = 0).numpy()})
+
+# Result
+classes = data.train_datasets.classes
+predicted, actual = classes[outputs[0][0].argmax(0)], classes[y]
+print(f'Predicted: "{predicted}", Actual: "{actual}"')

requirements.txt

@@ -0,0 +1,10 @@
+pytorch==2.0.0 
+torchvision==0.15.0 
+torchaudio==2.0.0
+torchmetrics
+onnx
+onnxruntime
+optuna
+wandb
+torch-summary
+matplotlib

results/hyp_tune.txt

@@ -0,0 +1,606 @@
+[I 2023-10-22 01:37:08,918] A new study created in memory with name: no-name-c840de1b-897a-43af-bd8e-df563b179162
+[I 2023-10-22 02:46:09,630] Trial 0 finished with value: 0.5360714197158813 and parameters: {'lr': 7.489632455796532e-09, 'n_estimators': 34, 'optimizer': 'SGD'}. Best is trial 0 with value: 0.5360714197158813.
+[I 2023-10-22 03:30:52,975] Trial 1 finished with value: 0.5364061594009399 and parameters: {'lr': 1.3698661828277252e-08, 'n_estimators': 25, 'optimizer': 'Adam'}. Best is trial 1 with value: 0.5364061594009399.
+[I 2023-10-22 03:59:25,244] Trial 2 finished with value: 0.5400202870368958 and parameters: {'lr': 1.813677229052636e-07, 'n_estimators': 25, 'optimizer': 'Adam'}. Best is trial 2 with value: 0.5400202870368958.
+[I 2023-10-22 04:32:04,511] Trial 3 finished with value: 0.5361357927322388 and parameters: {'lr': 8.336261616991292e-08, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 2 with value: 0.5400202870368958.
+[I 2023-10-22 04:56:30,858] Trial 4 finished with value: 0.5398440361022949 and parameters: {'lr': 3.8570428767211705e-07, 'n_estimators': 14, 'optimizer': 'Adam'}. Best is trial 2 with value: 0.5400202870368958.
+[I 2023-10-22 05:57:25,530] Trial 5 finished with value: 0.5384387969970703 and parameters: {'lr': 8.89697701761264e-09, 'n_estimators': 37, 'optimizer': 'SGD'}. Best is trial 2 with value: 0.5400202870368958.
+[I 2023-10-22 06:33:16,334] Trial 6 finished with value: 0.540566623210907 and parameters: {'lr': 5.558156142085502e-08, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 6 with value: 0.540566623210907.
+[I 2023-10-22 07:00:35,226] Trial 7 finished with value: 0.5460574626922607 and parameters: {'lr': 1.90286747029132e-07, 'n_estimators': 12, 'optimizer': 'Adam'}. Best is trial 7 with value: 0.5460574626922607.
+[I 2023-10-22 07:23:51,994] Trial 8 finished with value: 0.5440860390663147 and parameters: {'lr': 9.886624025118481e-09, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 7 with value: 0.5460574626922607.
+[I 2023-10-22 07:58:56,022] Trial 9 finished with value: 0.5461758971214294 and parameters: {'lr': 1.2078859717026416e-07, 'n_estimators': 26, 'optimizer': 'Adam'}. Best is trial 9 with value: 0.5461758971214294.
+[I 2023-10-22 08:45:40,341] Trial 10 finished with value: 0.5450535416603088 and parameters: {'lr': 3.411775194205487e-08, 'n_estimators': 20, 'optimizer': 'RMSprop'}. Best is trial 9 with value: 0.5461758971214294.
+[I 2023-10-22 09:08:56,407] Trial 11 finished with value: 0.5503671765327454 and parameters: {'lr': 1.2684791385173414e-07, 'n_estimators': 12, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 09:32:17,407] Trial 12 finished with value: 0.5460602045059204 and parameters: {'lr': 1.0571866651675711e-07, 'n_estimators': 19, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 09:59:36,467] Trial 13 finished with value: 0.5463402271270752 and parameters: {'lr': 3.143297264441004e-08, 'n_estimators': 50, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 10:26:49,809] Trial 14 finished with value: 0.5461093783378601 and parameters: {'lr': 3.0821157193882773e-08, 'n_estimators': 50, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 11:23:12,523] Trial 15 finished with value: 0.5502772331237793 and parameters: {'lr': 1.9747824446554313e-08, 'n_estimators': 32, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 12:08:08,005] Trial 16 finished with value: 0.5461443066596985 and parameters: {'lr': 1.8079689971330616e-08, 'n_estimators': 31, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 12:44:55,748] Trial 17 finished with value: 0.548389196395874 and parameters: {'lr': 1.8707046412210653e-08, 'n_estimators': 18, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 13:21:34,306] Trial 18 finished with value: 0.5481426119804382 and parameters: {'lr': 5.407206770131877e-08, 'n_estimators': 36, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 14:18:47,176] Trial 19 finished with value: 0.5486733317375183 and parameters: {'lr': 4.5994986446708456e-09, 'n_estimators': 30, 'optimizer': 'RMSprop'}. Best is trial 11 with value: 0.5503671765327454.
+[I 2023-10-22 15:03:22,294] Trial 20 finished with value: 0.5505406856536865 and parameters: {'lr': 4.3839971014050346e-08, 'n_estimators': 11, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 15:27:38,957] Trial 21 finished with value: 0.5479753613471985 and parameters: {'lr': 5.284735869059107e-08, 'n_estimators': 10, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 15:52:15,068] Trial 22 finished with value: 0.5478231310844421 and parameters: {'lr': 2.3240425242663824e-08, 'n_estimators': 15, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 16:16:38,410] Trial 23 finished with value: 0.5491703748703003 and parameters: {'lr': 4.4870264812671536e-08, 'n_estimators': 21, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 17:13:33,420] Trial 24 finished with value: 0.5481324195861816 and parameters: {'lr': 7.469682637842191e-08, 'n_estimators': 14, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 17:51:02,788] Trial 25 finished with value: 0.5498676300048828 and parameters: {'lr': 2.64698690852027e-08, 'n_estimators': 10, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 18:28:07,513] Trial 26 finished with value: 0.5503433346748352 and parameters: {'lr': 3.70518809559498e-08, 'n_estimators': 29, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 19:21:42,279] Trial 27 finished with value: 0.5504162907600403 and parameters: {'lr': 4.074495301401983e-08, 'n_estimators': 17, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 20:02:40,205] Trial 28 finished with value: 0.549625813961029 and parameters: {'lr': 4.287480774476896e-08, 'n_estimators': 16, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 20:52:10,303] Trial 29 finished with value: 0.5462378263473511 and parameters: {'lr': 6.806050743074082e-08, 'n_estimators': 12, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 22:02:28,677] Trial 30 finished with value: 0.5487064123153687 and parameters: {'lr': 9.595182919683205e-08, 'n_estimators': 17, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 22:27:17,590] Trial 31 finished with value: 0.5490727424621582 and parameters: {'lr': 3.789547094688714e-08, 'n_estimators': 22, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-22 23:04:22,026] Trial 32 finished with value: 0.5467727780342102 and parameters: {'lr': 6.407275318372976e-08, 'n_estimators': 27, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-23 00:31:00,911] Trial 33 finished with value: 0.5485678911209106 and parameters: {'lr': 3.542292563055157e-08, 'n_estimators': 23, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-23 01:18:08,441] Trial 34 finished with value: 0.5498937964439392 and parameters: {'lr': 4.538312987041805e-08, 'n_estimators': 12, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-23 01:41:30,297] Trial 35 finished with value: 0.5497119426727295 and parameters: {'lr': 2.6255028232030936e-08, 'n_estimators': 14, 'optimizer': 'RMSprop'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-23 02:24:02,801] Trial 36 finished with value: 0.5490474700927734 and parameters: {'lr': 1.3889162318307476e-08, 'n_estimators': 28, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-23 02:51:04,062] Trial 37 finished with value: 0.5489190816879272 and parameters: {'lr': 1.557848733105634e-07, 'n_estimators': 23, 'optimizer': 'SGD'}. Best is trial 20 with value: 0.5505406856536865.
+[I 2023-10-23 03:18:20,869] Trial 38 finished with value: 0.550723135471344 and parameters: {'lr': 7.232390810885362e-08, 'n_estimators': 39, 'optimizer': 'RMSprop'}. Best is trial 38 with value: 0.550723135471344.
+[I 2023-10-23 04:51:40,915] Trial 39 finished with value: 0.547078013420105 and parameters: {'lr': 8.156050875435005e-08, 'n_estimators': 40, 'optimizer': 'SGD'}. Best is trial 38 with value: 0.550723135471344.
+[I 2023-10-23 05:42:10,595] Trial 40 finished with value: 0.5495409369468689 and parameters: {'lr': 8.915478213601037e-08, 'n_estimators': 34, 'optimizer': 'Adam'}. Best is trial 38 with value: 0.550723135471344.
+[I 2023-10-23 06:21:10,896] Trial 41 finished with value: 0.5536542534828186 and parameters: {'lr': 5.934005291949064e-08, 'n_estimators': 38, 'optimizer': 'RMSprop'}. Best is trial 41 with value: 0.5536542534828186.
+[I 2023-10-23 06:48:23,716] Trial 42 finished with value: 0.552158534526825 and parameters: {'lr': 6.448909468854163e-08, 'n_estimators': 40, 'optimizer': 'RMSprop'}. Best is trial 41 with value: 0.5536542534828186.
+[I 2023-10-23 07:15:32,410] Trial 43 finished with value: 0.5526768565177917 and parameters: {'lr': 6.259029563811938e-08, 'n_estimators': 40, 'optimizer': 'RMSprop'}. Best is trial 41 with value: 0.5536542534828186.
+[I 2023-10-23 07:50:36,379] Trial 44 finished with value: 0.5551037788391113 and parameters: {'lr': 5.640725954569493e-08, 'n_estimators': 39, 'optimizer': 'RMSprop'}. Best is trial 44 with value: 0.5551037788391113.
+[I 2023-10-23 08:21:58,489] Trial 45 finished with value: 0.5539408922195435 and parameters: {'lr': 6.562450771563963e-08, 'n_estimators': 39, 'optimizer': 'RMSprop'}. Best is trial 44 with value: 0.5551037788391113.
+[I 2023-10-23 08:53:04,595] Trial 46 finished with value: 0.5582337379455566 and parameters: {'lr': 6.502594269660963e-08, 'n_estimators': 46, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 09:20:24,889] Trial 47 finished with value: 0.5553371906280518 and parameters: {'lr': 5.355589591906474e-08, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 09:43:43,726] Trial 48 finished with value: 0.556071400642395 and parameters: {'lr': 1.0555536719402554e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 10:07:21,421] Trial 49 finished with value: 0.5569140911102295 and parameters: {'lr': 1.0687358012554984e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 10:42:52,432] Trial 50 finished with value: 0.5552547574043274 and parameters: {'lr': 1.0462986496303203e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 11:10:10,852] Trial 51 finished with value: 0.5569719076156616 and parameters: {'lr': 1.0981543368796825e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 11:42:13,758] Trial 52 finished with value: 0.5563305020332336 and parameters: {'lr': 1.1460389239037368e-07, 'n_estimators': 47, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 12:15:05,177] Trial 53 finished with value: 0.5571174621582031 and parameters: {'lr': 1.3916428640610327e-07, 'n_estimators': 46, 'optimizer': 'RMSprop'}. Best is trial 46 with value: 0.5582337379455566.
+[I 2023-10-23 12:51:50,898] Trial 54 finished with value: 0.5606839060783386 and parameters: {'lr': 1.2238913630137383e-07, 'n_estimators': 45, 'optimizer': 'RMSprop'}. Best is trial 54 with value: 0.5606839060783386.
+[I 2023-10-23 13:16:21,095] Trial 55 finished with value: 0.5585974454879761 and parameters: {'lr': 1.3523286029041088e-07, 'n_estimators': 44, 'optimizer': 'RMSprop'}. Best is trial 54 with value: 0.5606839060783386.
+[I 2023-10-23 13:40:52,822] Trial 56 finished with value: 0.563145101070404 and parameters: {'lr': 1.321765003562829e-07, 'n_estimators': 44, 'optimizer': 'RMSprop'}. Best is trial 56 with value: 0.563145101070404.
+[I 2023-10-23 14:17:34,503] Trial 57 finished with value: 0.5649504661560059 and parameters: {'lr': 1.417339501340407e-07, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 57 with value: 0.5649504661560059.
+[I 2023-10-23 15:06:34,962] Trial 58 finished with value: 0.5648221969604492 and parameters: {'lr': 2.1270825073760548e-07, 'n_estimators': 44, 'optimizer': 'Adam'}. Best is trial 57 with value: 0.5649504661560059.
+[I 2023-10-23 15:43:43,175] Trial 59 finished with value: 0.5651143193244934 and parameters: {'lr': 2.0658178964301883e-07, 'n_estimators': 43, 'optimizer': 'Adam'}. Best is trial 59 with value: 0.5651143193244934.
+[I 2023-10-23 16:12:19,615] Trial 60 finished with value: 0.5670831203460693 and parameters: {'lr': 2.0425588190663883e-07, 'n_estimators': 43, 'optimizer': 'Adam'}. Best is trial 60 with value: 0.5670831203460693.
+[I 2023-10-23 16:36:55,347] Trial 61 finished with value: 0.570649266242981 and parameters: {'lr': 2.1582322766042557e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 61 with value: 0.570649266242981.
+[I 2023-10-23 17:14:30,922] Trial 62 finished with value: 0.5675162076950073 and parameters: {'lr': 2.3392754826613693e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 61 with value: 0.570649266242981.
+[I 2023-10-23 18:07:46,984] Trial 63 finished with value: 0.5706519484519958 and parameters: {'lr': 2.2526583690421322e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 63 with value: 0.5706519484519958.
+[I 2023-10-23 18:32:20,406] Trial 64 finished with value: 0.572399377822876 and parameters: {'lr': 2.2729787427772587e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 64 with value: 0.572399377822876.
+[I 2023-10-23 19:34:18,729] Trial 65 finished with value: 0.5735474824905396 and parameters: {'lr': 2.4068578827383907e-07, 'n_estimators': 43, 'optimizer': 'Adam'}. Best is trial 65 with value: 0.5735474824905396.
+[I 2023-10-23 19:58:59,725] Trial 66 finished with value: 0.5737140774726868 and parameters: {'lr': 2.507074965520075e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 66 with value: 0.5737140774726868.
+[I 2023-10-23 20:27:39,393] Trial 67 finished with value: 0.575024425983429 and parameters: {'lr': 2.7138721673308e-07, 'n_estimators': 42, 'optimizer': 'Adam'}. Best is trial 67 with value: 0.575024425983429.
+[I 2023-10-23 20:56:36,970] Trial 68 finished with value: 0.5768953561782837 and parameters: {'lr': 2.8770118447252704e-07, 'n_estimators': 36, 'optimizer': 'Adam'}. Best is trial 68 with value: 0.5768953561782837.
+[I 2023-10-23 21:21:26,705] Trial 69 finished with value: 0.578156590461731 and parameters: {'lr': 2.8443655413359384e-07, 'n_estimators': 36, 'optimizer': 'Adam'}. Best is trial 69 with value: 0.578156590461731.
+[I 2023-10-23 21:46:06,503] Trial 70 finished with value: 0.5815342664718628 and parameters: {'lr': 2.9882598602116e-07, 'n_estimators': 36, 'optimizer': 'Adam'}. Best is trial 70 with value: 0.5815342664718628.
+[I 2023-10-23 22:10:52,256] Trial 71 finished with value: 0.5806370377540588 and parameters: {'lr': 2.930792868424465e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 70 with value: 0.5815342664718628.
+
+Model Loaded.
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+[I 2023-10-23 22:40:28,127] Trial 72 finished with value: 0.584173321723938 and parameters: {'lr': 3.072183458986638e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 72 with value: 0.584173321723938.
+[I 2023-10-23 23:21:59,127] Trial 73 finished with value: 0.5868313312530518 and parameters: {'lr': 3.1505289704826233e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 73 with value: 0.5868313312530518.
+[I 2023-10-24 00:03:21,470] Trial 74 finished with value: 0.5852217674255371 and parameters: {'lr': 3.37944519125002e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 73 with value: 0.5868313312530518.
+[I 2023-10-24 00:28:20,202] Trial 75 finished with value: 0.5888196229934692 and parameters: {'lr': 3.4279797801428923e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 75 with value: 0.5888196229934692.
+[I 2023-10-24 01:10:32,472] Trial 76 finished with value: 0.5885568857192993 and parameters: {'lr': 3.7785727396953406e-07, 'n_estimators': 35, 'optimizer': 'Adam'}. Best is trial 75 with value: 0.5888196229934692.
+[I 2023-10-24 01:47:41,292] Trial 77 finished with value: 0.5886940956115723 and parameters: {'lr': 3.730182606568301e-07, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 75 with value: 0.5888196229934692.
+[I 2023-10-24 02:16:42,436] Trial 78 finished with value: 0.5928004384040833 and parameters: {'lr': 3.98740229310406e-07, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 78 with value: 0.5928004384040833.
+[I 2023-10-24 02:54:09,980] Trial 79 finished with value: 0.596867561340332 and parameters: {'lr': 3.951220434815224e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 79 with value: 0.596867561340332.
+[I 2023-10-24 03:18:56,660] Trial 80 finished with value: 0.5987726449966431 and parameters: {'lr': 3.9217337610956314e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 80 with value: 0.5987726449966431.
+[I 2023-10-24 03:43:56,636] Trial 81 finished with value: 0.6010125279426575 and parameters: {'lr': 3.8983258726821844e-07, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 81 with value: 0.6010125279426575.
+[I 2023-10-24 04:12:53,577] Trial 82 finished with value: 0.6015823483467102 and parameters: {'lr': 3.9805005014361763e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 82 with value: 0.6015823483467102.
+[I 2023-10-24 05:14:20,760] Trial 83 finished with value: 0.6092749238014221 and parameters: {'lr': 3.988667591679828e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221.
+[I 2023-10-24 05:38:45,088] Trial 84 finished with value: 0.6045148968696594 and parameters: {'lr': 3.801315324866347e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221.
+[I 2023-10-24 06:07:41,056] Trial 85 finished with value: 0.6064762473106384 and parameters: {'lr': 3.9351790098082646e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221.
+[I 2023-10-24 06:36:23,798] Trial 86 finished with value: 0.6090301871299744 and parameters: {'lr': 3.8907654722821625e-07, 'n_estimators': 31, 'optimizer': 'Adam'}. Best is trial 83 with value: 0.6092749238014221.
+[I 2023-10-24 07:05:17,713] Trial 87 finished with value: 0.6128934025764465 and parameters: {'lr': 3.7610862993862156e-07, 'n_estimators': 31, 'optimizer': 'Adam'}. Best is trial 87 with value: 0.6128934025764465.
+[I 2023-10-24 07:42:09,228] Trial 88 finished with value: 0.6093340516090393 and parameters: {'lr': 3.4982809838009586e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 87 with value: 0.6128934025764465.
+[I 2023-10-24 08:19:16,153] Trial 89 finished with value: 0.6141061186790466 and parameters: {'lr': 3.3943441707865654e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 89 with value: 0.6141061186790466.
+[I 2023-10-24 08:47:53,069] Trial 90 finished with value: 0.6181385517120361 and parameters: {'lr': 3.415242523802862e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 90 with value: 0.6181385517120361.
+[I 2023-10-24 09:16:44,855] Trial 91 finished with value: 0.6147274374961853 and parameters: {'lr': 3.3757698574028605e-07, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 90 with value: 0.6181385517120361.
+[I 2023-10-24 09:49:23,877] Trial 92 finished with value: 0.618079423904419 and parameters: {'lr': 3.369688207850333e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 90 with value: 0.6181385517120361.
+[I 2023-10-24 10:47:19,401] Trial 93 finished with value: 0.6206730008125305 and parameters: {'lr': 3.333427116728784e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 93 with value: 0.6206730008125305.
+[I 2023-10-24 11:24:06,673] Trial 94 finished with value: 0.6197251081466675 and parameters: {'lr': 3.461562208850868e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 93 with value: 0.6206730008125305.
+[I 2023-10-24 12:08:51,353] Trial 95 finished with value: 0.6243206262588501 and parameters: {'lr': 3.3584644827281837e-07, 'n_estimators': 30, 'optimizer': 'Adam'}. Best is trial 95 with value: 0.6243206262588501.
+[I 2023-10-24 12:33:05,496] Trial 96 finished with value: 0.6249201893806458 and parameters: {'lr': 3.287811197213919e-07, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 96 with value: 0.6249201893806458.
+[I 2023-10-24 13:02:04,516] Trial 97 finished with value: 0.6230003237724304 and parameters: {'lr': 3.354841731848597e-07, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 96 with value: 0.6249201893806458.
+[I 2023-10-24 13:27:03,953] Trial 98 finished with value: 0.6289621591567993 and parameters: {'lr': 3.241147612310783e-07, 'n_estimators': 26, 'optimizer': 'Adam'}. Best is trial 98 with value: 0.6289621591567993.
+[I 2023-10-24 13:52:14,918] Trial 99 finished with value: 0.625308096408844 and parameters: {'lr': 3.287683345694601e-07, 'n_estimators': 26, 'optimizer': 'Adam'}. Best is trial 98 with value: 0.6289621591567993.
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Study statistics: 
+  Number of finished trials:  100
+  Number of pruned trials:  0
+  Number of complete trials:  100
+Best trial:
+  Value:  0.6289621591567993
+  Params: 
+    lr: 3.241147612310783e-07
+    n_estimators: 26
+    optimizer: Adam

results/hyp_with_wandb.txt

@@ -0,0 +1,323 @@
+wandb: Currently logged in as: jashpatel8561 (maa_64). Use `wandb login --relogin` to force relogin
+wandb: Appending key for api.wandb.ai to your netrc file: /root/.netrc
+[I 2023-10-31 20:08:45,039] A new study created in memory with name: no-name-881464b3-8bd6-47d5-8e36-f91094c9f780
+wandb: Tracking run with wandb version 0.15.12
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_200845-8vkss7as
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run scary-treat-7
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/8vkss7as
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.47619
+wandb:          Train Loss 1.52292
+wandb: Validation Accuracy 0.53696
+wandb:     Validation Loss 1.31054
+wandb: 
+wandb: 🚀 View run scary-treat-7 at: https://wandb.ai/maa_64/food-app/runs/8vkss7as
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231031_200845-8vkss7as/logs
+[I 2023-10-31 20:53:40,276] Trial 0 finished with value: 0.5369605422019958 and parameters: {'lr': 3.746494558899465e-06, 'n_estimators': 31, 'optimizer': 'SGD'}. Best is trial 0 with value: 0.5369605422019958.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_205340-97cre3xd
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run strange-cape-8
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/97cre3xd
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.48417
+wandb:          Train Loss 1.49974
+wandb: Validation Accuracy 0.53469
+wandb:     Validation Loss 1.30942
+wandb: 
+wandb: 🚀 View run strange-cape-8 at: https://wandb.ai/maa_64/food-app/runs/97cre3xd
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231031_205340-97cre3xd/logs
+[I 2023-10-31 21:26:05,821] Trial 1 finished with value: 0.5346873998641968 and parameters: {'lr': 2.1962752768229064e-07, 'n_estimators': 31, 'optimizer': 'RMSprop'}. Best is trial 0 with value: 0.5369605422019958.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_212605-2ork0ogz
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run mischievous-wendigo-9
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/2ork0ogz
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.48504
+wandb:          Train Loss 1.49989
+wandb: Validation Accuracy 0.5386
+wandb:     Validation Loss 1.29999
+wandb: 
+wandb: 🚀 View run mischievous-wendigo-9 at: https://wandb.ai/maa_64/food-app/runs/2ork0ogz
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231031_212605-2ork0ogz/logs
+[I 2023-10-31 21:54:42,788] Trial 2 finished with value: 0.5385968685150146 and parameters: {'lr': 4.9567301150128566e-08, 'n_estimators': 43, 'optimizer': 'RMSprop'}. Best is trial 2 with value: 0.5385968685150146.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_215442-pegj5p38
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run masked-cape-10
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/pegj5p38
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.48078
+wandb:          Train Loss 1.49002
+wandb: Validation Accuracy 0.54252
+wandb:     Validation Loss 1.30474
+wandb: 
+wandb: 🚀 View run masked-cape-10 at: https://wandb.ai/maa_64/food-app/runs/pegj5p38
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231031_215442-pegj5p38/logs
+[I 2023-10-31 22:19:18,241] Trial 3 finished with value: 0.5425158739089966 and parameters: {'lr': 3.6384818250623903e-06, 'n_estimators': 33, 'optimizer': 'SGD'}. Best is trial 3 with value: 0.5425158739089966.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_221918-gr6ablo3
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run strange-wraith-11
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/gr6ablo3
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.48708
+wandb:          Train Loss 1.49079
+wandb: Validation Accuracy 0.54065
+wandb:     Validation Loss 1.29405
+wandb: 
+wandb: 🚀 View run strange-wraith-11 at: https://wandb.ai/maa_64/food-app/runs/gr6ablo3
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231031_221918-gr6ablo3/logs
+[I 2023-10-31 22:43:49,791] Trial 4 finished with value: 0.5406512022018433 and parameters: {'lr': 3.1757764065521707e-07, 'n_estimators': 32, 'optimizer': 'Adam'}. Best is trial 3 with value: 0.5425158739089966.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_224349-6h4rftgh
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run occult-banshee-12
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/6h4rftgh
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.48493
+wandb:          Train Loss 1.48231
+wandb: Validation Accuracy 0.54034
+wandb:     Validation Loss 1.29421
+wandb: 
+wandb: 🚀 View run occult-banshee-12 at: https://wandb.ai/maa_64/food-app/runs/6h4rftgh
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231031_224349-6h4rftgh/logs
+[I 2023-10-31 23:44:49,930] Trial 5 finished with value: 0.5403411984443665 and parameters: {'lr': 4.079324797271018e-08, 'n_estimators': 32, 'optimizer': 'RMSprop'}. Best is trial 3 with value: 0.5425158739089966.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231031_234449-7nbh63pb
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run occult-gargoyle-13
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/7nbh63pb
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.49463
+wandb:          Train Loss 1.45459
+wandb: Validation Accuracy 0.54955
+wandb:     Validation Loss 1.27562
+wandb: 
+wandb: 🚀 View run occult-gargoyle-13 at: https://wandb.ai/maa_64/food-app/runs/7nbh63pb
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231031_234449-7nbh63pb/logs
+[I 2023-11-01 00:13:51,934] Trial 6 finished with value: 0.5495542883872986 and parameters: {'lr': 4.678309863383098e-07, 'n_estimators': 35, 'optimizer': 'RMSprop'}. Best is trial 6 with value: 0.5495542883872986.
+wandb: Tracking run with wandb version 0.15.12
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_001351-jyel0gll
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run dreadful-whisper-14
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/jyel0gll
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.50997
+wandb:          Train Loss 1.39204
+wandb: Validation Accuracy 0.55506
+wandb:     Validation Loss 1.23738
+wandb: 
+wandb: 🚀 View run dreadful-whisper-14 at: https://wandb.ai/maa_64/food-app/runs/jyel0gll
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231101_001351-jyel0gll/logs
+[I 2023-11-01 00:38:23,845] Trial 7 finished with value: 0.5550567507743835 and parameters: {'lr': 1.7101161575656505e-06, 'n_estimators': 29, 'optimizer': 'Adam'}. Best is trial 7 with value: 0.5550567507743835.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_003823-t9dkdynt
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run mischievous-specter-15
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/t9dkdynt
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.50968
+wandb:          Train Loss 1.38178
+wandb: Validation Accuracy 0.55892
+wandb:     Validation Loss 1.22877
+wandb: 
+wandb: 🚀 View run mischievous-specter-15 at: https://wandb.ai/maa_64/food-app/runs/t9dkdynt
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231101_003823-t9dkdynt/logs
+[I 2023-11-01 01:18:49,091] Trial 8 finished with value: 0.5589190125465393 and parameters: {'lr': 2.807500616871119e-07, 'n_estimators': 35, 'optimizer': 'SGD'}. Best is trial 8 with value: 0.5589190125465393.
+wandb: Tracking run with wandb version 0.15.12
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_011849-ota77xrr
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run spooky-specter-16
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/ota77xrr
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.52209
+wandb:          Train Loss 1.34973
+wandb: Validation Accuracy 0.56162
+wandb:     Validation Loss 1.2041
+wandb: 
+wandb: 🚀 View run spooky-specter-16 at: https://wandb.ai/maa_64/food-app/runs/ota77xrr
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231101_011849-ota77xrr/logs
+[I 2023-11-01 01:46:58,059] Trial 9 finished with value: 0.5616196393966675 and parameters: {'lr': 1.0143024563093913e-06, 'n_estimators': 33, 'optimizer': 'Adam'}. Best is trial 9 with value: 0.5616196393966675.
+wandb: - Waiting for wandb.init()...
+wandb: Run data is saved locally in /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu
+wandb: Run `wandb offline` to turn off syncing.
+wandb: Syncing run fearful-candle-17
+wandb: ⭐️ View project at https://wandb.ai/maa_64/food-app
+wandb: 🚀 View run at https://wandb.ai/maa_64/food-app/runs/mj672tzu
+wandb: Waiting for W&B process to finish... (success).
+wandb: - 0.005 MB of 0.005 MB uploaded (0.000 MB deduped)
+wandb: Run history:
+wandb:      Train Accuracy ▁
+wandb:          Train Loss ▁
+wandb: Validation Accuracy ▁
+wandb:     Validation Loss ▁
+wandb: 
+wandb: Run summary:
+wandb:      Train Accuracy 0.52938
+wandb:          Train Loss 1.31677
+wandb: Validation Accuracy 0.56583
+wandb:     Validation Loss 1.20021
+wandb: 
+wandb: 🚀 View run fearful-candle-17 at: https://wandb.ai/maa_64/food-app/runs/mj672tzu
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20231101_014658-mj672tzu/logs
+[I 2023-11-01 02:11:05,590] Trial 10 finished with value: 0.5658254027366638 and parameters: {'lr': 9.441552500921141e-07, 'n_estimators': 25, 'optimizer': 'Adam'}. Best is trial 10 with value: 0.5658254027366638.
+[LOG]: Login Succesfull.
+
+Model Loaded.
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Saving Model At: save_model/best_model.pth
+
+Study statistics: 
+  Number of finished trials:  11
+  Number of pruned trials:  0
+  Number of complete trials:  11
+Best trial:
+  Value:  0.5658254027366638
+  Params: 
+    lr: 9.441552500921141e-07
+    n_estimators: 25
+    optimizer: Adam

results/inference.txt

@@ -0,0 +1,7 @@
+
+Model Loaded.
+============= Diagnostic Run torch.onnx.export version 2.0.1+cu117 =============
+verbose: False, log level: Level.ERROR
+======================= 0 NONE 0 NOTE 0 WARNING 0 ERROR ========================
+
+Predicted: "foie_gras", Actual: "churros"

results/norm_result.txt

@@ -0,0 +1,153 @@
+
  0%|          | 0/50 [00:00<?, ?it/s]
  2%|▏         | 1/50 [03:02<2:28:40, 182.04s/it]
  4%|▍         | 2/50 [05:57<2:22:29, 178.12s/it]
  6%|▌         | 3/50 [08:49<2:17:18, 175.29s/it]
  8%|▊         | 4/50 [11:46<2:15:02, 176.15s/it]
 10%|█         | 5/50 [14:39<2:11:06, 174.80s/it]
 12%|█▏        | 6/50 [17:31<2:07:33, 173.94s/it]
 14%|█▍        | 7/50 [20:24<2:04:25, 173.61s/it]
 16%|█▌        | 8/50 [23:12<2:00:19, 171.90s/it]
 18%|█▊        | 9/50 [25:59<1:56:24, 170.34s/it]
 20%|██        | 10/50 [28:56<1:54:58, 172.45s/it]
 22%|██▏       | 11/50 [31:48<1:51:52, 172.12s/it]
 24%|██▍       | 12/50 [34:42<1:49:32, 172.96s/it]
 26%|██▌       | 13/50 [37:32<1:45:59, 171.89s/it]
 28%|██▊       | 14/50 [40:22<1:42:50, 171.40s/it]
 30%|███       | 15/50 [43:12<1:39:37, 170.80s/it]
 32%|███▏      | 16/50 [46:03<1:36:57, 171.11s/it]
 34%|███▍      | 17/50 [48:56<1:34:22, 171.60s/it]
 36%|███▌      | 18/50 [51:47<1:31:21, 171.31s/it]
 38%|███▊      | 19/50 [54:34<1:27:56, 170.22s/it]
 40%|████      | 20/50 [57:27<1:25:26, 170.88s/it]
 42%|████▏     | 21/50 [1:00:18<1:22:35, 170.86s/it]
 44%|████▍     | 22/50 [1:03:09<1:19:51, 171.14s/it]
 46%|████▌     | 23/50 [1:05:59<1:16:47, 170.63s/it]
 48%|████▊     | 24/50 [1:08:46<1:13:25, 169.42s/it]
 50%|█████     | 25/50 [1:11:35<1:10:39, 169.58s/it]
 52%|█████▏    | 26/50 [1:14:27<1:08:04, 170.19s/it]
 54%|█████▍    | 27/50 [1:17:17<1:05:15, 170.24s/it]
 56%|█████▌    | 28/50 [1:20:06<1:02:11, 169.62s/it]
 58%|█████▊    | 29/50 [1:23:01<59:56, 171.25s/it]  
 60%|██████    | 30/50 [1:26:00<57:54, 173.71s/it]
 62%|██████▏   | 31/50 [1:28:59<55:27, 175.14s/it]
 64%|██████▍   | 32/50 [1:31:57<52:50, 176.11s/it]
 66%|██████▌   | 33/50 [1:34:56<50:06, 176.84s/it]
 68%|██████▊   | 34/50 [1:37:57<47:30, 178.14s/it]
 70%|███████   | 35/50 [1:40:55<44:32, 178.18s/it]
 72%|███████▏  | 36/50 [1:43:54<41:36, 178.30s/it]
 74%|███████▍  | 37/50 [1:46:52<38:39, 178.40s/it]
 76%|███████▌  | 38/50 [1:49:52<35:45, 178.83s/it]
 78%|███████▊  | 39/50 [1:52:54<32:57, 179.81s/it]
 80%|████████  | 40/50 [1:55:56<30:05, 180.53s/it]
 82%|████████▏ | 41/50 [1:58:57<27:04, 180.46s/it]
 84%|████████▍ | 42/50 [2:01:57<24:03, 180.41s/it]
 86%|████████▌ | 43/50 [2:04:58<21:03, 180.51s/it]
 88%|████████▊ | 44/50 [2:07:52<17:51, 178.63s/it]
 90%|█████████ | 45/50 [2:10:47<14:48, 177.70s/it]
 92%|█████████▏| 46/50 [2:13:48<11:53, 178.48s/it]
 94%|█████████▍| 47/50 [2:16:48<08:56, 178.91s/it]
 96%|█████████▌| 48/50 [2:19:44<05:56, 178.15s/it]
 98%|█████████▊| 49/50 [2:22:47<02:59, 179.71s/it]
+
+Epoch: 1
+Train Loss: 4.61298 | Test Loss: 4.57462 || Train Accuracy: 0.00807 | Test Accuracy: 0.01234
+
+Epoch: 2
+Train Loss: 4.54959 | Test Loss: 4.50744 || Train Accuracy: 0.01589 | Test Accuracy: 0.02649
+
+Epoch: 3
+Train Loss: 4.48293 | Test Loss: 4.42678 || Train Accuracy: 0.02941 | Test Accuracy: 0.05546
+
+Epoch: 4
+Train Loss: 4.39801 | Test Loss: 4.32090 || Train Accuracy: 0.05808 | Test Accuracy: 0.09146
+
+Epoch: 5
+Train Loss: 4.28898 | Test Loss: 4.18107 || Train Accuracy: 0.09220 | Test Accuracy: 0.13604
+
+Epoch: 6
+Train Loss: 4.14920 | Test Loss: 4.00449 || Train Accuracy: 0.12465 | Test Accuracy: 0.17627
+
+Epoch: 7
+Train Loss: 3.99599 | Test Loss: 3.80676 || Train Accuracy: 0.15511 | Test Accuracy: 0.20713
+
+Epoch: 8
+Train Loss: 3.82920 | Test Loss: 3.61080 || Train Accuracy: 0.17907 | Test Accuracy: 0.23579
+
+Epoch: 9
+Train Loss: 3.66800 | Test Loss: 3.41604 || Train Accuracy: 0.20330 | Test Accuracy: 0.25473
+
+Epoch: 10
+Train Loss: 3.50918 | Test Loss: 3.24932 || Train Accuracy: 0.21650 | Test Accuracy: 0.27300
+
+Epoch: 11
+Train Loss: 3.36123 | Test Loss: 3.09507 || Train Accuracy: 0.23772 | Test Accuracy: 0.29009
+
+Epoch: 12
+Train Loss: 3.21851 | Test Loss: 2.93883 || Train Accuracy: 0.25185 | Test Accuracy: 0.30072
+
+Epoch: 13
+Train Loss: 3.09051 | Test Loss: 2.81236 || Train Accuracy: 0.26342 | Test Accuracy: 0.31615
+
+Epoch: 14
+Train Loss: 2.97843 | Test Loss: 2.67688 || Train Accuracy: 0.27258 | Test Accuracy: 0.33303
+
+Epoch: 15
+Train Loss: 2.86855 | Test Loss: 2.59203 || Train Accuracy: 0.28676 | Test Accuracy: 0.34505
+
+Epoch: 16
+Train Loss: 2.76454 | Test Loss: 2.46366 || Train Accuracy: 0.29645 | Test Accuracy: 0.35687
+
+Epoch: 17
+Train Loss: 2.66858 | Test Loss: 2.35961 || Train Accuracy: 0.30969 | Test Accuracy: 0.36896
+
+Epoch: 18
+Train Loss: 2.57830 | Test Loss: 2.29113 || Train Accuracy: 0.32281 | Test Accuracy: 0.38295
+
+Epoch: 19
+Train Loss: 2.49827 | Test Loss: 2.21650 || Train Accuracy: 0.33347 | Test Accuracy: 0.39498
+
+Epoch: 20
+Train Loss: 2.42086 | Test Loss: 2.14059 || Train Accuracy: 0.34202 | Test Accuracy: 0.40597
+
+Epoch: 21
+Train Loss: 2.34476 | Test Loss: 2.06311 || Train Accuracy: 0.35893 | Test Accuracy: 0.41456
+
+Epoch: 22
+Train Loss: 2.27082 | Test Loss: 1.99609 || Train Accuracy: 0.36612 | Test Accuracy: 0.42452
+
+Epoch: 23
+Train Loss: 2.20527 | Test Loss: 1.94224 || Train Accuracy: 0.37433 | Test Accuracy: 0.43183
+
+Epoch: 24
+Train Loss: 2.14534 | Test Loss: 1.89414 || Train Accuracy: 0.39044 | Test Accuracy: 0.43945
+
+Epoch: 25
+Train Loss: 2.09084 | Test Loss: 1.84661 || Train Accuracy: 0.39262 | Test Accuracy: 0.45064
+
+Epoch: 26
+Train Loss: 2.02959 | Test Loss: 1.76794 || Train Accuracy: 0.40709 | Test Accuracy: 0.46126
+
+Epoch: 27
+Train Loss: 1.98474 | Test Loss: 1.73753 || Train Accuracy: 0.41029 | Test Accuracy: 0.46164
+
+Epoch: 28
+Train Loss: 1.93014 | Test Loss: 1.68975 || Train Accuracy: 0.42321 | Test Accuracy: 0.47281
+
+Epoch: 29
+Train Loss: 1.87827 | Test Loss: 1.65753 || Train Accuracy: 0.42787 | Test Accuracy: 0.47670
+
+Epoch: 30
+Train Loss: 1.83835 | Test Loss: 1.62035 || Train Accuracy: 0.43941 | Test Accuracy: 0.48588
+
+Epoch: 31
+Train Loss: 1.79406 | Test Loss: 1.58791 || Train Accuracy: 0.44584 | Test Accuracy: 0.49323
+
+Epoch: 32
+Train Loss: 1.75459 | Test Loss: 1.56129 || Train Accuracy: 0.45799 | Test Accuracy: 0.49591
+
+Epoch: 33
+Train Loss: 1.71615 | Test Loss: 1.52328 || Train Accuracy: 0.45958 | Test Accuracy: 0.50340
+
+Epoch: 34
+Train Loss: 1.68502 | Test Loss: 1.50471 || Train Accuracy: 0.46491 | Test Accuracy: 0.50596
+
+Epoch: 35
+Train Loss: 1.64473 | Test Loss: 1.46475 || Train Accuracy: 0.47465 | Test Accuracy: 0.51220
+
+Epoch: 36
+Train Loss: 1.61245 | Test Loss: 1.43655 || Train Accuracy: 0.48251 | Test Accuracy: 0.51640
+
+Epoch: 37
+Train Loss: 1.57778 | Test Loss: 1.42014 || Train Accuracy: 0.49320 | Test Accuracy: 0.51885
+
+Epoch: 38
+Train Loss: 1.54909 | Test Loss: 1.39183 || Train Accuracy: 0.48998 | Test Accuracy: 0.52207
+
+Epoch: 39
+Train Loss: 1.51083 | Test Loss: 1.37381 || Train Accuracy: 0.50204 | Test Accuracy: 0.52653
+
+Epoch: 40
+Train Loss: 1.49493 | Test Loss: 1.35692 || Train Accuracy: 0.50977 | Test Accuracy: 0.52963
+
+Epoch: 41
+Train Loss: 1.45872 | Test Loss: 1.33956 || Train Accuracy: 0.50905 | Test Accuracy: 0.53182
+
+Epoch: 42
+Train Loss: 1.42958 | Test Loss: 1.31400 || Train Accuracy: 0.52015 | Test Accuracy: 0.53211
+
+Epoch: 43
+Train Loss: 1.39766 | Test Loss: 1.30862 || Train Accuracy: 0.52711 | Test Accuracy: 0.53479
+
+Epoch: 44
+Train Loss: 1.37803 | Test Loss: 1.28477 || Train Accuracy: 0.53449 | Test Accuracy: 0.54357
+
+Epoch: 45
+Train Loss: 1.35104 | Test Loss: 1.26904 || Train Accuracy: 0.54045 | Test Accuracy: 0.54389
+
+Epoch: 46
+Train Loss: 1.32729 | Test Loss: 1.24979 || Train Accuracy: 0.54576 | Test Accuracy: 0.54600
+
+Epoch: 47
+Train Loss: 1.30269 | Test Loss: 1.24156 || Train Accuracy: 0.54711 | Test Accuracy: 0.54635
+
+Epoch: 48
+Train Loss: 1.28490 | Test Loss: 1.23012 || Train Accuracy: 0.55110 | Test Accuracy: 0.54742
+
+Epoch: 49
+Train Loss: 1.25896 | Test Loss: 1.21039 || Train Accuracy: 0.56024 | Test Accuracy: 0.55533
+
+Epoch: 50
+Train Loss: 1.23424 | Test Loss: 1.20630 || Train Accuracy: 0.56350 | Test Accuracy: 0.55643
+
+Saving Model At: save_model/train_model_4e-06.pth

results/result_20.txt

@@ -0,0 +1,78 @@
+
  0%|          | 0/25 [00:00<?, ?it/s]
  4%|▍         | 1/25 [03:38<1:27:23, 218.48s/it]
  8%|▊         | 2/25 [07:12<1:22:46, 215.92s/it]
 12%|█▏        | 3/25 [10:48<1:19:14, 216.13s/it]
 16%|█▌        | 4/25 [14:26<1:15:47, 216.56s/it]
 20%|██        | 5/25 [18:00<1:11:52, 215.62s/it]
 24%|██▍       | 6/25 [21:36<1:08:21, 215.88s/it]
 28%|██▊       | 7/25 [25:09<1:04:30, 215.03s/it]
 32%|███▏      | 8/25 [28:43<1:00:48, 214.61s/it]
 36%|███▌      | 9/25 [32:16<57:07, 214.23s/it]  
 40%|████      | 10/25 [35:58<54:08, 216.57s/it]
 44%|████▍     | 11/25 [39:39<50:51, 217.96s/it]
 48%|████▊     | 12/25 [43:21<47:27, 219.02s/it]
 52%|█████▏    | 13/25 [47:01<43:51, 219.26s/it]
 56%|█████▌    | 14/25 [50:43<40:21, 220.16s/it]
 60%|██████    | 15/25 [54:24<36:43, 220.40s/it]
 64%|██████▍   | 16/25 [57:57<32:44, 218.26s/it]
 68%|██████▊   | 17/25 [1:01:36<29:07, 218.45s/it]
 72%|███████▏  | 18/25 [1:05:08<25:16, 216.61s/it]
 76%|███████▌  | 19/25 [1:08:46<21:42, 217.08s/it]
 80%|████████  | 20/25 [1:12:34<18:21, 220.36s/it]
 84%|████████▍ | 21/25 [1:16:09<14:34, 218.67s/it]
 88%|████████▊ | 22/25 [1:19:45<10:53, 217.93s/it]
 92%|█████████▏| 23/25 [1:23:19<07:13, 216.67s/it]
 96%|█████████▌| 24/25 [1:26:54<03:36, 216.21s/it]
+
+Epoch: 1
+Train Loss: 4.54480 | Test Loss: 4.40086 || Train Accuracy: 0.02033 | Test Accuracy: 0.06554
+
+Epoch: 2
+Train Loss: 4.23497 | Test Loss: 3.91198 || Train Accuracy: 0.09910 | Test Accuracy: 0.19270
+
+Epoch: 3
+Train Loss: 3.72633 | Test Loss: 3.27485 || Train Accuracy: 0.18921 | Test Accuracy: 0.27117
+
+Epoch: 4
+Train Loss: 3.23844 | Test Loss: 2.80989 || Train Accuracy: 0.24752 | Test Accuracy: 0.31899
+
+Epoch: 5
+Train Loss: 2.84906 | Test Loss: 2.44867 || Train Accuracy: 0.28938 | Test Accuracy: 0.36460
+
+Epoch: 6
+Train Loss: 2.54602 | Test Loss: 2.17645 || Train Accuracy: 0.33014 | Test Accuracy: 0.39797
+
+Epoch: 7
+Train Loss: 2.30744 | Test Loss: 1.95661 || Train Accuracy: 0.36248 | Test Accuracy: 0.43370
+
+Epoch: 8
+Train Loss: 2.11624 | Test Loss: 1.79439 || Train Accuracy: 0.38684 | Test Accuracy: 0.45905
+
+Epoch: 9
+Train Loss: 1.94724 | Test Loss: 1.65070 || Train Accuracy: 0.41640 | Test Accuracy: 0.47721
+
+Epoch: 10
+Train Loss: 1.81705 | Test Loss: 1.55378 || Train Accuracy: 0.43854 | Test Accuracy: 0.49538
+
+Epoch: 11
+Train Loss: 1.70089 | Test Loss: 1.45779 || Train Accuracy: 0.45786 | Test Accuracy: 0.51296
+
+Epoch: 12
+Train Loss: 1.59961 | Test Loss: 1.37595 || Train Accuracy: 0.47796 | Test Accuracy: 0.52727
+
+Epoch: 13
+Train Loss: 1.51026 | Test Loss: 1.30817 || Train Accuracy: 0.49863 | Test Accuracy: 0.54201
+
+Epoch: 14
+Train Loss: 1.43015 | Test Loss: 1.25472 || Train Accuracy: 0.51487 | Test Accuracy: 0.54927
+
+Epoch: 15
+Train Loss: 1.35515 | Test Loss: 1.21607 || Train Accuracy: 0.53062 | Test Accuracy: 0.55633
+
+Epoch: 16
+Train Loss: 1.29236 | Test Loss: 1.16553 || Train Accuracy: 0.54390 | Test Accuracy: 0.57092
+
+Epoch: 17
+Train Loss: 1.23855 | Test Loss: 1.13611 || Train Accuracy: 0.55646 | Test Accuracy: 0.57811
+
+Epoch: 18
+Train Loss: 1.17125 | Test Loss: 1.11371 || Train Accuracy: 0.57771 | Test Accuracy: 0.58057
+
+Epoch: 19
+Train Loss: 1.12487 | Test Loss: 1.08567 || Train Accuracy: 0.58283 | Test Accuracy: 0.58822
+
+Epoch: 20
+Train Loss: 1.07632 | Test Loss: 1.05419 || Train Accuracy: 0.60253 | Test Accuracy: 0.59501
+
+Epoch: 21
+Train Loss: 1.02422 | Test Loss: 1.02926 || Train Accuracy: 0.61423 | Test Accuracy: 0.60114
+
+Epoch: 22
+Train Loss: 0.98334 | Test Loss: 1.01630 || Train Accuracy: 0.62051 | Test Accuracy: 0.60067
+
+Epoch: 23
+Train Loss: 0.94386 | Test Loss: 1.00281 || Train Accuracy: 0.63427 | Test Accuracy: 0.60282
+
+Epoch: 24
+Train Loss: 0.90378 | Test Loss: 0.98175 || Train Accuracy: 0.64794 | Test Accuracy: 0.60909
+
+Epoch: 25
+Train Loss: 0.86799 | Test Loss: 0.97405 || Train Accuracy: 0.65758 | Test Accuracy: 0.60982
+
+Saving Model At: save_model/train_model_1e-05.pth

results/result_50.txt

@@ -0,0 +1,153 @@
+
  0%|          | 0/50 [00:00<?, ?it/s]
  2%|▏         | 1/50 [02:55<2:23:19, 175.50s/it]
  4%|▍         | 2/50 [05:50<2:20:01, 175.03s/it]
  6%|▌         | 3/50 [08:49<2:18:31, 176.84s/it]
  8%|▊         | 4/50 [11:47<2:16:10, 177.61s/it]
 10%|█         | 5/50 [14:43<2:12:43, 176.97s/it]
 12%|█▏        | 6/50 [17:38<2:09:10, 176.14s/it]
 14%|█▍        | 7/50 [20:29<2:05:10, 174.67s/it]
 16%|█▌        | 8/50 [23:28<2:03:03, 175.80s/it]
 18%|█▊        | 9/50 [26:26<2:00:34, 176.45s/it]
 20%|██        | 10/50 [29:23<1:57:43, 176.60s/it]
 22%|██▏       | 11/50 [32:19<1:54:50, 176.68s/it]
 24%|██▍       | 12/50 [35:13<1:51:20, 175.80s/it]
 26%|██▌       | 13/50 [38:07<1:48:02, 175.20s/it]
 28%|██▊       | 14/50 [40:56<1:43:56, 173.24s/it]
 30%|███       | 15/50 [43:45<1:40:23, 172.11s/it]
 32%|███▏      | 16/50 [46:34<1:36:56, 171.07s/it]
 34%|███▍      | 17/50 [49:25<1:34:09, 171.19s/it]
 36%|███▌      | 18/50 [52:19<1:31:37, 171.80s/it]
 38%|███▊      | 19/50 [55:10<1:28:42, 171.69s/it]
 40%|████      | 20/50 [58:07<1:26:34, 173.16s/it]
 42%|████▏     | 21/50 [1:00:57<1:23:18, 172.38s/it]
 44%|████▍     | 22/50 [1:03:47<1:20:02, 171.50s/it]
 46%|████▌     | 23/50 [1:06:38<1:17:13, 171.62s/it]
 48%|████▊     | 24/50 [1:09:35<1:15:00, 173.11s/it]
 50%|█████     | 25/50 [1:12:30<1:12:23, 173.76s/it]
 52%|█████▏    | 26/50 [1:15:25<1:09:38, 174.11s/it]
 54%|█████▍    | 27/50 [1:18:18<1:06:35, 173.73s/it]
 56%|█████▌    | 28/50 [1:21:13<1:03:49, 174.06s/it]
 58%|█████▊    | 29/50 [1:24:04<1:00:39, 173.31s/it]
 60%|██████    | 30/50 [1:26:57<57:38, 172.92s/it]  
 62%|██████▏   | 31/50 [1:29:44<54:17, 171.43s/it]
 64%|██████▍   | 32/50 [1:32:32<51:03, 170.17s/it]
 66%|██████▌   | 33/50 [1:35:23<48:20, 170.61s/it]
 68%|██████▊   | 34/50 [1:38:15<45:35, 170.95s/it]
 70%|███████   | 35/50 [1:41:06<42:43, 170.87s/it]
 72%|███████▏  | 36/50 [1:43:52<39:34, 169.59s/it]
 74%|███████▍  | 37/50 [1:46:38<36:29, 168.39s/it]
 76%|███████▌  | 38/50 [1:49:26<33:38, 168.17s/it]
 78%|███████▊  | 39/50 [1:52:17<30:59, 169.06s/it]
 80%|████████  | 40/50 [1:55:08<28:16, 169.62s/it]
 82%|████████▏ | 41/50 [1:58:02<25:40, 171.17s/it]
 84%|████████▍ | 42/50 [2:00:55<22:53, 171.70s/it]
 86%|████████▌ | 43/50 [2:03:41<19:48, 169.79s/it]
 88%|████████▊ | 44/50 [2:06:31<16:59, 169.92s/it]
 90%|█████████ | 45/50 [2:09:16<14:03, 168.61s/it]
 92%|█████████▏| 46/50 [2:12:03<11:11, 167.94s/it]
 94%|█████████▍| 47/50 [2:14:53<08:25, 168.53s/it]
 96%|█████████▌| 48/50 [2:17:41<05:36, 168.42s/it]
 98%|█████████▊| 49/50 [2:20:31<02:48, 168.77s/it]
+
+Epoch: 1
+Train Loss: 4.61311 | Test Loss: 4.57140 || Train Accuracy: 0.00822 | Test Accuracy: 0.01238
+
+Epoch: 2
+Train Loss: 4.55110 | Test Loss: 4.50663 || Train Accuracy: 0.01618 | Test Accuracy: 0.02801
+
+Epoch: 3
+Train Loss: 4.48176 | Test Loss: 4.42202 || Train Accuracy: 0.03321 | Test Accuracy: 0.05799
+
+Epoch: 4
+Train Loss: 4.39320 | Test Loss: 4.31635 || Train Accuracy: 0.06030 | Test Accuracy: 0.09786
+
+Epoch: 5
+Train Loss: 4.28416 | Test Loss: 4.17219 || Train Accuracy: 0.09393 | Test Accuracy: 0.14194
+
+Epoch: 6
+Train Loss: 4.13886 | Test Loss: 3.98871 || Train Accuracy: 0.12748 | Test Accuracy: 0.17830
+
+Epoch: 7
+Train Loss: 3.97565 | Test Loss: 3.79934 || Train Accuracy: 0.16209 | Test Accuracy: 0.21215
+
+Epoch: 8
+Train Loss: 3.80685 | Test Loss: 3.58789 || Train Accuracy: 0.18250 | Test Accuracy: 0.23079
+
+Epoch: 9
+Train Loss: 3.64305 | Test Loss: 3.45338 || Train Accuracy: 0.20222 | Test Accuracy: 0.25743
+
+Epoch: 10
+Train Loss: 3.49677 | Test Loss: 3.26330 || Train Accuracy: 0.21688 | Test Accuracy: 0.27583
+
+Epoch: 11
+Train Loss: 3.34792 | Test Loss: 3.10469 || Train Accuracy: 0.23781 | Test Accuracy: 0.29091
+
+Epoch: 12
+Train Loss: 3.20788 | Test Loss: 2.94270 || Train Accuracy: 0.25400 | Test Accuracy: 0.30615
+
+Epoch: 13
+Train Loss: 3.08369 | Test Loss: 2.82091 || Train Accuracy: 0.26199 | Test Accuracy: 0.32230
+
+Epoch: 14
+Train Loss: 2.96920 | Test Loss: 2.70113 || Train Accuracy: 0.28136 | Test Accuracy: 0.33737
+
+Epoch: 15
+Train Loss: 2.86222 | Test Loss: 2.57494 || Train Accuracy: 0.29079 | Test Accuracy: 0.34906
+
+Epoch: 16
+Train Loss: 2.75817 | Test Loss: 2.47304 || Train Accuracy: 0.30299 | Test Accuracy: 0.36729
+
+Epoch: 17
+Train Loss: 2.65771 | Test Loss: 2.38487 || Train Accuracy: 0.31518 | Test Accuracy: 0.37654
+
+Epoch: 18
+Train Loss: 2.57432 | Test Loss: 2.29379 || Train Accuracy: 0.32641 | Test Accuracy: 0.38946
+
+Epoch: 19
+Train Loss: 2.47982 | Test Loss: 2.20300 || Train Accuracy: 0.34461 | Test Accuracy: 0.40082
+
+Epoch: 20
+Train Loss: 2.40916 | Test Loss: 2.12339 || Train Accuracy: 0.35451 | Test Accuracy: 0.40934
+
+Epoch: 21
+Train Loss: 2.32771 | Test Loss: 2.05966 || Train Accuracy: 0.36562 | Test Accuracy: 0.41964
+
+Epoch: 22
+Train Loss: 2.26082 | Test Loss: 1.99939 || Train Accuracy: 0.37409 | Test Accuracy: 0.43084
+
+Epoch: 23
+Train Loss: 2.19902 | Test Loss: 1.93918 || Train Accuracy: 0.38025 | Test Accuracy: 0.43846
+
+Epoch: 24
+Train Loss: 2.13738 | Test Loss: 1.89079 || Train Accuracy: 0.39498 | Test Accuracy: 0.44064
+
+Epoch: 25
+Train Loss: 2.08157 | Test Loss: 1.83362 || Train Accuracy: 0.39733 | Test Accuracy: 0.44783
+
+Epoch: 26
+Train Loss: 2.02082 | Test Loss: 1.77859 || Train Accuracy: 0.41026 | Test Accuracy: 0.45830
+
+Epoch: 27
+Train Loss: 1.97100 | Test Loss: 1.73532 || Train Accuracy: 0.42021 | Test Accuracy: 0.46355
+
+Epoch: 28
+Train Loss: 1.92201 | Test Loss: 1.70182 || Train Accuracy: 0.42769 | Test Accuracy: 0.47038
+
+Epoch: 29
+Train Loss: 1.87888 | Test Loss: 1.66066 || Train Accuracy: 0.43447 | Test Accuracy: 0.47528
+
+Epoch: 30
+Train Loss: 1.83671 | Test Loss: 1.62636 || Train Accuracy: 0.43882 | Test Accuracy: 0.48363
+
+Epoch: 31
+Train Loss: 1.79174 | Test Loss: 1.59488 || Train Accuracy: 0.44832 | Test Accuracy: 0.48278
+
+Epoch: 32
+Train Loss: 1.75153 | Test Loss: 1.54587 || Train Accuracy: 0.45677 | Test Accuracy: 0.49407
+
+Epoch: 33
+Train Loss: 1.70986 | Test Loss: 1.51729 || Train Accuracy: 0.46725 | Test Accuracy: 0.49687
+
+Epoch: 34
+Train Loss: 1.67850 | Test Loss: 1.49519 || Train Accuracy: 0.47018 | Test Accuracy: 0.50080
+
+Epoch: 35
+Train Loss: 1.64491 | Test Loss: 1.47187 || Train Accuracy: 0.48170 | Test Accuracy: 0.50509
+
+Epoch: 36
+Train Loss: 1.60652 | Test Loss: 1.43719 || Train Accuracy: 0.48456 | Test Accuracy: 0.51574
+
+Epoch: 37
+Train Loss: 1.57641 | Test Loss: 1.42869 || Train Accuracy: 0.48982 | Test Accuracy: 0.51615
+
+Epoch: 38
+Train Loss: 1.54231 | Test Loss: 1.38871 || Train Accuracy: 0.49637 | Test Accuracy: 0.52524
+
+Epoch: 39
+Train Loss: 1.51172 | Test Loss: 1.37745 || Train Accuracy: 0.50456 | Test Accuracy: 0.52450
+
+Epoch: 40
+Train Loss: 1.47853 | Test Loss: 1.35302 || Train Accuracy: 0.51194 | Test Accuracy: 0.52710
+
+Epoch: 41
+Train Loss: 1.45852 | Test Loss: 1.34218 || Train Accuracy: 0.51391 | Test Accuracy: 0.52890
+
+Epoch: 42
+Train Loss: 1.42956 | Test Loss: 1.31330 || Train Accuracy: 0.52436 | Test Accuracy: 0.53586
+
+Epoch: 43
+Train Loss: 1.39965 | Test Loss: 1.29385 || Train Accuracy: 0.52816 | Test Accuracy: 0.53908
+
+Epoch: 44
+Train Loss: 1.37700 | Test Loss: 1.28448 || Train Accuracy: 0.53431 | Test Accuracy: 0.54071
+
+Epoch: 45
+Train Loss: 1.34920 | Test Loss: 1.27006 || Train Accuracy: 0.53668 | Test Accuracy: 0.54439
+
+Epoch: 46
+Train Loss: 1.32792 | Test Loss: 1.25679 || Train Accuracy: 0.54473 | Test Accuracy: 0.55152
+
+Epoch: 47
+Train Loss: 1.30758 | Test Loss: 1.24068 || Train Accuracy: 0.54716 | Test Accuracy: 0.55219
+
+Epoch: 48
+Train Loss: 1.27623 | Test Loss: 1.22565 || Train Accuracy: 0.55646 | Test Accuracy: 0.55421
+
+Epoch: 49
+Train Loss: 1.26111 | Test Loss: 1.21290 || Train Accuracy: 0.55763 | Test Accuracy: 0.55621
+
+Epoch: 50
+Train Loss: 1.23552 | Test Loss: 1.20254 || Train Accuracy: 0.56427 | Test Accuracy: 0.55813
+
+Saving Model At: save_model/train_model_4e-06.pth

save_model/best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0dd5c48ebebdf5d612dfdb5953c697cf18b08e0aa9657e7fed836f5b00214bc
+size 16846383

save_model/food.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab54d244d505ce13c0a3b0aef8f66e6ad0d4388eb77aaa4e4b182e2519e8357b
+size 16537711

save_model/food_cpu.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d35b986fd20f707b49802235f9d9dc57e1b7382d7db83a4cd2dd4f9f247b11af
+size 16537711

save_model/train_model_3e-06.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:030297cc2ee62e55d77bb51010afa3f730c934ad049e0e1f1cadd1ec56f5e632
+size 16848917

save_model/train_model_4e-06.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dc3bd0d0ac67a410bfbc24815d59787b493566a897aab2eede98a33de99f47a
+size 16848917

train.py

@@ -0,0 +1,49 @@
+import torch
+import torch.nn as nn
+from torchvision.models import efficientnet_b0, EfficientNet_B0_Weights
+from torchmetrics.classification import MulticlassAccuracy
+
+import data, engine, utils
+
+
+# if __name__ == "__main__":
+    # HYPERPARAMETERS
+SEED = 64
+NUM_EPOCH = 50
+LEARNIGN_RATE = 4e-6 # 3e-4, 4e-5, 7e-6, 5e-7, 3e-9
+
+NUM_CLASSES = 101
+# CUDA_LAUNCH_BLOCKING=1
+
+device = torch.device("cuda:3" if torch.cuda.is_available() else 'cpu')
+# print(device)
+
+
+if __name__ == "__main__":
+
+    torch.manual_seed(SEED)
+    torch.cuda.manual_seed(SEED)
+    model = efficientnet_b0(weights=EfficientNet_B0_Weights.DEFAULT)
+    model.classifier = nn.Sequential(
+                                        nn.Dropout(p = 0.2, inplace = True),
+                                        nn.Linear(1280, NUM_CLASSES),
+                                        # nn.Softmax()
+                                    )
+    model = model.to(device)
+    # print(model)
+
+
+    loss_fn = torch.nn.CrossEntropyLoss()
+    accuracy_fn = MulticlassAccuracy(num_classes = NUM_CLASSES).to(device)
+    optimizer = torch.optim.Adam(model.parameters(), lr = LEARNIGN_RATE)
+
+
+    # print(f"lr: {scheduler.optimizer.param_groups[0]['lr']}")
+
+    train_losses, test_losses, train_accs, test_accs, train_model = engine.train(model, data.train_dataloader, data.test_dataloader,
+                                                                                    optimizer, loss_fn, accuracy_fn, NUM_EPOCH, device)
+
+    utils.save_model(model = train_model, target_dir = "./save_model", model_name = f"train_model_{LEARNIGN_RATE}.pth")
+
+    utils.plot_graph(train_losses = train_losses, test_losses = test_losses, train_accs = train_accs, 
+                        test_accs = test_accs, fig_name = f"plots/cnn_train_Loss_and_accuracy_plot_{LEARNIGN_RATE}.jpg")

utils.py

@@ -0,0 +1,84 @@
+import torch
+from pathlib import Path
+import matplotlib.pyplot as plt
+
+
+# SAVE MODEL
+def save_model(model: torch.nn.Module, target_dir: str, model_name: str):
+    """
+    Save pytorch model to a traget dir
+    Args:
+        model: A traget pytorch model
+        target_dir: Directory to save the model to
+        model_name: File name to save model. should include ".pth" or ".pt" at the end of the file extention
+
+    Example usage:
+        save_model(model = model_0, target_dir = "models", model_name="model.pth")
+
+    """
+
+    target_dir_path = Path(target_dir)
+    target_dir_path.mkdir(parents = True, exist_ok = True)
+
+    assert model_name.endswith(".pth") or model_name.endswith(".pt"), "model name should be end with .pth or .pt"
+    model_save_path = target_dir_path / model_name
+
+    print(f"\nSaving Model At: {model_save_path}")
+    torch.save(obj = model.state_dict(), f = model_save_path)
+
+
+# LOAD MODEL
+def load_model(model: torch.nn.Module, model_path: str):
+    """
+    Load pytorch model from source dir
+    Args:
+        model: A model which need to load
+        source_dir: path where trained model is saved. should be full path including model name
+
+    Example usage:
+        load_model(model = model_0, source_path = "models/model.pth")
+    """
+    model.load_state_dict(torch.load(f = model_path, map_location=torch.device('cpu')))
+    print("\nModel Loaded.")
+
+    return model
+
+
+
+# PLOT Function
+def plot_graph(train_losses: list, test_losses: list, train_accs: list, test_accs: list, fig_name: str):
+    """
+    Plot the grapoh of loss abd accuray of the model
+    Args:
+        train_losses: list of train loss
+        test_losses: list of test loss
+        train_accs: list of train accuracy
+        test_accs: list of test accuracy
+        fig_name: name of image file which with you want to save plot image and must include .jpg 
+
+    Example usage:
+        plot_graph(train_losses = train_loss, test_losses = test_loss, train_accs = train_acc,
+                   test_accs = test_acc, fig_name = "plot.jpg")
+    """
+    plt.figure(figsize = (20, 8))
+    plt.subplot(1, 2, 1)
+    plt.plot(range(len(train_losses)), train_losses, label = "Train Loss")
+    plt.plot(range(len(test_losses)), test_losses, label = "Test Loss")
+    plt.legend()
+    plt.xlabel("Epoches")
+    plt.ylabel("Loss")
+    # plt.show()
+
+    plt.subplot(1, 2, 2)
+    plt.plot(range(len(train_accs)), train_accs, label = "Train Accuracy")
+    plt.plot(range(len(test_accs)), test_accs, label = "Test Accuracy")
+    plt.legend()
+    plt.xlabel("Epoches")
+    plt.ylabel("Accuracy")
+    # plt.show()
+    plt.savefig(fig_name)
+
+
+# Optimize Model with ONNX
+def onnx_inference(model: torch.nn.Module, path: str, device: str):
+    torch.onnx.export(model, torch.randn(1, 3, 224, 224).to(device), path, verbose=False, input_names=['input'], output_names=['output'], export_params=True)

wandb/debug-internal.log

@@ -0,0 +1,797 @@
+2023-11-01 01:46:58,588 INFO    StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-11-01 01:46:58.587580
+2023-11-01 01:46:58,590 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status
+2023-11-01 01:46:58,592 INFO    WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb
+2023-11-01 01:46:58,595 DEBUG   SenderThread:34227 [sender.py:send():380] send: header
+2023-11-01 01:46:58,595 DEBUG   SenderThread:34227 [sender.py:send():380] send: run
+2023-11-01 01:46:59,448 INFO    SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files
+2023-11-01 01:46:59,448 INFO    SenderThread:34227 [sender.py:_start_run_threads():1122] run started: mj672tzu with start time 1698788818.109446
+2023-11-01 01:46:59,469 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version
+2023-11-01 01:46:59,470 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: check_version
+2023-11-01 01:47:00,265 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start
+2023-11-01 01:47:00,267 DEBUG   HandlerThread:34227 [system_info.py:__init__():32] System info init
+2023-11-01 01:47:00,267 DEBUG   HandlerThread:34227 [system_info.py:__init__():47] System info init done
+2023-11-01 01:47:00,267 INFO    HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor
+2023-11-01 01:47:00,267 INFO    SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2023-11-01 01:47:00,267 INFO    HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info
+2023-11-01 01:47:00,268 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring
+2023-11-01 01:47:00,269 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring
+2023-11-01 01:47:00,270 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring
+2023-11-01 01:47:00,271 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring
+2023-11-01 01:47:00,272 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring
+2023-11-01 01:47:00,334 DEBUG   HandlerThread:34227 [system_info.py:probe():196] Probing system
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_info.py:probe():244] Probing system done
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T21:47:00.334647', 'startedAt': '2023-10-31T21:46:58.105196', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.519625, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.297, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.465, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.697, 'min': 1200.0, 'max': 3600.0}, {'current': 1.476, 'min': 1200.0, 'max': 3600.0}, {'current': 2.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.196, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 2.7, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.498, 'min': 1200.0, 'max': 3600.0}, {'current': 1.268, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.251, 'min': 1200.0, 'max': 3600.0}, {'current': 1.697, 'min': 1200.0, 'max': 3600.0}, {'current': 1.534, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9305191040039}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}}
+2023-11-01 01:47:00,335 INFO    HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info
+2023-11-01 01:47:00,335 INFO    HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done
+2023-11-01 01:47:00,336 DEBUG   HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment
+2023-11-01 01:47:00,451 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt
+2023-11-01 01:47:00,452 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml
+2023-11-01 01:47:06,925 DEBUG   HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done
+2023-11-01 01:47:06,927 INFO    HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info
+2023-11-01 01:47:06,927 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:06,928 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive
+2023-11-01 01:47:06,929 DEBUG   SenderThread:34227 [sender.py:send():380] send: files
+2023-11-01 01:47:06,929 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now
+2023-11-01 01:47:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:07,442 DEBUG   SenderThread:34227 [sender.py:send():380] send: telemetry
+2023-11-01 01:47:07,450 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml
+2023-11-01 01:47:07,451 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json
+2023-11-01 01:47:09,285 INFO    wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpqi5xsr4gwandb/rhtw7848-wandb-metadata.json
+2023-11-01 01:47:09,443 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:14,444 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:19,445 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:21,940 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:21,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:21,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:25,370 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:30,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:31,458 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml
+2023-11-01 01:47:35,747 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:36,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:36,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:36,981 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:41,296 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:46,297 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:51,298 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:51,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:51,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:51,981 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:57,290 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:00,272 DEBUG   SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2023-11-01 01:48:00,276 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:48:03,277 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:06,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:06,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:08,317 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:13,318 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:18,319 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:21,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:21,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:21,981 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:24,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:29,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:30,278 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:48:35,279 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:36,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:36,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:40,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:45,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:50,333 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:51,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:51,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:56,275 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:00,280 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:49:01,281 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:06,282 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:06,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:12,279 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:17,279 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:21,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:21,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:22,346 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:27,347 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:30,282 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:49:33,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:36,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:38,291 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:43,292 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:48,293 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:51,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:51,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:53,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:58,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:00,284 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:50:04,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:06,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:09,358 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:14,359 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:19,359 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:21,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:21,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:24,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:29,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:30,286 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:50:35,287 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:36,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:36,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:40,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:45,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:50,362 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:51,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:51,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:56,350 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:00,288 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:51:02,290 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:07,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:09,530 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 01:51:11,530 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 01:51:12,954 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:17,954 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:21,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:21,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:21,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:23,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:28,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:30,290 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:51:34,292 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:36,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:39,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:44,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:49,327 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:51,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:51,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:55,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:00,293 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:52:01,295 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:06,295 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:11,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:16,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:21,327 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:21,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:21,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:26,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:30,295 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:52:32,297 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:38,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:43,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:48,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:51,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:51,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:53,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:58,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:00,297 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:53:04,299 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:06,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:09,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:14,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:19,370 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:21,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:21,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:25,282 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:30,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:30,299 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:53:35,300 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:41,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:46,287 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:51,287 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:51,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:51,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:56,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:00,301 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:54:02,302 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:06,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:06,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:07,360 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:12,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:17,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:21,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:21,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:23,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:28,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:30,302 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:54:34,304 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:36,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:39,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:44,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:49,368 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:51,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:51,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:55,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:00,306 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:55:01,307 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:06,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:06,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:06,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:12,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:17,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:21,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:21,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:22,370 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:27,371 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:30,308 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:55:33,309 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:36,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:36,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:38,354 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:43,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:48,356 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:51,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:51,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:54,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:59,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:00,309 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:56:04,311 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:06,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:06,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:09,328 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:14,328 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:19,329 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:21,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:21,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:24,356 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:29,357 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:30,311 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:56:35,313 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:36,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:36,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:40,339 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:45,340 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:50,340 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:51,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:51,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:55,379 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:00,313 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:57:01,314 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:06,315 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:06,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:06,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:12,281 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:17,282 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:21,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:21,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:22,295 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:27,296 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:30,315 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:57:32,316 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:36,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:37,365 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:42,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:47,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:51,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:51,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:53,336 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:58,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:00,318 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:58:04,319 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:06,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:06,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:09,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:14,338 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:19,339 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:21,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:21,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:24,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:29,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:30,320 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:58:35,322 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:36,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:36,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:40,450 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:45,451 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:50,451 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:51,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:51,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:56,294 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:00,322 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:59:01,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:06,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:06,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:06,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:11,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:16,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:21,352 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:21,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:21,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:27,314 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:30,324 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:59:32,325 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:36,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:38,312 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:43,313 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:48,313 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:51,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:51,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:54,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:59,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:00,326 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:00:04,328 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:06,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:06,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:09,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:14,338 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:19,338 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:21,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:21,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:25,276 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:30,277 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:30,329 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:00:35,331 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:36,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:40,373 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:45,374 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:50,375 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:51,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:51,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:56,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:00,330 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:01:01,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:06,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:06,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:06,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:12,304 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:17,305 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:21,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:21,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:22,315 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:27,316 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:30,332 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:01:32,334 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:36,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:36,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:37,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:42,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:47,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:51,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:51,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:52,385 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:57,385 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:00,335 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:02:03,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:06,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:06,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:09,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:14,284 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:19,284 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:21,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:21,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:24,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:29,327 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:30,338 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:02:34,340 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:36,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:36,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:39,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:44,352 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:49,353 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:51,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:51,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:55,275 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:00,276 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:00,341 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:03:05,344 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:06,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:06,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:10,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:15,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:20,382 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:21,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:21,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:25,389 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:30,344 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:03:31,346 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:36,347 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:36,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:36,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:41,357 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:46,357 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:51,358 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:51,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:51,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:56,622 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:00,347 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:04:02,349 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:06,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:06,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:08,335 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:13,335 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:18,336 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:21,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:21,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:24,298 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:29,298 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:30,351 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:04:34,353 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:36,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:36,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:39,360 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:44,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:49,362 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:51,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:51,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:54,372 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:59,373 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:00,352 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:05:05,354 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:06,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:06,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:11,299 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:16,299 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:21,300 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:21,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:21,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:27,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:30,354 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:05:32,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:36,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:36,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:38,348 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:43,348 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:48,349 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:51,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:51,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:53,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:58,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:00,357 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:06:04,359 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:06,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:06,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:10,333 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:15,334 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:20,335 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:21,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:21,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:26,315 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:30,359 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:06:31,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:36,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:36,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:36,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:36,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:42,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:47,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:51,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:51,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:53,293 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:58,293 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:00,361 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:07:03,362 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:06,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:06,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:09,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:14,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:19,321 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:21,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:21,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:24,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:29,379 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:30,363 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:07:35,365 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:36,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:36,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:36,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:41,323 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:46,323 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:51,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:51,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:51,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:56,393 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:00,366 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:08:02,368 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:06,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:06,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:07,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:12,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:17,388 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:21,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:21,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:23,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:28,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:30,368 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:08:34,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:36,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:36,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:39,446 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:44,447 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:49,447 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:51,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:51,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:55,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:00,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:00,372 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:09:05,374 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:06,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:06,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:11,276 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:16,277 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:21,278 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:21,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:21,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:26,312 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:30,376 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:09:31,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:36,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:36,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:41,398 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:46,398 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:51,399 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:51,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:51,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:57,396 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:00,379 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:10:03,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:06,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:06,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:09,330 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:14,331 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:19,331 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:21,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:21,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:21,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:25,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:30,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:30,382 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:10:35,383 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:36,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:41,349 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:46,350 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:51,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:51,953 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:51,953 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:51,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:57,312 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:57,847 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history
+2023-11-01 02:10:57,849 DEBUG   SenderThread:34227 [sender.py:send():380] send: history
+2023-11-01 02:10:57,849 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: summary_record
+2023-11-01 02:10:57,849 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
+2023-11-01 02:10:57,889 DEBUG   SenderThread:34227 [sender.py:send():380] send: telemetry
+2023-11-01 02:10:57,889 DEBUG   SenderThread:34227 [sender.py:send():380] send: exit
+2023-11-01 02:10:57,889 INFO    SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0
+2023-11-01 02:10:57,889 INFO    SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1437
+2023-11-01 02:10:57,890 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
+2023-11-01 02:10:57,890 INFO    SenderThread:34227 [sender.py:send_exit():593] send defer
+2023-11-01 02:10:57,890 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:57,890 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0
+2023-11-01 02:10:57,891 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:57,891 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0
+2023-11-01 02:10:57,891 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 1
+2023-11-01 02:10:57,891 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:57,891 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1
+2023-11-01 02:10:57,891 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:57,892 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1
+2023-11-01 02:10:57,892 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 2
+2023-11-01 02:10:57,892 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:57,892 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2
+2023-11-01 02:10:57,892 INFO    HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor
+2023-11-01 02:10:57,892 DEBUG   SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2023-11-01 02:10:57,893 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor
+2023-11-01 02:10:57,893 DEBUG   SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics
+2023-11-01 02:10:57,893 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor
+2023-11-01 02:10:57,953 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json
+2023-11-01 02:10:59,649 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor
+2023-11-01 02:10:59,650 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor
+2023-11-01 02:10:59,650 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor
+2023-11-01 02:10:59,650 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:10:59,651 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,651 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2
+2023-11-01 02:10:59,652 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 3
+2023-11-01 02:10:59,652 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:10:59,652 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:10:59,653 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,653 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3
+2023-11-01 02:10:59,653 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 4
+2023-11-01 02:10:59,654 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,654 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4
+2023-11-01 02:10:59,654 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 5
+2023-11-01 02:10:59,655 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,655 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5
+2023-11-01 02:10:59,655 DEBUG   SenderThread:34227 [sender.py:send():380] send: summary
+2023-11-01 02:10:59,656 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
+2023-11-01 02:10:59,656 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,656 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5
+2023-11-01 02:10:59,656 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 6
+2023-11-01 02:10:59,656 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,657 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6
+2023-11-01 02:10:59,657 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,657 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6
+2023-11-01 02:10:59,662 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:59,849 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:10:59,954 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json
+2023-11-01 02:11:00,056 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 7
+2023-11-01 02:11:00,056 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:00,056 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,057 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7
+2023-11-01 02:11:00,057 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,057 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7
+2023-11-01 02:11:00,230 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 8
+2023-11-01 02:11:00,231 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,231 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8
+2023-11-01 02:11:00,231 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,231 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8
+2023-11-01 02:11:00,232 INFO    SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact
+2023-11-01 02:11:00,232 INFO    SenderThread:34227 [job_builder.py:_get_source_type():404] no source found
+2023-11-01 02:11:00,232 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 9
+2023-11-01 02:11:00,233 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,233 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9
+2023-11-01 02:11:00,233 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,233 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9
+2023-11-01 02:11:00,233 INFO    SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher
+2023-11-01 02:11:00,849 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:00,955 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml
+2023-11-01 02:11:00,955 INFO    SenderThread:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 02:11:00,956 INFO    SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files
+2023-11-01 02:11:00,956 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log output.log
+2023-11-01 02:11:00,957 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt requirements.txt
+2023-11-01 02:11:00,962 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml conda-environment.yaml
+2023-11-01 02:11:00,963 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json wandb-summary.json
+2023-11-01 02:11:00,967 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml config.yaml
+2023-11-01 02:11:00,980 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json wandb-metadata.json
+2023-11-01 02:11:00,981 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 10
+2023-11-01 02:11:00,981 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:00,982 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,989 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10
+2023-11-01 02:11:00,990 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,990 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10
+2023-11-01 02:11:00,990 INFO    SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher
+2023-11-01 02:11:01,845 INFO    wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 02:11:01,850 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:01,851 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:02,100 INFO    wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt
+2023-11-01 02:11:02,147 INFO    wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json
+2023-11-01 02:11:02,154 INFO    wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml
+2023-11-01 02:11:02,851 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:02,852 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:03,196 INFO    wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml
+2023-11-01 02:11:03,396 INFO    Thread-132:34227 [sender.py:transition_state():613] send defer: 11
+2023-11-01 02:11:03,397 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,397 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11
+2023-11-01 02:11:03,398 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,398 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11
+2023-11-01 02:11:03,398 INFO    SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher
+2023-11-01 02:11:03,398 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 12
+2023-11-01 02:11:03,398 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,399 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12
+2023-11-01 02:11:03,399 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,399 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12
+2023-11-01 02:11:03,399 INFO    SenderThread:34227 [file_stream.py:finish():595] file stream finish called
+2023-11-01 02:11:03,757 INFO    SenderThread:34227 [file_stream.py:finish():599] file stream finish is done
+2023-11-01 02:11:03,757 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 13
+2023-11-01 02:11:03,758 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,758 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13
+2023-11-01 02:11:03,758 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,758 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13
+2023-11-01 02:11:03,759 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 14
+2023-11-01 02:11:03,759 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,759 DEBUG   SenderThread:34227 [sender.py:send():380] send: final
+2023-11-01 02:11:03,760 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14
+2023-11-01 02:11:03,760 DEBUG   SenderThread:34227 [sender.py:send():380] send: footer
+2023-11-01 02:11:03,760 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,760 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14
+2023-11-01 02:11:03,762 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:11:03,763 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:03,763 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:03,764 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info
+2023-11-01 02:11:03,764 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary
+2023-11-01 02:11:03,765 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history
+2023-11-01 02:11:03,765 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: server_info
+2023-11-01 02:11:03,771 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info
+2023-11-01 02:11:04,116 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: job_info
+2023-11-01 02:11:04,117 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown
+2023-11-01 02:11:04,117 INFO    HandlerThread:34227 [handler.py:finish():866] shutting down handler
+2023-11-01 02:11:04,772 INFO    WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb
+2023-11-01 02:11:05,116 INFO    SenderThread:34227 [sender.py:finish():1534] shutting down sender
+2023-11-01 02:11:05,116 INFO    SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher
+2023-11-01 02:11:05,116 INFO    SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher

wandb/debug.log

@@ -0,0 +1,34 @@
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/logs/debug.log
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/logs/debug-internal.log
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():568] calling init triggers
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {}
+config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 25}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():618] starting backend
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():622] setting up manager
+2023-11-01 01:46:58,109 INFO    MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-11-01 01:46:58,109 INFO    MainThread:34070 [wandb_init.py:init():628] backend started and connected
+2023-11-01 01:46:58,111 INFO    MainThread:34070 [wandb_init.py:init():720] updated telemetry
+2023-11-01 01:46:58,112 INFO    MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2023-11-01 01:46:59,468 INFO    MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version
+2023-11-01 01:47:00,159 INFO    MainThread:34070 [wandb_run.py:_on_init():2229] got version response 
+2023-11-01 01:47:00,159 INFO    MainThread:34070 [wandb_init.py:init():804] starting run threads in backend
+2023-11-01 01:47:06,941 INFO    MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg
+2023-11-01 01:47:06,942 INFO    MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw
+2023-11-01 01:47:06,942 INFO    MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams.
+2023-11-01 01:47:06,942 INFO    MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed.
+2023-11-01 01:47:06,943 INFO    MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/mj672tzu
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_restore():2151] restore
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_restore():2157] restore done
+2023-11-01 02:11:05,119 INFO    MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history
+2023-11-01 02:11:05,120 INFO    MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary
+2023-11-01 02:11:05,127 INFO    MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files

wandb/latest-run/files/conda-environment.yaml

@@ -0,0 +1,197 @@
+name: env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - asttokens=2.4.0=pyhd8ed1ab_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0
+  - ca-certificates=2023.7.22=hbcca054_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - debugpy=1.6.7=py38h6a678d5_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - ipykernel=6.25.2=pyh2140261_0
+  - ipython=8.12.0=pyh41d4057_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jupyter_client=7.3.4=pyhd8ed1ab_0
+  - jupyter_core=5.4.0=py38h578d9bd_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - ncurses=6.4=h6a678d5_0
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - openssl=3.0.11=h7f8727e_2
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pip=23.2.1=py38h06a4308_0
+  - platformdirs=3.11.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.39=pyha770c72_0
+  - prompt_toolkit=3.0.39=hd8ed1ab_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pygments=2.16.1=pyhd8ed1ab_0
+  - python=3.8.18=h955ad1f_0
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.8=2_cp38
+  - pyzmq=25.1.0=py38h6a678d5_0
+  - readline=8.2=h5eee18b_0
+  - setuptools=68.0.0=py38h06a4308_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - tk=8.6.12=h1ccaba5_0
+  - tornado=6.1=py38h0a891b7_3
+  - traitlets=5.11.2=pyhd8ed1ab_0
+  - typing-extensions=4.8.0=hd8ed1ab_0
+  - typing_extensions=4.8.0=pyha770c72_0
+  - wcwidth=0.2.8=pyhd8ed1ab_0
+  - wheel=0.38.4=py38h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - zeromq=4.3.4=h2531618_0
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+    - accelerate==0.22.0
+    - aiofiles==23.2.1
+    - aiohttp==3.8.5
+    - aiosignal==1.3.1
+    - alembic==1.12.0
+    - altair==5.1.2
+    - annotated-types==0.6.0
+    - anyio==3.7.1
+    - appdirs==1.4.4
+    - async-timeout==4.0.3
+    - attrs==23.1.0
+    - certifi==2022.12.7
+    - charset-normalizer==2.1.1
+    - click==8.1.7
+    - cmake==3.25.0
+    - colorama==0.4.6
+    - coloredlogs==15.0.1
+    - colorlog==6.7.0
+    - contourpy==1.1.0
+    - cycler==0.11.0
+    - datasets==2.14.5
+    - diffusers==0.21.0
+    - dill==0.3.7
+    - docker-pycreds==0.4.0
+    - einops==0.6.1
+    - exceptiongroup==1.1.3
+    - fastapi==0.104.1
+    - ffmpy==0.3.1
+    - filelock==3.9.0
+    - flatbuffers==23.5.26
+    - fonttools==4.42.1
+    - frozenlist==1.4.0
+    - fsspec==2023.6.0
+    - gitdb==4.0.11
+    - gitpython==3.1.40
+    - gradio==4.0.0
+    - gradio-client==0.7.0
+    - greenlet==3.0.0
+    - h11==0.14.0
+    - httpcore==0.18.0
+    - httpx==0.25.0
+    - huggingface-hub==0.17.1
+    - humanfriendly==10.0
+    - idna==3.4
+    - importlib-metadata==6.8.0
+    - importlib-resources==6.0.1
+    - jinja2==3.1.2
+    - joblib==1.3.2
+    - jsonschema==4.19.2
+    - jsonschema-specifications==2023.7.1
+    - kiwisolver==1.4.5
+    - lightning-utilities==0.9.0
+    - lit==15.0.7
+    - lpips==0.1.4
+    - mako==1.2.4
+    - markdown-it-py==3.0.0
+    - markupsafe==2.1.2
+    - matplotlib==3.7.3
+    - mdurl==0.1.2
+    - mpmath==1.2.1
+    - multidict==6.0.4
+    - multiprocess==0.70.15
+    - networkx==3.0
+    - numpy==1.24.1
+    - nvidia-cublas-cu11==11.10.3.66
+    - nvidia-cuda-cupti-cu11==11.7.101
+    - nvidia-cuda-nvrtc-cu11==11.7.99
+    - nvidia-cuda-runtime-cu11==11.7.99
+    - nvidia-cudnn-cu11==8.5.0.96
+    - nvidia-cufft-cu11==10.9.0.58
+    - nvidia-curand-cu11==10.2.10.91
+    - nvidia-cusolver-cu11==11.4.0.1
+    - nvidia-cusparse-cu11==11.7.4.91
+    - nvidia-nccl-cu11==2.14.3
+    - nvidia-nvtx-cu11==11.7.91
+    - onnx==1.15.0
+    - onnxruntime==1.16.1
+    - onnxruntime-gpu==1.16.1
+    - optuna==3.4.0
+    - orjson==3.9.10
+    - packaging==23.1
+    - pandas==2.0.3
+    - pathtools==0.1.2
+    - pillow==9.3.0
+    - pkgutil-resolve-name==1.3.10
+    - protobuf==4.24.4
+    - psutil==5.9.5
+    - pyarrow==13.0.0
+    - pydantic==2.4.2
+    - pydantic-core==2.10.1
+    - pydub==0.25.1
+    - pyparsing==3.1.1
+    - python-multipart==0.0.6
+    - pytz==2023.3.post1
+    - pyyaml==6.0.1
+    - referencing==0.30.2
+    - regex==2023.8.8
+    - requests==2.28.1
+    - rich==13.6.0
+    - rpds-py==0.10.6
+    - safetensors==0.3.3
+    - scikit-learn==1.3.1
+    - scipy==1.10.1
+    - semantic-version==2.10.0
+    - sentry-sdk==1.32.0
+    - setproctitle==1.3.3
+    - shellingham==1.5.4
+    - smmap==5.0.1
+    - sniffio==1.3.0
+    - sqlalchemy==2.0.22
+    - starlette==0.27.0
+    - sympy==1.11.1
+    - threadpoolctl==3.2.0
+    - tokenizers==0.13.3
+    - tomlkit==0.12.0
+    - toolz==0.12.0
+    - torch==2.0.1
+    - torch-fidelity==0.3.0
+    - torch-summary==1.4.5
+    - torchaudio==2.0.2
+    - torchmetrics==1.1.2
+    - torchvision==0.15.2
+    - tqdm==4.66.1
+    - transformers==4.33.1
+    - triton==2.0.0
+    - typer==0.9.0
+    - tzdata==2023.3
+    - urllib3==1.26.13
+    - uvicorn==0.23.2
+    - wandb==0.15.12
+    - websockets==11.0.3
+    - xxhash==3.3.0
+    - yarl==1.9.2
+    - zipp==3.16.2
+prefix: /opt/conda/envs/env

wandb/latest-run/files/config.yaml

@@ -0,0 +1,53 @@
+wandb_version: 1
+
+optimizer:
+  desc: null
+  value: Adam
+architecture:
+  desc: null
+  value: Efficientnet B0
+dataset:
+  desc: null
+  value: Food101
+epochs:
+  desc: null
+  value: 25
+_wandb:
+  desc: null
+  value:
+    python_version: 3.8.18
+    cli_version: 0.15.12
+    framework: huggingface
+    huggingface_version: 4.33.1
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1698788818.109446
+    t:
+      1:
+      - 1
+      - 11
+      - 35
+      - 41
+      - 49
+      - 55
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 35
+      - 41
+      - 49
+      - 55
+      - 71
+      - 105
+      3:
+      - 2
+      - 16
+      - 23
+      4: 3.8.18
+      5: 0.15.12
+      6: 4.33.1
+      8:
+      - 5
+      13: linux-x86_64

wandb/latest-run/files/output.log

@@ -0,0 +1,2 @@
+
+Saving Model At: save_model/best_model.pth

wandb/latest-run/files/requirements.txt

@@ -0,0 +1,168 @@
+accelerate==0.22.0
+aiofiles==23.2.1
+aiohttp==3.8.5
+aiosignal==1.3.1
+alembic==1.12.0
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+appdirs==1.4.4
+asttokens==2.4.0
+async-timeout==4.0.3
+attrs==23.1.0
+backcall==0.2.0
+backports.functools-lru-cache==1.6.5
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.7
+cmake==3.25.0
+colorama==0.4.6
+coloredlogs==15.0.1
+colorlog==6.7.0
+comm==0.1.4
+contourpy==1.1.0
+cycler==0.11.0
+datasets==2.14.5
+debugpy==1.6.7
+decorator==5.1.1
+diffusers==0.21.0
+dill==0.3.7
+docker-pycreds==0.4.0
+einops==0.6.1
+entrypoints==0.4
+exceptiongroup==1.1.3
+executing==1.2.0
+fastapi==0.104.1
+ffmpy==0.3.1
+filelock==3.9.0
+flatbuffers==23.5.26
+fonttools==4.42.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+gitdb==4.0.11
+gitpython==3.1.40
+gradio-client==0.7.0
+gradio==4.0.0
+greenlet==3.0.0
+h11==0.14.0
+httpcore==0.18.0
+httpx==0.25.0
+huggingface-hub==0.17.1
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.0.1
+ipykernel==6.25.2
+ipython==8.12.0
+jedi==0.19.1
+jinja2==3.1.2
+joblib==1.3.2
+jsonschema-specifications==2023.7.1
+jsonschema==4.19.2
+jupyter-client==7.3.4
+jupyter-core==5.4.0
+kiwisolver==1.4.5
+lightning-utilities==0.9.0
+lit==15.0.7
+lpips==0.1.4
+mako==1.2.4
+markdown-it-py==3.0.0
+markupsafe==2.1.2
+matplotlib-inline==0.1.6
+matplotlib==3.7.3
+mdurl==0.1.2
+mpmath==1.2.1
+multidict==6.0.4
+multiprocess==0.70.15
+nest-asyncio==1.5.8
+networkx==3.0
+numpy==1.24.1
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+onnx==1.15.0
+onnxruntime-gpu==1.16.1
+onnxruntime==1.16.1
+optuna==3.4.0
+orjson==3.9.10
+packaging==23.1
+pandas==2.0.3
+parso==0.8.3
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.3.0
+pip==23.2.1
+pkgutil-resolve-name==1.3.10
+platformdirs==3.11.0
+prompt-toolkit==3.0.39
+protobuf==4.24.4
+psutil==5.9.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==13.0.0
+pydantic-core==2.10.1
+pydantic==2.4.2
+pydub==0.25.1
+pygments==2.16.1
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.0
+referencing==0.30.2
+regex==2023.8.8
+requests==2.28.1
+rich==13.6.0
+rpds-py==0.10.6
+safetensors==0.3.3
+scikit-learn==1.3.1
+scipy==1.10.1
+semantic-version==2.10.0
+sentry-sdk==1.32.0
+setproctitle==1.3.3
+setuptools==68.0.0
+shellingham==1.5.4
+six==1.16.0
+smmap==5.0.1
+sniffio==1.3.0
+sqlalchemy==2.0.22
+stack-data==0.6.2
+starlette==0.27.0
+sympy==1.11.1
+threadpoolctl==3.2.0
+tokenizers==0.13.3
+tomlkit==0.12.0
+toolz==0.12.0
+torch-fidelity==0.3.0
+torch-summary==1.4.5
+torch==2.0.1
+torchaudio==2.0.2
+torchmetrics==1.1.2
+torchvision==0.15.2
+tornado==6.1
+tqdm==4.66.1
+traitlets==5.11.2
+transformers==4.33.1
+triton==2.0.0
+typer==0.9.0
+typing-extensions==4.8.0
+tzdata==2023.3
+urllib3==1.26.13
+uvicorn==0.23.2
+wandb==0.15.12
+wcwidth==0.2.8
+websockets==11.0.3
+wheel==0.38.4
+xxhash==3.3.0
+yarl==1.9.2
+zipp==3.16.2

wandb/latest-run/files/wandb-metadata.json

@@ -0,0 +1,254 @@
+{
+    "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17",
+    "python": "3.8.18",
+    "heartbeatAt": "2023-10-31T21:47:00.334647",
+    "startedAt": "2023-10-31T21:46:58.105196",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "hyperpara.py",
+    "codePathLocal": "hyperpara.py",
+    "codePath": "hyperpara.py",
+    "host": "76ecfeea2960",
+    "username": "root",
+    "executable": "/opt/conda/envs/env/bin/python",
+    "cpu_count": 20,
+    "cpu_count_logical": 40,
+    "cpu_freq": {
+        "current": 1.519625,
+        "min": 1200.0,
+        "max": 3600.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2.698,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.197,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.197,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.208,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.297,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.698,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.499,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.465,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.299,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.697,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.476,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.208,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.698,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.196,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.205,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.2,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.204,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.299,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.7,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.498,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.268,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.2,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.251,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.697,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.534,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 1758.95463180542,
+            "used": 866.9305191040039
+        }
+    },
+    "gpu": "Tesla V100-DGXS-32GB",
+    "gpu_count": 4,
+    "gpu_devices": [
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        },
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        },
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        },
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        }
+    ],
+    "memory": {
+        "total": 251.8238754272461
+    }
+}

wandb/latest-run/files/wandb-summary.json

@@ -0,0 +1 @@
+{"Train Loss": 1.3167718648910522, "Train Accuracy": 0.5293803811073303, "Validation Loss": 1.200211763381958, "Validation Accuracy": 0.5658254027366638, "_timestamp": 1698790257.8465717, "_runtime": 1439.737125635147, "_step": 0, "_wandb": {"runtime": 1437}}

wandb/latest-run/logs/debug-internal.log

@@ -0,0 +1,797 @@
+2023-11-01 01:46:58,588 INFO    StreamThr :34227 [internal.py:wandb_internal():86] W&B internal server running at pid: 34227, started at: 2023-11-01 01:46:58.587580
+2023-11-01 01:46:58,590 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status
+2023-11-01 01:46:58,592 INFO    WriterThread:34227 [datastore.py:open_for_write():85] open: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb
+2023-11-01 01:46:58,595 DEBUG   SenderThread:34227 [sender.py:send():380] send: header
+2023-11-01 01:46:58,595 DEBUG   SenderThread:34227 [sender.py:send():380] send: run
+2023-11-01 01:46:59,448 INFO    SenderThread:34227 [dir_watcher.py:__init__():211] watching files in: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files
+2023-11-01 01:46:59,448 INFO    SenderThread:34227 [sender.py:_start_run_threads():1122] run started: mj672tzu with start time 1698788818.109446
+2023-11-01 01:46:59,469 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: check_version
+2023-11-01 01:46:59,470 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: check_version
+2023-11-01 01:47:00,265 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: run_start
+2023-11-01 01:47:00,267 DEBUG   HandlerThread:34227 [system_info.py:__init__():32] System info init
+2023-11-01 01:47:00,267 DEBUG   HandlerThread:34227 [system_info.py:__init__():47] System info init done
+2023-11-01 01:47:00,267 INFO    HandlerThread:34227 [system_monitor.py:start():194] Starting system monitor
+2023-11-01 01:47:00,267 INFO    SystemMonitor:34227 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2023-11-01 01:47:00,267 INFO    HandlerThread:34227 [system_monitor.py:probe():214] Collecting system info
+2023-11-01 01:47:00,268 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started cpu monitoring
+2023-11-01 01:47:00,269 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started disk monitoring
+2023-11-01 01:47:00,270 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started gpu monitoring
+2023-11-01 01:47:00,271 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started memory monitoring
+2023-11-01 01:47:00,272 INFO    SystemMonitor:34227 [interfaces.py:start():190] Started network monitoring
+2023-11-01 01:47:00,334 DEBUG   HandlerThread:34227 [system_info.py:probe():196] Probing system
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [gitlib.py:_init_repo():53] git repository is invalid
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_info.py:probe():244] Probing system done
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_monitor.py:probe():223] {'os': 'Linux-4.15.0-55-generic-x86_64-with-glibc2.17', 'python': '3.8.18', 'heartbeatAt': '2023-10-31T21:47:00.334647', 'startedAt': '2023-10-31T21:46:58.105196', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'hyperpara.py', 'codePathLocal': 'hyperpara.py', 'codePath': 'hyperpara.py', 'host': '76ecfeea2960', 'username': 'root', 'executable': '/opt/conda/envs/env/bin/python', 'cpu_count': 20, 'cpu_count_logical': 40, 'cpu_freq': {'current': 1.519625, 'min': 1200.0, 'max': 3600.0}, 'cpu_freq_per_core': [{'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.197, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.297, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.499, 'min': 1200.0, 'max': 3600.0}, {'current': 1.465, 'min': 1200.0, 'max': 3600.0}, {'current': 1.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 1.697, 'min': 1200.0, 'max': 3600.0}, {'current': 1.476, 'min': 1200.0, 'max': 3600.0}, {'current': 2.208, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 2.698, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.196, 'min': 1200.0, 'max': 3600.0}, {'current': 1.205, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.204, 'min': 1200.0, 'max': 3600.0}, {'current': 1.299, 'min': 1200.0, 'max': 3600.0}, {'current': 2.7, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.498, 'min': 1200.0, 'max': 3600.0}, {'current': 1.268, 'min': 1200.0, 'max': 3600.0}, {'current': 1.2, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.251, 'min': 1200.0, 'max': 3600.0}, {'current': 1.697, 'min': 1200.0, 'max': 3600.0}, {'current': 1.534, 'min': 1200.0, 'max': 3600.0}, {'current': 2.198, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}, {'current': 1.199, 'min': 1200.0, 'max': 3600.0}], 'disk': {'/': {'total': 1758.95463180542, 'used': 866.9305191040039}}, 'gpu': 'Tesla V100-DGXS-32GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}, {'name': 'Tesla V100-DGXS-32GB', 'memory_total': 34359738368}], 'memory': {'total': 251.8238754272461}}
+2023-11-01 01:47:00,335 INFO    HandlerThread:34227 [system_monitor.py:probe():224] Finished collecting system info
+2023-11-01 01:47:00,335 INFO    HandlerThread:34227 [system_monitor.py:probe():227] Publishing system info
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment
+2023-11-01 01:47:00,335 DEBUG   HandlerThread:34227 [system_info.py:_save_pip():68] Saving pip packages done
+2023-11-01 01:47:00,336 DEBUG   HandlerThread:34227 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment
+2023-11-01 01:47:00,451 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt
+2023-11-01 01:47:00,452 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml
+2023-11-01 01:47:06,925 DEBUG   HandlerThread:34227 [system_info.py:_save_conda():87] Saving conda packages done
+2023-11-01 01:47:06,927 INFO    HandlerThread:34227 [system_monitor.py:probe():229] Finished publishing system info
+2023-11-01 01:47:06,927 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:06,928 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: keepalive
+2023-11-01 01:47:06,929 DEBUG   SenderThread:34227 [sender.py:send():380] send: files
+2023-11-01 01:47:06,929 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-metadata.json with policy now
+2023-11-01 01:47:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:07,442 DEBUG   SenderThread:34227 [sender.py:send():380] send: telemetry
+2023-11-01 01:47:07,450 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml
+2023-11-01 01:47:07,451 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json
+2023-11-01 01:47:09,285 INFO    wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /tmp/tmpqi5xsr4gwandb/rhtw7848-wandb-metadata.json
+2023-11-01 01:47:09,443 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:14,444 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:19,445 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:21,940 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:21,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:21,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:25,370 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:30,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:31,458 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml
+2023-11-01 01:47:35,747 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:36,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:36,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:36,981 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:41,296 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:46,297 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:51,298 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:47:51,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:47:51,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:47:51,981 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:47:57,290 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:00,272 DEBUG   SystemMonitor:34227 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2023-11-01 01:48:00,276 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:48:03,277 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:06,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:06,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:08,317 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:13,318 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:18,319 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:21,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:21,941 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:21,981 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:24,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:29,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:30,278 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:48:35,279 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:36,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:36,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:40,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:45,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:50,333 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:48:51,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:48:51,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:48:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:48:56,275 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:00,280 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:49:01,281 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:06,282 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:06,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:12,279 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:17,279 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:21,941 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:21,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:22,346 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:27,347 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:30,282 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:49:33,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:36,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:38,291 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:43,292 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:48,293 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:51,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:49:51,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:49:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:49:53,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:49:58,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:00,284 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:50:04,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:06,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:09,358 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:14,359 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:19,359 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:21,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:21,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:24,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:29,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:30,286 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:50:35,287 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:36,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:36,942 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:40,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:45,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:50,362 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:50:51,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:50:51,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:50:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:50:56,350 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:00,288 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:51:02,290 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:07,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:09,530 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 01:51:11,530 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 01:51:12,954 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:17,954 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:21,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:21,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:21,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:23,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:28,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:30,290 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:51:34,292 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:36,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:39,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:44,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:49,327 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:51:51,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:51:51,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:51:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:51:55,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:00,293 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:52:01,295 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:06,295 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:06,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:11,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:16,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:21,327 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:21,942 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:21,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:26,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:30,295 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:52:32,297 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:38,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:43,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:48,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:51,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:52:51,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:52:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:52:53,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:52:58,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:00,297 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:53:04,299 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:06,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:06,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:09,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:14,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:19,370 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:21,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:21,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:25,282 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:30,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:30,299 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:53:35,300 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:36,943 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:41,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:46,287 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:51,287 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:53:51,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:53:51,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:53:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:53:56,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:00,301 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:54:02,302 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:06,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:06,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:07,360 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:12,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:17,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:21,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:21,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:23,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:28,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:30,302 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:54:34,304 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:36,943 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:36,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:39,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:44,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:49,368 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:54:51,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:54:51,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:54:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:54:55,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:00,306 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:55:01,307 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:06,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:06,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:06,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:12,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:17,308 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:21,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:21,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:22,370 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:27,371 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:30,308 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:55:33,309 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:36,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:36,944 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:38,354 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:43,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:48,356 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:51,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:55:51,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:55:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:55:54,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:55:59,306 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:00,309 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:56:04,311 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:06,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:06,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:09,328 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:14,328 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:19,329 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:21,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:21,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:24,356 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:29,357 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:30,311 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:56:35,313 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:36,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:36,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:36,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:40,339 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:45,340 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:50,340 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:56:51,944 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:56:51,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:56:51,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:56:55,379 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:00,313 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:57:01,314 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:06,315 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:06,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:06,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:06,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:12,281 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:17,282 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:21,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:21,945 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:21,985 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:22,295 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:27,296 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:30,315 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:57:32,316 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:36,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:37,365 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:42,366 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:47,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:51,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:57:51,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:57:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:57:53,336 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:57:58,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:00,318 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:58:04,319 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:06,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:06,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:09,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:14,338 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:19,339 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:21,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:21,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:24,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:29,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:30,320 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:58:35,322 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:36,945 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:36,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:40,450 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:45,451 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:50,451 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:58:51,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:58:51,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:58:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:58:56,294 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:00,322 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:59:01,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:06,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:06,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:06,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:11,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:16,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:21,352 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:21,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:21,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:27,314 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:30,324 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 01:59:32,325 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:36,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:38,312 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:43,313 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:48,313 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:51,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 01:59:51,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 01:59:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 01:59:54,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 01:59:59,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:00,326 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:00:04,328 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:06,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:06,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:09,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:14,338 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:19,338 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:21,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:21,946 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:25,276 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:30,277 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:30,329 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:00:35,331 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:36,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:36,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:40,373 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:45,374 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:50,375 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:00:51,946 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:00:51,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:00:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:00:56,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:00,330 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:01:01,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:06,332 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:06,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:06,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:12,304 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:17,305 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:21,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:21,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:22,315 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:27,316 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:30,332 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:01:32,334 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:36,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:36,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:37,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:42,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:47,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:51,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:01:51,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:01:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:01:52,385 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:01:57,385 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:00,335 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:02:03,337 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:06,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:06,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:09,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:14,284 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:19,284 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:21,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:21,947 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:24,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:29,327 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:30,338 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:02:34,340 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:36,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:36,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:39,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:44,352 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:49,353 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:02:51,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:02:51,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:02:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:02:55,275 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:00,276 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:00,341 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:03:05,344 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:06,947 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:06,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:10,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:15,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:20,382 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:21,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:21,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:25,389 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:30,344 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:03:31,346 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:36,347 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:36,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:36,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:41,357 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:46,357 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:51,358 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:03:51,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:03:51,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:03:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:03:56,622 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:00,347 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:04:02,349 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:06,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:06,948 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:08,335 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:13,335 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:18,336 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:21,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:21,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:24,298 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:29,298 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:30,351 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:04:34,353 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:36,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:36,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:39,360 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:44,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:49,362 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:51,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:04:51,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:04:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:04:54,372 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:04:59,373 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:00,352 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:05:05,354 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:06,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:06,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:11,299 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:16,299 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:21,300 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:21,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:21,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:27,283 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:30,354 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:05:32,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:36,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:36,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:36,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:38,348 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:43,348 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:48,349 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:51,948 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:05:51,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:05:51,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:05:53,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:05:58,367 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:00,357 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:06:04,359 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:06,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:06,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:06,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:10,333 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:15,334 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:20,335 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:21,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:21,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:21,989 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:26,315 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:30,359 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:06:31,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:36,361 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:36,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:36,949 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:36,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:42,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:47,355 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:51,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:06:51,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:06:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:06:53,293 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:06:58,293 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:00,361 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:07:03,362 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:06,949 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:06,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:09,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:14,320 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:19,321 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:21,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:21,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:24,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:29,379 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:30,363 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:07:35,365 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:36,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:36,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:36,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:41,323 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:46,323 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:51,324 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:07:51,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:07:51,950 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:07:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:07:56,393 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:00,366 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:08:02,368 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:06,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:06,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:07,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:12,387 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:17,388 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:21,950 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:21,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:23,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:28,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:30,368 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:08:34,369 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:36,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:36,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:39,446 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:44,447 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:49,447 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:08:51,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:08:51,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:08:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:08:55,285 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:00,286 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:00,372 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:09:05,374 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:06,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:06,951 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:11,276 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:16,277 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:21,278 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:21,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:21,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:21,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:26,312 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:30,376 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:09:31,377 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:36,378 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:36,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:41,398 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:46,398 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:51,399 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:09:51,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:09:51,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:09:51,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:09:57,396 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:00,379 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:10:03,381 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:06,951 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:06,952 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:06,993 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:09,330 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:14,331 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:19,331 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:21,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:21,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:21,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:25,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:30,326 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:30,382 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:10:35,383 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:36,952 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:36,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:41,349 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:46,350 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:51,351 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:51,953 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: stop_status
+2023-11-01 02:10:51,953 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:10:51,953 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: stop_status
+2023-11-01 02:10:57,312 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:57,847 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: partial_history
+2023-11-01 02:10:57,849 DEBUG   SenderThread:34227 [sender.py:send():380] send: history
+2023-11-01 02:10:57,849 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: summary_record
+2023-11-01 02:10:57,849 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
+2023-11-01 02:10:57,889 DEBUG   SenderThread:34227 [sender.py:send():380] send: telemetry
+2023-11-01 02:10:57,889 DEBUG   SenderThread:34227 [sender.py:send():380] send: exit
+2023-11-01 02:10:57,889 INFO    SenderThread:34227 [sender.py:send_exit():585] handling exit code: 0
+2023-11-01 02:10:57,889 INFO    SenderThread:34227 [sender.py:send_exit():587] handling runtime: 1437
+2023-11-01 02:10:57,890 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
+2023-11-01 02:10:57,890 INFO    SenderThread:34227 [sender.py:send_exit():593] send defer
+2023-11-01 02:10:57,890 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:57,890 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 0
+2023-11-01 02:10:57,891 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:57,891 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 0
+2023-11-01 02:10:57,891 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 1
+2023-11-01 02:10:57,891 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:57,891 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 1
+2023-11-01 02:10:57,891 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:57,892 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 1
+2023-11-01 02:10:57,892 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 2
+2023-11-01 02:10:57,892 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:57,892 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 2
+2023-11-01 02:10:57,892 INFO    HandlerThread:34227 [system_monitor.py:finish():203] Stopping system monitor
+2023-11-01 02:10:57,892 DEBUG   SystemMonitor:34227 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2023-11-01 02:10:57,893 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined cpu monitor
+2023-11-01 02:10:57,893 DEBUG   SystemMonitor:34227 [system_monitor.py:_start():183] Publishing last batch of metrics
+2023-11-01 02:10:57,893 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined disk monitor
+2023-11-01 02:10:57,953 INFO    Thread-133:34227 [dir_watcher.py:_on_file_created():271] file/dir created: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json
+2023-11-01 02:10:59,649 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined gpu monitor
+2023-11-01 02:10:59,650 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined memory monitor
+2023-11-01 02:10:59,650 INFO    HandlerThread:34227 [interfaces.py:finish():202] Joined network monitor
+2023-11-01 02:10:59,650 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:10:59,651 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,651 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 2
+2023-11-01 02:10:59,652 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 3
+2023-11-01 02:10:59,652 DEBUG   SenderThread:34227 [sender.py:send():380] send: stats
+2023-11-01 02:10:59,652 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:10:59,653 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,653 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 3
+2023-11-01 02:10:59,653 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 3
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 4
+2023-11-01 02:10:59,654 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,654 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 4
+2023-11-01 02:10:59,654 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 4
+2023-11-01 02:10:59,654 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 5
+2023-11-01 02:10:59,655 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,655 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 5
+2023-11-01 02:10:59,655 DEBUG   SenderThread:34227 [sender.py:send():380] send: summary
+2023-11-01 02:10:59,656 INFO    SenderThread:34227 [sender.py:_save_file():1378] saving file wandb-summary.json with policy end
+2023-11-01 02:10:59,656 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,656 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 5
+2023-11-01 02:10:59,656 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 6
+2023-11-01 02:10:59,656 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:10:59,657 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 6
+2023-11-01 02:10:59,657 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:10:59,657 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 6
+2023-11-01 02:10:59,662 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: status_report
+2023-11-01 02:10:59,849 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:10:59,954 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json
+2023-11-01 02:11:00,056 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 7
+2023-11-01 02:11:00,056 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:00,056 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,057 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 7
+2023-11-01 02:11:00,057 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,057 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 7
+2023-11-01 02:11:00,230 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 8
+2023-11-01 02:11:00,231 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,231 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 8
+2023-11-01 02:11:00,231 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,231 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 8
+2023-11-01 02:11:00,232 INFO    SenderThread:34227 [job_builder.py:build():281] Attempting to build job artifact
+2023-11-01 02:11:00,232 INFO    SenderThread:34227 [job_builder.py:_get_source_type():404] no source found
+2023-11-01 02:11:00,232 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 9
+2023-11-01 02:11:00,233 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,233 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 9
+2023-11-01 02:11:00,233 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,233 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 9
+2023-11-01 02:11:00,233 INFO    SenderThread:34227 [dir_watcher.py:finish():358] shutting down directory watcher
+2023-11-01 02:11:00,849 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:00,955 INFO    Thread-133:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml
+2023-11-01 02:11:00,955 INFO    SenderThread:34227 [dir_watcher.py:_on_file_modified():288] file/dir modified: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 02:11:00,956 INFO    SenderThread:34227 [dir_watcher.py:finish():388] scan: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files
+2023-11-01 02:11:00,956 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log output.log
+2023-11-01 02:11:00,957 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt requirements.txt
+2023-11-01 02:11:00,962 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml conda-environment.yaml
+2023-11-01 02:11:00,963 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json wandb-summary.json
+2023-11-01 02:11:00,967 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml config.yaml
+2023-11-01 02:11:00,980 INFO    SenderThread:34227 [dir_watcher.py:finish():402] scan save: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-metadata.json wandb-metadata.json
+2023-11-01 02:11:00,981 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 10
+2023-11-01 02:11:00,981 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:00,982 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:00,989 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 10
+2023-11-01 02:11:00,990 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:00,990 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 10
+2023-11-01 02:11:00,990 INFO    SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher
+2023-11-01 02:11:01,845 INFO    wandb-upload_0:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/output.log
+2023-11-01 02:11:01,850 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:01,851 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:02,100 INFO    wandb-upload_1:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/requirements.txt
+2023-11-01 02:11:02,147 INFO    wandb-upload_3:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/wandb-summary.json
+2023-11-01 02:11:02,154 INFO    wandb-upload_4:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/config.yaml
+2023-11-01 02:11:02,851 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:02,852 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:03,196 INFO    wandb-upload_2:34227 [upload_job.py:push():131] Uploaded file /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/files/conda-environment.yaml
+2023-11-01 02:11:03,396 INFO    Thread-132:34227 [sender.py:transition_state():613] send defer: 11
+2023-11-01 02:11:03,397 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,397 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 11
+2023-11-01 02:11:03,398 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,398 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 11
+2023-11-01 02:11:03,398 INFO    SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher
+2023-11-01 02:11:03,398 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 12
+2023-11-01 02:11:03,398 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,399 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 12
+2023-11-01 02:11:03,399 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,399 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 12
+2023-11-01 02:11:03,399 INFO    SenderThread:34227 [file_stream.py:finish():595] file stream finish called
+2023-11-01 02:11:03,757 INFO    SenderThread:34227 [file_stream.py:finish():599] file stream finish is done
+2023-11-01 02:11:03,757 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 13
+2023-11-01 02:11:03,758 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,758 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 13
+2023-11-01 02:11:03,758 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,758 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 13
+2023-11-01 02:11:03,759 INFO    SenderThread:34227 [sender.py:transition_state():613] send defer: 14
+2023-11-01 02:11:03,759 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: defer
+2023-11-01 02:11:03,759 DEBUG   SenderThread:34227 [sender.py:send():380] send: final
+2023-11-01 02:11:03,760 INFO    HandlerThread:34227 [handler.py:handle_request_defer():172] handle defer: 14
+2023-11-01 02:11:03,760 DEBUG   SenderThread:34227 [sender.py:send():380] send: footer
+2023-11-01 02:11:03,760 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: defer
+2023-11-01 02:11:03,760 INFO    SenderThread:34227 [sender.py:send_request_defer():609] handle sender defer: 14
+2023-11-01 02:11:03,762 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: internal_messages
+2023-11-01 02:11:03,763 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: poll_exit
+2023-11-01 02:11:03,763 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: poll_exit
+2023-11-01 02:11:03,764 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: server_info
+2023-11-01 02:11:03,764 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: get_summary
+2023-11-01 02:11:03,765 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: sampled_history
+2023-11-01 02:11:03,765 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: server_info
+2023-11-01 02:11:03,771 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: job_info
+2023-11-01 02:11:04,116 DEBUG   SenderThread:34227 [sender.py:send_request():407] send_request: job_info
+2023-11-01 02:11:04,117 DEBUG   HandlerThread:34227 [handler.py:handle_request():146] handle_request: shutdown
+2023-11-01 02:11:04,117 INFO    HandlerThread:34227 [handler.py:finish():866] shutting down handler
+2023-11-01 02:11:04,772 INFO    WriterThread:34227 [datastore.py:close():294] close: /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/run-mj672tzu.wandb
+2023-11-01 02:11:05,116 INFO    SenderThread:34227 [sender.py:finish():1534] shutting down sender
+2023-11-01 02:11:05,116 INFO    SenderThread:34227 [file_pusher.py:finish():175] shutting down file pusher
+2023-11-01 02:11:05,116 INFO    SenderThread:34227 [file_pusher.py:join():181] waiting for file pusher

wandb/latest-run/logs/debug.log

@@ -0,0 +1,34 @@
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Current SDK version is 0.15.12
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Configure stats pid to 34070
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /root/.config/wandb/settings
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from /jash/Workspace/Food/wandb/settings
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
+2023-11-01 01:46:58,106 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'hyperpara.py', 'program_abspath': '/jash/Workspace/Food/hyperpara.py', 'program': 'hyperpara.py'}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_setup.py:_flush():76] Applying login settings: {'api_key': '***REDACTED***'}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:_log_setup():528] Logging user logs to /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/logs/debug.log
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:_log_setup():529] Logging internal logs to /jash/Workspace/Food/wandb/run-20231101_014658-mj672tzu/logs/debug-internal.log
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():568] calling init triggers
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():575] wandb.init called with sweep_config: {}
+config: {'optimizer': 'Adam', 'architecture': 'Efficientnet B0', 'dataset': 'Food101', 'epochs': 25}
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():618] starting backend
+2023-11-01 01:46:58,107 INFO    MainThread:34070 [wandb_init.py:init():622] setting up manager
+2023-11-01 01:46:58,109 INFO    MainThread:34070 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2023-11-01 01:46:58,109 INFO    MainThread:34070 [wandb_init.py:init():628] backend started and connected
+2023-11-01 01:46:58,111 INFO    MainThread:34070 [wandb_init.py:init():720] updated telemetry
+2023-11-01 01:46:58,112 INFO    MainThread:34070 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout
+2023-11-01 01:46:59,468 INFO    MainThread:34070 [wandb_run.py:_on_init():2220] communicating current version
+2023-11-01 01:47:00,159 INFO    MainThread:34070 [wandb_run.py:_on_init():2229] got version response 
+2023-11-01 01:47:00,159 INFO    MainThread:34070 [wandb_init.py:init():804] starting run threads in backend
+2023-11-01 01:47:06,941 INFO    MainThread:34070 [wandb_run.py:_console_start():2199] atexit reg
+2023-11-01 01:47:06,942 INFO    MainThread:34070 [wandb_run.py:_redirect():2054] redirect: wrap_raw
+2023-11-01 01:47:06,942 INFO    MainThread:34070 [wandb_run.py:_redirect():2119] Wrapping output streams.
+2023-11-01 01:47:06,942 INFO    MainThread:34070 [wandb_run.py:_redirect():2144] Redirects installed.
+2023-11-01 01:47:06,943 INFO    MainThread:34070 [wandb_init.py:init():845] run started, returning control to user process
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_finish():1934] finishing run maa_64/food-app/mj672tzu
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_atexit_cleanup():2168] got exitcode: 0
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_restore():2151] restore
+2023-11-01 02:10:57,847 INFO    MainThread:34070 [wandb_run.py:_restore():2157] restore done
+2023-11-01 02:11:05,119 INFO    MainThread:34070 [wandb_run.py:_footer_history_summary_info():3599] rendering history
+2023-11-01 02:11:05,120 INFO    MainThread:34070 [wandb_run.py:_footer_history_summary_info():3631] rendering summary
+2023-11-01 02:11:05,127 INFO    MainThread:34070 [wandb_run.py:_footer_sync_info():3558] logging synced files

wandb/run-20231031_141809-v2q0e45b/files/conda-environment.yaml

@@ -0,0 +1,162 @@
+name: env
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main
+  - _openmp_mutex=5.1=1_gnu
+  - asttokens=2.4.0=pyhd8ed1ab_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0
+  - ca-certificates=2023.7.22=hbcca054_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - debugpy=1.6.7=py38h6a678d5_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - ipykernel=6.25.2=pyh2140261_0
+  - ipython=8.12.0=pyh41d4057_0
+  - jedi=0.19.1=pyhd8ed1ab_0
+  - jupyter_client=7.3.4=pyhd8ed1ab_0
+  - jupyter_core=5.4.0=py38h578d9bd_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=11.2.0=h1234567_1
+  - libgomp=11.2.0=h1234567_1
+  - libsodium=1.0.18=h36c2ea0_1
+  - libstdcxx-ng=11.2.0=h1234567_1
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - ncurses=6.4=h6a678d5_0
+  - nest-asyncio=1.5.8=pyhd8ed1ab_0
+  - openssl=3.0.11=h7f8727e_2
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pip=23.2.1=py38h06a4308_0
+  - platformdirs=3.11.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.39=pyha770c72_0
+  - prompt_toolkit=3.0.39=hd8ed1ab_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pygments=2.16.1=pyhd8ed1ab_0
+  - python=3.8.18=h955ad1f_0
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.8=2_cp38
+  - pyzmq=25.1.0=py38h6a678d5_0
+  - readline=8.2=h5eee18b_0
+  - setuptools=68.0.0=py38h06a4308_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - tk=8.6.12=h1ccaba5_0
+  - tornado=6.1=py38h0a891b7_3
+  - traitlets=5.11.2=pyhd8ed1ab_0
+  - typing_extensions=4.8.0=pyha770c72_0
+  - wcwidth=0.2.8=pyhd8ed1ab_0
+  - wheel=0.38.4=py38h06a4308_0
+  - xz=5.4.2=h5eee18b_0
+  - zeromq=4.3.4=h2531618_0
+  - zlib=1.2.13=h5eee18b_0
+  - pip:
+    - accelerate==0.22.0
+    - aiohttp==3.8.5
+    - aiosignal==1.3.1
+    - alembic==1.12.0
+    - appdirs==1.4.4
+    - async-timeout==4.0.3
+    - attrs==23.1.0
+    - certifi==2022.12.7
+    - charset-normalizer==2.1.1
+    - click==8.1.7
+    - cmake==3.25.0
+    - coloredlogs==15.0.1
+    - colorlog==6.7.0
+    - contourpy==1.1.0
+    - cycler==0.11.0
+    - datasets==2.14.5
+    - diffusers==0.21.0
+    - dill==0.3.7
+    - docker-pycreds==0.4.0
+    - einops==0.6.1
+    - filelock==3.9.0
+    - flatbuffers==23.5.26
+    - fonttools==4.42.1
+    - frozenlist==1.4.0
+    - fsspec==2023.6.0
+    - gitdb==4.0.11
+    - gitpython==3.1.40
+    - greenlet==3.0.0
+    - huggingface-hub==0.17.1
+    - humanfriendly==10.0
+    - idna==3.4
+    - importlib-metadata==6.8.0
+    - importlib-resources==6.0.1
+    - jinja2==3.1.2
+    - joblib==1.3.2
+    - kiwisolver==1.4.5
+    - lightning-utilities==0.9.0
+    - lit==15.0.7
+    - lpips==0.1.4
+    - mako==1.2.4
+    - markupsafe==2.1.2
+    - matplotlib==3.7.3
+    - mpmath==1.2.1
+    - multidict==6.0.4
+    - multiprocess==0.70.15
+    - networkx==3.0
+    - numpy==1.24.1
+    - nvidia-cublas-cu11==11.10.3.66
+    - nvidia-cuda-cupti-cu11==11.7.101
+    - nvidia-cuda-nvrtc-cu11==11.7.99
+    - nvidia-cuda-runtime-cu11==11.7.99
+    - nvidia-cudnn-cu11==8.5.0.96
+    - nvidia-cufft-cu11==10.9.0.58
+    - nvidia-curand-cu11==10.2.10.91
+    - nvidia-cusolver-cu11==11.4.0.1
+    - nvidia-cusparse-cu11==11.7.4.91
+    - nvidia-nccl-cu11==2.14.3
+    - nvidia-nvtx-cu11==11.7.91
+    - onnx==1.15.0
+    - onnxruntime==1.16.1
+    - onnxruntime-gpu==1.16.1
+    - optuna==3.4.0
+    - packaging==23.1
+    - pandas==2.0.3
+    - pathtools==0.1.2
+    - pillow==9.3.0
+    - protobuf==4.24.4
+    - psutil==5.9.5
+    - pyarrow==13.0.0
+    - pyparsing==3.1.1
+    - pytz==2023.3.post1
+    - pyyaml==6.0.1
+    - regex==2023.8.8
+    - requests==2.28.1
+    - safetensors==0.3.3
+    - scikit-learn==1.3.1
+    - scipy==1.10.1
+    - sentry-sdk==1.32.0
+    - setproctitle==1.3.3
+    - smmap==5.0.1
+    - sqlalchemy==2.0.22
+    - sympy==1.11.1
+    - threadpoolctl==3.2.0
+    - tokenizers==0.13.3
+    - torch==2.0.1
+    - torch-fidelity==0.3.0
+    - torch-summary==1.4.5
+    - torchaudio==2.0.2
+    - torchmetrics==1.1.2
+    - torchvision==0.15.2
+    - tqdm==4.66.1
+    - transformers==4.33.1
+    - triton==2.0.0
+    - typing-extensions==4.4.0
+    - tzdata==2023.3
+    - urllib3==1.26.13
+    - wandb==0.15.12
+    - xxhash==3.3.0
+    - yarl==1.9.2
+    - zipp==3.16.2
+prefix: /opt/conda/envs/env

wandb/run-20231031_141809-v2q0e45b/files/config.yaml

@@ -0,0 +1,50 @@
+wandb_version: 1
+
+optimizer:
+  desc: null
+  value: Adam
+architecture:
+  desc: null
+  value: Efficientnet B0
+dataset:
+  desc: null
+  value: Food101
+epochs:
+  desc: null
+  value: 49
+_wandb:
+  desc: null
+  value:
+    python_version: 3.8.18
+    cli_version: 0.15.12
+    framework: huggingface
+    huggingface_version: 4.33.1
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1698747489.560302
+    t:
+      1:
+      - 1
+      - 11
+      - 35
+      - 41
+      - 49
+      - 55
+      - 71
+      2:
+      - 1
+      - 11
+      - 35
+      - 41
+      - 49
+      - 55
+      - 71
+      3:
+      - 16
+      - 23
+      4: 3.8.18
+      5: 0.15.12
+      6: 4.33.1
+      8:
+      - 5
+      13: linux-x86_64

wandb/run-20231031_141809-v2q0e45b/files/output.log

@@ -0,0 +1,24 @@
+[W 2023-10-31 14:20:40,518] Trial 0 failed with parameters: {'lr': 1.0615227326434047e-06, 'n_estimators': 49, 'optimizer': 'Adam'} because of the following error: TypeError("test_loop() got an unexpected keyword argument 'log_images'").
+Traceback (most recent call last):
+  File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial
+    value_or_values = func(trial)
+  File "hyperpara.py", line 68, in objective
+    val_loss, val_acc = engine.test_loop(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn,
+TypeError: test_loop() got an unexpected keyword argument 'log_images'
+[W 2023-10-31 14:20:40,519] Trial 0 failed with value None.
+Traceback (most recent call last):
+  File "hyperpara.py", line 110, in <module>
+    study.optimize(objective, n_trials=11)
+  File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/study.py", line 451, in optimize
+    _optimize(
+  File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 66, in _optimize
+    _optimize_sequential(
+  File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 163, in _optimize_sequential
+    frozen_trial = _run_trial(study, func, catch)
+  File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 251, in _run_trial
+    raise func_err
+  File "/opt/conda/envs/env/lib/python3.8/site-packages/optuna/study/_optimize.py", line 200, in _run_trial
+    value_or_values = func(trial)
+  File "hyperpara.py", line 68, in objective
+    val_loss, val_acc = engine.test_loop(model = model, dataloader = data.valid_dataloader, loss_fn = loss_fn,
+TypeError: test_loop() got an unexpected keyword argument 'log_images'

wandb/run-20231031_141809-v2q0e45b/files/requirements.txt

@@ -0,0 +1,133 @@
+accelerate==0.22.0
+aiohttp==3.8.5
+aiosignal==1.3.1
+alembic==1.12.0
+appdirs==1.4.4
+asttokens==2.4.0
+async-timeout==4.0.3
+attrs==23.1.0
+backcall==0.2.0
+backports.functools-lru-cache==1.6.5
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.7
+cmake==3.25.0
+coloredlogs==15.0.1
+colorlog==6.7.0
+comm==0.1.4
+contourpy==1.1.0
+cycler==0.11.0
+datasets==2.14.5
+debugpy==1.6.7
+decorator==5.1.1
+diffusers==0.21.0
+dill==0.3.7
+docker-pycreds==0.4.0
+einops==0.6.1
+entrypoints==0.4
+executing==1.2.0
+filelock==3.9.0
+flatbuffers==23.5.26
+fonttools==4.42.1
+frozenlist==1.4.0
+fsspec==2023.6.0
+gitdb==4.0.11
+gitpython==3.1.40
+greenlet==3.0.0
+huggingface-hub==0.17.1
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.0.1
+ipykernel==6.25.2
+ipython==8.12.0
+jedi==0.19.1
+jinja2==3.1.2
+joblib==1.3.2
+jupyter-client==7.3.4
+jupyter-core==5.4.0
+kiwisolver==1.4.5
+lightning-utilities==0.9.0
+lit==15.0.7
+lpips==0.1.4
+mako==1.2.4
+markupsafe==2.1.2
+matplotlib-inline==0.1.6
+matplotlib==3.7.3
+mpmath==1.2.1
+multidict==6.0.4
+multiprocess==0.70.15
+nest-asyncio==1.5.8
+networkx==3.0
+numpy==1.24.1
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+onnx==1.15.0
+onnxruntime-gpu==1.16.1
+onnxruntime==1.16.1
+optuna==3.4.0
+packaging==23.1
+pandas==2.0.3
+parso==0.8.3
+pathtools==0.1.2
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==9.3.0
+pip==23.2.1
+platformdirs==3.11.0
+prompt-toolkit==3.0.39
+protobuf==4.24.4
+psutil==5.9.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==13.0.0
+pygments==2.16.1
+pyparsing==3.1.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+pyyaml==6.0.1
+pyzmq==25.1.0
+regex==2023.8.8
+requests==2.28.1
+safetensors==0.3.3
+scikit-learn==1.3.1
+scipy==1.10.1
+sentry-sdk==1.32.0
+setproctitle==1.3.3
+setuptools==68.0.0
+six==1.16.0
+smmap==5.0.1
+sqlalchemy==2.0.22
+stack-data==0.6.2
+sympy==1.11.1
+threadpoolctl==3.2.0
+tokenizers==0.13.3
+torch-fidelity==0.3.0
+torch-summary==1.4.5
+torch==2.0.1
+torchaudio==2.0.2
+torchmetrics==1.1.2
+torchvision==0.15.2
+tornado==6.1
+tqdm==4.66.1
+traitlets==5.11.2
+transformers==4.33.1
+triton==2.0.0
+typing-extensions==4.4.0
+tzdata==2023.3
+urllib3==1.26.13
+wandb==0.15.12
+wcwidth==0.2.8
+wheel==0.38.4
+xxhash==3.3.0
+yarl==1.9.2
+zipp==3.16.2

wandb/run-20231031_141809-v2q0e45b/files/wandb-metadata.json

@@ -0,0 +1,254 @@
+{
+    "os": "Linux-4.15.0-55-generic-x86_64-with-glibc2.17",
+    "python": "3.8.18",
+    "heartbeatAt": "2023-10-31T10:18:11.054433",
+    "startedAt": "2023-10-31T10:18:09.552938",
+    "docker": null,
+    "cuda": null,
+    "args": [],
+    "state": "running",
+    "program": "hyperpara.py",
+    "codePathLocal": "hyperpara.py",
+    "codePath": "hyperpara.py",
+    "host": "76ecfeea2960",
+    "username": "root",
+    "executable": "/opt/conda/envs/env/bin/python",
+    "cpu_count": 20,
+    "cpu_count_logical": 40,
+    "cpu_freq": {
+        "current": 1.4950249999999998,
+        "min": 1200.0,
+        "max": 3600.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2.698,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.191,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.601,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.541,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.52,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.195,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.204,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.7,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.698,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.205,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.2,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.2,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.203,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.207,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.203,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.597,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.595,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.548,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.195,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.197,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.198,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.199,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 1.207,
+            "min": 1200.0,
+            "max": 3600.0
+        },
+        {
+            "current": 2.698,
+            "min": 1200.0,
+            "max": 3600.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 1758.95463180542,
+            "used": 866.7576103210449
+        }
+    },
+    "gpu": "Tesla V100-DGXS-32GB",
+    "gpu_count": 4,
+    "gpu_devices": [
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        },
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        },
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        },
+        {
+            "name": "Tesla V100-DGXS-32GB",
+            "memory_total": 34359738368
+        }
+    ],
+    "memory": {
+        "total": 251.8238754272461
+    }
+}